From 0800af0b8ae01fd808e54be53d2cf12eca1d0638 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 6 Jul 2026 00:16:38 -0700 Subject: [PATCH] =?UTF-8?q?perf(cli):=20TTFT=20round=202=20=E2=80=94=20liv?= =?UTF-8?q?e=20reasoning=20by=20default,=20partial-line=20streaming,=20pro?= =?UTF-8?q?mpt-build=20cache,=20stale=20budget-warning=20docs=20(#59389)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #59332 targeting the remaining PERCEIVED first-token latency (the wire streaming was already per-token; these fix what the user sees): 1. display.show_reasoning default ON. On thinking models the reasoning phase streams for tens of seconds; with the display off users stare at a spinner the whole time and read it as a stall. Flipped in DEFAULT_CONFIG, load_cli_config defaults, tui_gateway raw-YAML fallbacks, and the hermes setup status line (all four read sites kept in sync). Gateway per-platform defaults intentionally stay off — messaging chats shouldn't fill with thinking text. /reasoning hide still turns it off and persists. 2. Response box force-flushes long partial lines. _emit_stream_text only painted on newline, so a response opening with a long paragraph stayed invisible until the first \n — seconds of blank box. Now partial lines wrap at terminal width and paint as tokens arrive (mirrors the reasoning box's 80-char force-flush that existed since day one). Table blocks remain batch-aligned; no content loss at wrap boundaries (regression tests added). 3. hermes_time timezone resolution uses read_raw_config (mtime-cached + libyaml C loader) instead of a raw yaml.safe_load of config.yaml (~110-140ms measured) inside the FIRST system prompt build. First build drops 320ms -> ~155ms on a 200-skill install. 4. Stale docs: configuration.md (en+zh) still documented the 70%/90% [BUDGET WARNING] tool-result injections. Those were removed in April 2026 (c8aff7463) precisely because they hurt task completion; current behavior is exhaustion-message + one grace call, no mid-loop injection, no cache impact. Docs now describe reality. Verified: token-count compression decisions already use API-reported last_prompt_tokens (rough estimators are preflight-only and cost ~1.7ms even on 1.7MB histories — not worth touching). --- cli.py | 33 ++++++- hermes_cli/config.py | 8 +- hermes_time.py | 21 ++++- tests/cli/test_reasoning_command.py | 5 +- tests/cli/test_stream_partial_line_flush.py | 94 +++++++++++++++++++ tui_gateway/server.py | 6 +- website/docs/user-guide/configuration.md | 15 +-- .../current/user-guide/configuration.md | 15 +-- 8 files changed, 163 insertions(+), 34 deletions(-) create mode 100644 tests/cli/test_stream_partial_line_flush.py diff --git a/cli.py b/cli.py index 56a5b796d14..4dc546bac4e 100644 --- a/cli.py +++ b/cli.py @@ -455,7 +455,9 @@ def load_cli_config() -> Dict[str, Any]: "resume_max_assistant_chars": 200, "resume_max_assistant_lines": 3, "resume_skip_tool_only": True, - "show_reasoning": False, + # Live reasoning display default ON — keep in sync with + # hermes_cli/config.py DEFAULT_CONFIG (display.show_reasoning). + "show_reasoning": True, "reasoning_full": False, "streaming": True, "busy_input_mode": "interrupt", @@ -3708,7 +3710,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): # bell_on_complete: play terminal bell (\a) when agent finishes a response self.bell_on_complete = CLI_CONFIG["display"].get("bell_on_complete", False) # show_reasoning: display model thinking/reasoning before the response - self.show_reasoning = CLI_CONFIG["display"].get("show_reasoning", False) + self.show_reasoning = CLI_CONFIG["display"].get("show_reasoning", True) # reasoning_full: when reasoning display is on, print the post-response # recap box uncollapsed instead of clamping to the first 10 lines. self.reasoning_full = CLI_CONFIG["display"].get("reasoning_full", False) @@ -5836,6 +5838,33 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): line = _strip_markdown_syntax(line) _emit_one(line) + # Force-flush long partial lines so a response that opens with a + # long paragraph paints as tokens arrive instead of staying blank + # until the first newline (TTFT perception fix — the reasoning box + # has done this at 80 chars since day one; the response box never + # did). Wrap at the terminal's visible width so we only ever emit + # text that would have line-broken at that point anyway; the + # remainder stays buffered as the logical line's continuation. + # Table-shaped partials are exempt — they need the whole block for + # realignment (see the table side-buffer above). + if ( + self._stream_buf + and not self._in_stream_table + and not self._stream_buf.lstrip().startswith("|") + ): + wrap_w = max(40, _terminal_width_for_streaming()) + while len(self._stream_buf) >= wrap_w: + cut = self._stream_buf.rfind(" ", 0, wrap_w) + if cut <= 0: + cut = wrap_w # single unbreakable run — hard wrap + chunk, self._stream_buf = ( + self._stream_buf[:cut], + self._stream_buf[cut:].lstrip(" "), + ) + if self.final_response_markdown == "strip": + chunk = _strip_markdown_syntax(chunk) + _emit_one(chunk) + def _flush_stream(self) -> None: """Emit any remaining partial line from the stream buffer and close the box.""" # If we're still inside a "reasoning block" at end-of-stream, it was diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 70e88a227a2..050f1975db9 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -1701,7 +1701,11 @@ DEFAULT_CONFIG = { # dashboard. Set false to suppress the hint. "tui_agents_nudge": True, "bell_on_complete": False, - "show_reasoning": False, + # Stream the model's reasoning/thinking live before the response. + # Default ON: on thinking models the reasoning phase can run tens of + # seconds, and with this off the user stares at a spinner the whole + # time even though tokens are streaming. Set false for quiet output. + "show_reasoning": True, # When reasoning display is on, the post-response "Reasoning" recap box # collapses long thinking to the first 10 lines. Set true to print the # complete thinking text uncollapsed (live streaming is always full). @@ -7755,7 +7759,7 @@ def show_config(): print(color("◆ Display", Colors.CYAN, Colors.BOLD)) display = config.get('display', {}) print(f" Personality: {display.get('personality') or 'none'}") - print(f" Reasoning: {'on' if display.get('show_reasoning', False) else 'off'}") + print(f" Reasoning: {'on' if display.get('show_reasoning', True) else 'off'}") print(f" Bell: {'on' if display.get('bell_on_complete', False) else 'off'}") ump = display.get('user_message_preview', {}) if isinstance(display.get('user_message_preview', {}), dict) else {} ump_first = ump.get('first_lines', 2) diff --git a/hermes_time.py b/hermes_time.py index c956836ad44..08f865333ba 100644 --- a/hermes_time.py +++ b/hermes_time.py @@ -47,11 +47,22 @@ def _resolve_timezone_name() -> str: # 2. config.yaml ``timezone`` key try: - import yaml - config_path = get_config_path() - if config_path.exists(): - with open(config_path, encoding="utf-8") as f: - cfg = yaml.safe_load(f) or {} + # Prefer the shared cached raw-config reader (mtime/size-keyed cache + + # libyaml C loader) — a direct yaml.safe_load of a large config.yaml + # costs ~100ms+ and this used to run inside the FIRST system prompt + # build, on the time-to-first-token critical path. + try: + from hermes_cli.config import read_raw_config + cfg = read_raw_config() or {} + except Exception: + import yaml + config_path = get_config_path() + if config_path.exists(): + with open(config_path, encoding="utf-8") as f: + cfg = yaml.safe_load(f) or {} + else: + cfg = {} + if cfg: # Managed scope: an administrator can pin ``timezone`` too. Overlay # via the shared helper (fail-open) since this reads config.yaml directly. try: diff --git a/tests/cli/test_reasoning_command.py b/tests/cli/test_reasoning_command.py index 5091256a399..08185689e73 100644 --- a/tests/cli/test_reasoning_command.py +++ b/tests/cli/test_reasoning_command.py @@ -550,7 +550,10 @@ class TestConfigDefault(unittest.TestCase): from hermes_cli.config import DEFAULT_CONFIG display = DEFAULT_CONFIG.get("display", {}) self.assertIn("show_reasoning", display) - self.assertFalse(display["show_reasoning"]) + # Default ON (July 2026 TTFT-perception change): thinking models + # stream reasoning for tens of seconds; hiding it left users staring + # at a spinner. The key must exist and be a bool. + self.assertTrue(display["show_reasoning"]) class TestCommandRegistered(unittest.TestCase): diff --git a/tests/cli/test_stream_partial_line_flush.py b/tests/cli/test_stream_partial_line_flush.py new file mode 100644 index 00000000000..7d5e96a60a2 --- /dev/null +++ b/tests/cli/test_stream_partial_line_flush.py @@ -0,0 +1,94 @@ +"""Streaming display force-flush: long partial lines must paint before the +first newline arrives (TTFT-perception fix, July 2026). + +Previously ``_emit_stream_text`` only emitted on ``"\\n"``, so a response +opening with a long paragraph stayed invisible until the model produced a +newline — seconds of blank box on slow models. Now partial lines are +force-flushed at terminal width (mirroring the reasoning box's 80-char rule). +""" +import os +import re +import sys + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + + +def _strip_ansi(s: str) -> str: + return re.sub(r"\x1b\[[0-9;]*m", "", s) + + +@pytest.fixture +def cli_stub(monkeypatch): + from cli import HermesCLI + import cli as climod + + cli = HermesCLI.__new__(HermesCLI) + cli.show_reasoning = False + cli.final_response_markdown = "raw" + cli.show_timestamps = False + cli._reset_stream_state() + + emitted = [] + monkeypatch.setattr(climod, "_cprint", lambda s: emitted.append(s)) + # Deterministic width regardless of the test runner's terminal + monkeypatch.setattr(climod, "_terminal_width_for_streaming", lambda: 74) + return cli, emitted + + +class TestPartialLineForceFlush: + def test_long_paragraph_paints_before_first_newline(self, cli_stub): + cli, emitted = cli_stub + text = ( + "This is a long opening paragraph that would previously sit " + "invisible in the buffer until the model finally produced a " + "newline character, which on a slow model could take seconds. " + ) * 3 + for i in range(0, len(text), 12): + cli._stream_delta(text[i : i + 12]) + # Box header + several wrapped lines painted with NO newline seen yet + assert len(emitted) > 3 + + def test_no_content_lost_across_wraps(self, cli_stub): + cli, emitted = cli_stub + words = [f"word{i}" for i in range(120)] + text = " ".join(words) + for i in range(0, len(text), 7): + cli._stream_delta(text[i : i + 7]) + cli._flush_stream() + plain = " ".join(_strip_ansi("\n".join(emitted)).split()) + for w in words: + assert w in plain, f"lost {w} at a wrap boundary" + + def test_short_partial_stays_buffered(self, cli_stub): + cli, emitted = cli_stub + cli._stream_delta("short line, no newline") + # Under wrap width: the box header may open, but the text itself + # stays buffered until a newline or the width threshold. + plain = _strip_ansi("\n".join(emitted)) + assert "short line" not in plain + assert cli._stream_buf == "short line, no newline" + + def test_table_rows_not_force_flushed(self, cli_stub): + cli, emitted = cli_stub + # A long partial table row must stay buffered for block realignment + row = "| " + " | ".join(f"cell{i}" for i in range(20)) + " |" + cli._stream_delta(row) # no newline + plain = _strip_ansi("\n".join(emitted)) + assert "cell19" not in plain + + def test_newline_lines_still_emit_normally(self, cli_stub): + cli, emitted = cli_stub + cli._stream_delta("line one\nline two\n") + plain = _strip_ansi("\n".join(emitted)) + assert "line one" in plain + assert "line two" in plain + + def test_unbreakable_run_hard_wraps(self, cli_stub): + cli, emitted = cli_stub + blob = "x" * 300 # no spaces + cli._stream_delta(blob) + cli._flush_stream() + plain = _strip_ansi("\n".join(emitted)) + assert plain.count("x") == 300 diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 7c9287aaab0..2721783d84e 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -2364,7 +2364,9 @@ def _load_provider_routing() -> dict: def _load_show_reasoning() -> bool: - return bool((_load_cfg().get("display") or {}).get("show_reasoning", False)) + # Fallback True — keep in sync with DEFAULT_CONFIG display.show_reasoning + # (this loader reads the raw user YAML without the DEFAULT_CONFIG merge). + return bool((_load_cfg().get("display") or {}).get("show_reasoning", True)) def _load_memory_notifications() -> str: @@ -10828,7 +10830,7 @@ def _(rid, params: dict) -> dict: effort = str(raw_effort or "medium") display = ( "show" - if bool((cfg.get("display") or {}).get("show_reasoning", False)) + if bool((cfg.get("display") or {}).get("show_reasoning", True)) else "hide" ) return _ok(rid, {"value": effort, "display": display}) diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 0c7132b1448..46afc11bcfd 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -823,16 +823,11 @@ Plugin engines are **never auto-activated** — you must explicitly set `context See [Memory Providers](/user-guide/features/memory-providers) for the analogous single-select system for memory plugins. -## Iteration Budget Pressure +## Iteration Budget -When the agent is working on a complex task with many tool calls, it can burn through its iteration budget (default: 90 turns) without realizing it's running low. Budget pressure automatically warns the model as it approaches the limit: +When the agent is working on a complex task with many tool calls, it can burn through its iteration budget (default: 90 turns). Hermes does **not** inject mid-task pressure warnings — earlier builds warned the model at 70%/90% budget, which caused models to abandon complex tasks prematurely and was removed in April 2026. -| Threshold | Level | What the model sees | -|-----------|-------|---------------------| -| **70%** | Caution | `[BUDGET: 63/90. 27 iterations left. Start consolidating.]` | -| **90%** | Warning | `[BUDGET WARNING: 81/90. Only 9 left. Respond NOW.]` | - -Warnings are injected into the last tool result's JSON (as a `_budget_warning` field) rather than as separate messages — this preserves prompt caching and doesn't disrupt the conversation structure. +Instead, when the budget is actually exhausted (90/90), Hermes injects one message asking the model to wrap up and allows a single **grace call** so it can deliver a final response. If that grace call still doesn't produce text, the agent is asked to summarise what it accomplished. ```yaml agent: @@ -840,9 +835,7 @@ agent: api_max_retries: 3 # Retries per provider before fallback engages (default: 3) ``` -Budget pressure is enabled by default. The agent sees warnings naturally as part of tool results, encouraging it to consolidate its work and deliver a response before running out of iterations. - -When the iteration budget is fully exhausted, the CLI shows a notification to the user: `⚠ Iteration budget reached (90/90) — response may be incomplete`. If the budget runs out during active work, the agent generates a summary of what was accomplished before stopping. +When the iteration budget is fully exhausted, the CLI shows a notification to the user: `⚠ Iteration budget reached (90/90) — response may be incomplete`. `agent.api_max_retries` controls how many times Hermes retries a provider API call on transient errors (rate limits, connection drops, 5xx) **before** fallback-provider switching engages. The default is `3` — four attempts total. If you have [fallback providers](/user-guide/features/fallback-providers) configured and want to fail over faster, drop this to `0` so the first transient error on your primary immediately hands off to the fallback instead of churning retries against the flaky endpoint. diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/configuration.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/configuration.md index cd3748530d3..c3ea44ef957 100644 --- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/configuration.md +++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/user-guide/configuration.md @@ -635,16 +635,11 @@ context: 有关内存插件的类似单选系统，请参阅[内存 Providers](/user-guide/features/memory-providers)。 -## 迭代预算压力 +## 迭代预算 -当 agent 在处理具有许多工具调用的复杂任务时，它可能会在没有意识到预算不足的情况下耗尽其迭代预算（默认：90 轮）。预算压力会在模型接近限制时自动发出警告： +当 agent 在处理具有许多工具调用的复杂任务时，它可能会耗尽其迭代预算（默认：90 轮）。Hermes **不会**在任务中途注入压力警告 —— 早期版本会在预算达到 70%/90% 时警告模型，这会导致模型过早放弃复杂任务，该机制已于 2026 年 4 月移除。 -| 阈值 | 级别 | 模型看到的内容 | -|-----------|-------|---------------------| -| **70%** | 注意 | `[BUDGET: 63/90. 27 iterations left. Start consolidating.]` | -| **90%** | 警告 | `[BUDGET WARNING: 81/90. Only 9 left. Respond NOW.]` | - -警告注入到最后一个工具结果的 JSON 中（作为 `_budget_warning` 字段），而不是作为单独的消息 —— 这保留了 prompt 缓存，不会破坏对话结构。 +取而代之的是，当预算真正耗尽（90/90）时，Hermes 注入一条消息要求模型收尾，并允许一次**宽限调用**以便其给出最终响应。如果该宽限调用仍未产生文本，则会要求 agent 总结已完成的工作。 ```yaml agent: @@ -652,9 +647,7 @@ agent: api_max_retries: 3 # 回退启动前每个 provider 的重试次数（默认：3） ``` -预算压力默认启用。Agent 自然地将警告视为工具结果的一部分，鼓励它在耗尽迭代之前整合工作并提供响应。 - -当迭代预算完全耗尽时，CLI 向用户显示通知：`⚠ Iteration budget reached (90/90) — response may be incomplete`。如果预算在活跃工作期间耗尽，agent 会在停止前生成已完成内容的摘要。 +当迭代预算完全耗尽时，CLI 向用户显示通知：`⚠ Iteration budget reached (90/90) — response may be incomplete`。 `agent.api_max_retries` 控制 Hermes 在回退 provider 切换启动**之前**对瞬时错误（速率限制、连接断开、5xx）重试 provider API 调用的次数。默认为 `3` —— 总共四次尝试。如果您配置了[回退 providers](/user-guide/features/fallback-providers) 并希望更快地故障转移，请将其降至 `0`，这样主 provider 上的第一个瞬时错误会立即切换到回退，而不是对不稳定的端点进行重试。