hermes-agent/tests/agent/test_onboarding.py
Teknium 83c1c201f6
feat(onboarding): contextual first-touch hints for /busy and /verbose (#16046)
Instead of a blocking first-run questionnaire, show a one-time hint the first
time the user hits each behavior fork:

1. First message while the agent is working — appends a hint to the busy-ack
   explaining the /busy queue vs /busy interrupt knob, phrased to match the
   mode that was just applied (don't tell a queue-mode user to switch to
   queue).

2. First tool that runs for >= 30s in the noisiest progress mode
   (tool_progress: all) — prints a hint about /verbose to cycle display
   modes (all -> new -> off -> verbose). Gated on /verbose actually being
   usable on the surface: always shown on CLI; on gateway only shown when
   display.tool_progress_command is enabled.

Each hint is latched in config.yaml under onboarding.seen.<flag>, so it
fires exactly once per install across CLI, gateway, and cron, then never
again. Users can wipe the section to re-see hints.

New:
- agent/onboarding.py — is_seen / mark_seen / hint strings, shared by
  both CLI and gateway.
- onboarding.seen in DEFAULT_CONFIG (hermes_cli/config.py) and in
  load_cli_config defaults (cli.py). No _config_version bump — deep
  merge handles new keys.

Wired:
- gateway/run.py: _handle_active_session_busy_message appends the hint
  after building the ack.  progress_callback tracks tool.completed
  duration and queues the tool-progress hint into the progress bubble.
- cli.py: CLI input loop appends the busy-input hint on the first busy
  Enter; _on_tool_progress appends the tool-progress hint on the first
  >=30s tool completion.  In-memory CLI_CONFIG is also updated so
  subsequent fires in the same process are suppressed immediately.

All writes go through atomic_yaml_write and are wrapped in try/except
so onboarding can never break the input/busy-ack paths.
2026-04-26 06:06:27 -07:00

164 lines
5.7 KiB
Python

"""Tests for agent/onboarding.py — contextual first-touch hint helpers."""
from __future__ import annotations
import yaml
import pytest
from agent.onboarding import (
BUSY_INPUT_FLAG,
TOOL_PROGRESS_FLAG,
busy_input_hint_cli,
busy_input_hint_gateway,
is_seen,
mark_seen,
tool_progress_hint_cli,
tool_progress_hint_gateway,
)
class TestIsSeen:
def test_empty_config_unseen(self):
assert is_seen({}, BUSY_INPUT_FLAG) is False
def test_missing_onboarding_unseen(self):
assert is_seen({"display": {}}, BUSY_INPUT_FLAG) is False
def test_onboarding_not_dict_unseen(self):
assert is_seen({"onboarding": "nope"}, BUSY_INPUT_FLAG) is False
def test_seen_dict_missing_flag(self):
assert is_seen({"onboarding": {"seen": {}}}, BUSY_INPUT_FLAG) is False
def test_seen_flag_true(self):
cfg = {"onboarding": {"seen": {BUSY_INPUT_FLAG: True}}}
assert is_seen(cfg, BUSY_INPUT_FLAG) is True
def test_seen_flag_falsy(self):
cfg = {"onboarding": {"seen": {BUSY_INPUT_FLAG: False}}}
assert is_seen(cfg, BUSY_INPUT_FLAG) is False
def test_other_flags_isolated(self):
cfg = {"onboarding": {"seen": {BUSY_INPUT_FLAG: True}}}
assert is_seen(cfg, TOOL_PROGRESS_FLAG) is False
class TestMarkSeen:
def test_creates_missing_file_and_sets_flag(self, tmp_path):
cfg_path = tmp_path / "config.yaml"
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
loaded = yaml.safe_load(cfg_path.read_text())
assert loaded["onboarding"]["seen"][BUSY_INPUT_FLAG] is True
def test_preserves_other_config(self, tmp_path):
cfg_path = tmp_path / "config.yaml"
cfg_path.write_text(yaml.safe_dump({
"model": {"default": "claude-sonnet-4.6"},
"display": {"skin": "default"},
}))
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
loaded = yaml.safe_load(cfg_path.read_text())
assert loaded["model"]["default"] == "claude-sonnet-4.6"
assert loaded["display"]["skin"] == "default"
assert loaded["onboarding"]["seen"][BUSY_INPUT_FLAG] is True
def test_preserves_other_seen_flags(self, tmp_path):
cfg_path = tmp_path / "config.yaml"
cfg_path.write_text(yaml.safe_dump({
"onboarding": {"seen": {TOOL_PROGRESS_FLAG: True}},
}))
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
loaded = yaml.safe_load(cfg_path.read_text())
assert loaded["onboarding"]["seen"][TOOL_PROGRESS_FLAG] is True
assert loaded["onboarding"]["seen"][BUSY_INPUT_FLAG] is True
def test_idempotent(self, tmp_path):
cfg_path = tmp_path / "config.yaml"
mark_seen(cfg_path, BUSY_INPUT_FLAG)
first = cfg_path.read_text()
# Second call must be a no-op on-disk content (file may be touched,
# but the YAML contents should be identical).
mark_seen(cfg_path, BUSY_INPUT_FLAG)
second = cfg_path.read_text()
assert yaml.safe_load(first) == yaml.safe_load(second)
def test_handles_non_dict_onboarding(self, tmp_path):
cfg_path = tmp_path / "config.yaml"
cfg_path.write_text(yaml.safe_dump({"onboarding": "corrupted"}))
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
loaded = yaml.safe_load(cfg_path.read_text())
assert loaded["onboarding"]["seen"][BUSY_INPUT_FLAG] is True
def test_handles_non_dict_seen(self, tmp_path):
cfg_path = tmp_path / "config.yaml"
cfg_path.write_text(yaml.safe_dump({"onboarding": {"seen": "corrupted"}}))
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
loaded = yaml.safe_load(cfg_path.read_text())
assert loaded["onboarding"]["seen"][BUSY_INPUT_FLAG] is True
class TestHintMessages:
def test_busy_input_hint_gateway_interrupt(self):
msg = busy_input_hint_gateway("interrupt")
assert "/busy queue" in msg
assert "interrupted" in msg.lower()
def test_busy_input_hint_gateway_queue(self):
msg = busy_input_hint_gateway("queue")
assert "/busy interrupt" in msg
assert "queued" in msg.lower()
def test_busy_input_hint_cli_interrupt(self):
msg = busy_input_hint_cli("interrupt")
assert "/busy queue" in msg
def test_busy_input_hint_cli_queue(self):
msg = busy_input_hint_cli("queue")
assert "/busy interrupt" in msg
def test_tool_progress_hints_mention_verbose(self):
assert "/verbose" in tool_progress_hint_gateway()
assert "/verbose" in tool_progress_hint_cli()
def test_hints_are_not_empty(self):
for hint in (
busy_input_hint_gateway("queue"),
busy_input_hint_gateway("interrupt"),
busy_input_hint_cli("queue"),
busy_input_hint_cli("interrupt"),
tool_progress_hint_gateway(),
tool_progress_hint_cli(),
):
assert hint.strip()
class TestRoundTrip:
"""After mark_seen, is_seen on the re-loaded config must return True."""
def test_mark_then_is_seen(self, tmp_path):
cfg_path = tmp_path / "config.yaml"
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
loaded = yaml.safe_load(cfg_path.read_text())
assert is_seen(loaded, BUSY_INPUT_FLAG) is True
assert is_seen(loaded, TOOL_PROGRESS_FLAG) is False
def test_mark_both_flags_independently(self, tmp_path):
cfg_path = tmp_path / "config.yaml"
mark_seen(cfg_path, BUSY_INPUT_FLAG)
mark_seen(cfg_path, TOOL_PROGRESS_FLAG)
loaded = yaml.safe_load(cfg_path.read_text())
assert is_seen(loaded, BUSY_INPUT_FLAG) is True
assert is_seen(loaded, TOOL_PROGRESS_FLAG) is True