"""Tests for the ``hermes prompt-size`` diagnostic (issue #34667).""" import json import sys from pathlib import Path from types import SimpleNamespace import pytest from hermes_cli.prompt_size import ( _SKILLS_BLOCK_RE, _build_inspection_agent, _compute_skills_breakdown, compute_prompt_breakdown, render_breakdown, ) def _seed_memory(hermes_home, memory_text="", user_text=""): mem_dir = hermes_home / "memories" mem_dir.mkdir(parents=True, exist_ok=True) if memory_text: (mem_dir / "MEMORY.md").write_text(memory_text, encoding="utf-8") if user_text: (mem_dir / "USER.md").write_text(user_text, encoding="utf-8") def _seed_skill(hermes_home, name, description): skill_dir = hermes_home / "skills" / "demo" / name skill_dir.mkdir(parents=True, exist_ok=True) (skill_dir / "SKILL.md").write_text( f"---\nname: {name}\ndescription: {description}\n---\n# {name}\nbody\n", encoding="utf-8", ) @pytest.fixture def isolated_home(tmp_path, monkeypatch): hermes_home = tmp_path / ".hermes" hermes_home.mkdir() monkeypatch.setenv("HERMES_HOME", str(hermes_home)) monkeypatch.chdir(tmp_path) # avoid picking up the repo's AGENTS.md return hermes_home def test_breakdown_keys_and_shape(isolated_home): """The breakdown exposes every documented key with int byte/char counts.""" data = compute_prompt_breakdown("cli") assert set(data) >= { "platform", "model", "system_prompt", "skills_index", "memory", "user_profile", "tools", "sections", } assert data["platform"] == "cli" for key in ("system_prompt", "skills_index", "memory", "user_profile"): assert data[key]["bytes"] >= 0 assert data[key]["chars"] >= 0 assert data["tools"]["count"] >= 0 assert data["tools"]["json_bytes"] >= 0 # System prompt is non-trivial even with empty home (identity + guidance). assert data["system_prompt"]["bytes"] > 0 def test_runs_offline_without_credentials(isolated_home, monkeypatch): """No provider credentials configured → still produces a breakdown.""" for var in ("OPENROUTER_API_KEY", "OPENAI_API_KEY", "NOUS_API_KEY", "ANTHROPIC_API_KEY"): monkeypatch.delenv(var, raising=False) data = compute_prompt_breakdown("cli") assert data["system_prompt"]["bytes"] > 0 def test_inspection_agent_uses_resolved_platform_toolsets(monkeypatch): """Inspection must match real CLI tool resolution, including disables.""" captured = {} class FakeAIAgent: def __init__(self, **kwargs): captured.update(kwargs) cfg = { "model": {"default": "test/model"}, "agent": {"disabled_toolsets": ["memory"]}, } monkeypatch.setitem( sys.modules, "run_agent", SimpleNamespace(AIAgent=FakeAIAgent), ) monkeypatch.setattr("hermes_cli.config.load_config", lambda: cfg) monkeypatch.setattr( "hermes_cli.tools_config._get_platform_tools", lambda passed_cfg, platform: {"terminal", "file"}, ) _build_inspection_agent("cli") assert captured["model"] == "test/model" assert captured["platform"] == "cli" assert captured["enabled_toolsets"] == ["file", "terminal"] assert captured["disabled_toolsets"] == ["memory"] def test_blank_slate_prompt_size_counts_only_minimal_tools(isolated_home): """Blank Slate prompt-size should report file + terminal schemas only.""" from hermes_cli.config import save_config from hermes_cli.setup import ( _blank_slate_minimal_toolsets, _blank_slate_minimize_config, ) cfg = {"model": {"default": "MiniMax-M2.7"}} _blank_slate_minimal_toolsets(cfg) _blank_slate_minimize_config(cfg) save_config(cfg) data = compute_prompt_breakdown("cli") assert data["tools"]["count"] == 6 def test_skills_index_reflects_installed_skills(isolated_home): """Installing a skill makes the skills-index block non-empty. Note: the skills prompt is cached per-process (in-process LRU + disk snapshot), so we seed the skill BEFORE the first build rather than comparing before/after within one process. """ _seed_skill(isolated_home, "hello", "a demo skill for size testing") data = compute_prompt_breakdown("cli") assert data["skills_index"]["bytes"] > 0 def test_memory_and_profile_are_attributed(isolated_home): """Memory and user-profile blocks are measured separately.""" _seed_memory( isolated_home, memory_text="Project uses pytest.\n", user_text="User is a developer.\n", ) data = compute_prompt_breakdown("cli") assert data["memory"]["bytes"] > 0 assert data["user_profile"]["bytes"] > 0 def test_skills_block_regex_matches_tagged_block(): text = "preamble\n\n cat:\n - a: b\n\ntail" m = _SKILLS_BLOCK_RE.search(text) assert m is not None assert m.group(0).startswith("") assert m.group(0).endswith("") def test_toolsets_breakdown_reconciles_and_sorted(isolated_home): """Per-toolset schema bytes attribute every tool exactly once. Each resolved tool belongs to one registry toolset, so the grand total of per-toolset json bytes equals the whole-array total minus JSON framing (``2 * count`` bytes: brackets + ``", "`` separators between items). """ data = compute_prompt_breakdown("cli") toolsets = data["toolsets_breakdown"] assert toolsets # CLI always resolves at least terminal + file for ts in toolsets: assert set(ts) >= {"toolset", "tool_count", "json_bytes"} assert ts["tool_count"] >= 1 assert ts["json_bytes"] > 0 # Sorted largest-first. byte_sizes = [ts["json_bytes"] for ts in toolsets] assert byte_sizes == sorted(byte_sizes, reverse=True) # Every tool attributed to exactly one toolset. assert sum(ts["tool_count"] for ts in toolsets) == data["tools"]["count"] # Bytes reconcile to the existing whole-array total. grand = sum(ts["json_bytes"] for ts in toolsets) assert grand == data["tools"]["json_bytes"] - 2 * data["tools"]["count"] def test_skills_breakdown_shape_sorted_and_attributed(isolated_home): """Per-skill breakdown reports index-line + on-disk SKILL.md bytes. Seeded before the first build (skills prompt is cached per-process). """ _seed_skill(isolated_home, "small-skill", "short desc") _seed_skill(isolated_home, "big-skill", "a much longer description " * 20) data = compute_prompt_breakdown("cli") skills = data["skills_breakdown"] names = {s["name"] for s in skills} assert {"small-skill", "big-skill"} <= names for s in skills: assert set(s) >= {"name", "index_line_bytes", "skill_md_bytes", "path"} assert s["index_line_bytes"] > 0 # Sorted largest-first by on-disk SKILL.md size. md_sizes = [s["skill_md_bytes"] or 0 for s in skills] assert md_sizes == sorted(md_sizes, reverse=True) # On-disk bytes match the real file; big-skill's SKILL.md is the larger. by_name = {s["name"]: s for s in skills} big = by_name["big-skill"] assert big["path"] and Path(big["path"]).stat().st_size == big["skill_md_bytes"] assert big["skill_md_bytes"] > by_name["small-skill"]["skill_md_bytes"] # Per-skill index lines are a subset of the whole block, # so they never exceed it (on-disk SKILL.md bytes are separate and don't). assert sum(s["index_line_bytes"] for s in skills) <= data["skills_index"]["bytes"] def test_skills_breakdown_unmapped_name_is_none(): """A skill line with no matching SKILL.md on disk reports None, not a crash.""" block = ( "\n" " demo:\n" " - phantom-skill: not on disk\n" "\n" ) entries = _compute_skills_breakdown(block) assert len(entries) == 1 assert entries[0]["name"] == "phantom-skill" assert entries[0]["skill_md_bytes"] is None assert entries[0]["path"] == "" assert entries[0]["index_line_bytes"] > 0 def test_skills_breakdown_parses_namespaced_names(): """Namespaced names (``ns:skill``) survive the ``name: desc`` split.""" block = ( "\n" " plugins:\n" " - codex:rescue: rescue helper\n" "\n" ) entries = _compute_skills_breakdown(block) assert [e["name"] for e in entries] == ["codex:rescue"] def test_skills_breakdown_attributes_demoted_category_shared_line(isolated_home): """A real posture-demoted category retains every skill in the breakdown.""" from agent.prompt_builder import build_skills_system_prompt _seed_skill(isolated_home, "alpha-skill", "alpha description") _seed_skill(isolated_home, "beta-skill", "beta description") prompt = build_skills_system_prompt(compact_categories=frozenset({"demo"})) skills_match = _SKILLS_BLOCK_RE.search(prompt) assert skills_match is not None skills_block = skills_match.group(0) shared_line = next( line for line in skills_block.splitlines() if "demo [names only]" in line ) entries = _compute_skills_breakdown(skills_block) by_name = {entry["name"]: entry for entry in entries} assert set(by_name) == {"alpha-skill", "beta-skill"} shared_line_bytes = len(shared_line.encode("utf-8")) assert sum(entry["index_line_bytes"] for entry in entries) == shared_line_bytes for entry in entries: assert entry["index_line_total_bytes"] == shared_line_bytes assert entry["index_line_shared_bytes"] > 0 assert entry["index_line_skill_count"] == 2 def test_render_includes_per_component_tables(isolated_home): """The rendered report gains the two new sorted tables (additive).""" _seed_skill(isolated_home, "demo-skill", "a demo skill") data = compute_prompt_breakdown("cli") out = render_breakdown(data) assert "Toolsets by size" in out assert "Skills by size" in out def test_render_breakdown_is_plain_text(isolated_home): data = compute_prompt_breakdown("cli") out = render_breakdown(data) assert "System prompt total" in out assert "skills index" in out assert "Tool schemas" in out # Plain text — no JSON braces leaking in. assert not out.strip().startswith("{") def test_json_serializable(isolated_home): data = compute_prompt_breakdown("cli") # Round-trips cleanly for ``--json`` output. assert json.loads(json.dumps(data)) == json.loads(json.dumps(data))