diff --git a/agent/subdirectory_hints.py b/agent/subdirectory_hints.py new file mode 100644 index 0000000000..a6ca2adc51 --- /dev/null +++ b/agent/subdirectory_hints.py @@ -0,0 +1,219 @@ +"""Progressive subdirectory hint discovery. + +As the agent navigates into subdirectories via tool calls (read_file, terminal, +search_files, etc.), this module discovers and loads project context files +(AGENTS.md, CLAUDE.md, .cursorrules) from those directories. Discovered hints +are appended to the tool result so the model gets relevant context at the moment +it starts working in a new area of the codebase. + +This complements the startup context loading in ``prompt_builder.py`` which only +loads from the CWD. Subdirectory hints are discovered lazily and injected into +the conversation without modifying the system prompt (preserving prompt caching). + +Inspired by Block/goose's SubdirectoryHintTracker. +""" + +import logging +import os +import re +import shlex +from pathlib import Path +from typing import Dict, Any, Optional, Set + +from agent.prompt_builder import _scan_context_content + +logger = logging.getLogger(__name__) + +# Context files to look for in subdirectories, in priority order. +# Same filenames as prompt_builder.py but we load ALL found (not first-wins) +# since different subdirectories may use different conventions. +_HINT_FILENAMES = [ + "AGENTS.md", "agents.md", + "CLAUDE.md", "claude.md", + ".cursorrules", +] + +# Maximum chars per hint file to prevent context bloat +_MAX_HINT_CHARS = 8_000 + +# Tool argument keys that typically contain file paths +_PATH_ARG_KEYS = {"path", "file_path", "workdir"} + +# Tools that take shell commands where we should extract paths +_COMMAND_TOOLS = {"terminal"} + +# How many parent directories to walk up when looking for hints. +# Prevents scanning all the way to / for deeply nested paths. +_MAX_ANCESTOR_WALK = 5 + +class SubdirectoryHintTracker: + """Track which directories the agent visits and load hints on first access. + + Usage:: + + tracker = SubdirectoryHintTracker(working_dir="/path/to/project") + + # After each tool call: + hints = tracker.check_tool_call("read_file", {"path": "backend/src/main.py"}) + if hints: + tool_result += hints # append to the tool result string + """ + + def __init__(self, working_dir: Optional[str] = None): + self.working_dir = Path(working_dir or os.getcwd()).resolve() + self._loaded_dirs: Set[Path] = set() + # Pre-mark the working dir as loaded (startup context handles it) + self._loaded_dirs.add(self.working_dir) + + def check_tool_call( + self, + tool_name: str, + tool_args: Dict[str, Any], + ) -> Optional[str]: + """Check tool call arguments for new directories and load any hint files. + + Returns formatted hint text to append to the tool result, or None. + """ + dirs = self._extract_directories(tool_name, tool_args) + if not dirs: + return None + + all_hints = [] + for d in dirs: + hints = self._load_hints_for_directory(d) + if hints: + all_hints.append(hints) + + if not all_hints: + return None + + return "\n\n" + "\n\n".join(all_hints) + + def _extract_directories( + self, tool_name: str, args: Dict[str, Any] + ) -> list: + """Extract directory paths from tool call arguments.""" + candidates: Set[Path] = set() + + # Direct path arguments + for key in _PATH_ARG_KEYS: + val = args.get(key) + if isinstance(val, str) and val.strip(): + self._add_path_candidate(val, candidates) + + # Shell commands — extract path-like tokens + if tool_name in _COMMAND_TOOLS: + cmd = args.get("command", "") + if isinstance(cmd, str): + self._extract_paths_from_command(cmd, candidates) + + return list(candidates) + + def _add_path_candidate(self, raw_path: str, candidates: Set[Path]): + """Resolve a raw path and add its directory + ancestors to candidates. + + Walks up from the resolved directory toward the filesystem root, + stopping at the first directory already in ``_loaded_dirs`` (or after + ``_MAX_ANCESTOR_WALK`` levels). This ensures that reading + ``project/src/main.py`` discovers ``project/AGENTS.md`` even when + ``project/src/`` has no hint files of its own. + """ + try: + p = Path(raw_path).expanduser() + if not p.is_absolute(): + p = self.working_dir / p + p = p.resolve() + # Use parent if it's a file path (has extension or doesn't exist as dir) + if p.suffix or (p.exists() and p.is_file()): + p = p.parent + # Walk up ancestors — stop at already-loaded or root + for _ in range(_MAX_ANCESTOR_WALK): + if p in self._loaded_dirs: + break + if self._is_valid_subdir(p): + candidates.add(p) + parent = p.parent + if parent == p: + break # filesystem root + p = parent + except (OSError, ValueError): + pass + + def _extract_paths_from_command(self, cmd: str, candidates: Set[Path]): + """Extract path-like tokens from a shell command string.""" + try: + tokens = shlex.split(cmd) + except ValueError: + tokens = cmd.split() + + for token in tokens: + # Skip flags + if token.startswith("-"): + continue + # Must look like a path (contains / or .) + if "/" not in token and "." not in token: + continue + # Skip URLs + if token.startswith(("http://", "https://", "git@")): + continue + self._add_path_candidate(token, candidates) + + def _is_valid_subdir(self, path: Path) -> bool: + """Check if path is a valid directory to scan for hints.""" + if not path.is_dir(): + return False + if path in self._loaded_dirs: + return False + return True + + def _load_hints_for_directory(self, directory: Path) -> Optional[str]: + """Load hint files from a directory. Returns formatted text or None.""" + self._loaded_dirs.add(directory) + + found_hints = [] + for filename in _HINT_FILENAMES: + hint_path = directory / filename + if not hint_path.is_file(): + continue + try: + content = hint_path.read_text(encoding="utf-8").strip() + if not content: + continue + # Same security scan as startup context loading + content = _scan_context_content(content, filename) + if len(content) > _MAX_HINT_CHARS: + content = ( + content[:_MAX_HINT_CHARS] + + f"\n\n[...truncated {filename}: {len(content):,} chars total]" + ) + # Best-effort relative path for display + rel_path = str(hint_path) + try: + rel_path = str(hint_path.relative_to(self.working_dir)) + except ValueError: + try: + rel_path = str(hint_path.relative_to(Path.home())) + rel_path = "~/" + rel_path + except ValueError: + pass # keep absolute + found_hints.append((rel_path, content)) + # First match wins per directory (like startup loading) + break + except Exception as exc: + logger.debug("Could not read %s: %s", hint_path, exc) + + if not found_hints: + return None + + sections = [] + for rel_path, content in found_hints: + sections.append( + f"[Subdirectory context discovered: {rel_path}]\n{content}" + ) + + logger.debug( + "Loaded subdirectory hints from %s: %s", + directory, + [h[0] for h in found_hints], + ) + return "\n\n".join(sections) diff --git a/run_agent.py b/run_agent.py index ecb628d729..050678928d 100644 --- a/run_agent.py +++ b/run_agent.py @@ -88,6 +88,7 @@ from agent.model_metadata import ( save_context_length, is_local_endpoint, ) from agent.context_compressor import ContextCompressor +from agent.subdirectory_hints import SubdirectoryHintTracker from agent.prompt_caching import apply_anthropic_cache_control from agent.prompt_builder import build_skills_system_prompt, build_context_files_prompt, load_soul_md, TOOL_USE_ENFORCEMENT_GUIDANCE, TOOL_USE_ENFORCEMENT_MODELS, DEVELOPER_ROLE_MODELS, GOOGLE_MODEL_OPERATIONAL_GUIDANCE from agent.usage_pricing import estimate_usage_cost, normalize_usage @@ -1234,6 +1235,9 @@ class AIAgent: provider=self.provider, ) self.compression_enabled = compression_enabled + self._subdirectory_hints = SubdirectoryHintTracker( + working_dir=os.getenv("TERMINAL_CWD") or None, + ) self._user_turn_count = 0 # Cumulative token usage for the session @@ -6155,6 +6159,11 @@ class AIAgent: # Save oversized results to file instead of destructive truncation function_result = _save_oversized_tool_result(name, function_result) + # Discover subdirectory context files from tool arguments + subdir_hints = self._subdirectory_hints.check_tool_call(name, args) + if subdir_hints: + function_result += subdir_hints + # Append tool result message in order tool_msg = { "role": "tool", @@ -6438,6 +6447,11 @@ class AIAgent: # Save oversized results to file instead of destructive truncation function_result = _save_oversized_tool_result(function_name, function_result) + # Discover subdirectory context files from tool arguments + subdir_hints = self._subdirectory_hints.check_tool_call(function_name, function_args) + if subdir_hints: + function_result += subdir_hints + tool_msg = { "role": "tool", "content": function_result, diff --git a/tests/agent/test_subdirectory_hints.py b/tests/agent/test_subdirectory_hints.py new file mode 100644 index 0000000000..7d2bc607c8 --- /dev/null +++ b/tests/agent/test_subdirectory_hints.py @@ -0,0 +1,191 @@ +"""Tests for progressive subdirectory hint discovery.""" + +import os +import pytest +from pathlib import Path + +from agent.subdirectory_hints import SubdirectoryHintTracker + + +@pytest.fixture +def project(tmp_path): + """Create a mock project tree with hint files in subdirectories.""" + # Root — already loaded at startup + (tmp_path / "AGENTS.md").write_text("Root project instructions") + + # backend/ — has its own AGENTS.md + backend = tmp_path / "backend" + backend.mkdir() + (backend / "AGENTS.md").write_text("Backend-specific instructions:\n- Use FastAPI\n- Always add type hints") + + # backend/src/ — no hints + (backend / "src").mkdir() + (backend / "src" / "main.py").write_text("print('hello')") + + # frontend/ — has CLAUDE.md + frontend = tmp_path / "frontend" + frontend.mkdir() + (frontend / "CLAUDE.md").write_text("Frontend rules:\n- Use TypeScript\n- No any types") + + # docs/ — no hints + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "README.md").write_text("Documentation") + + # deep/nested/path/ — has .cursorrules + deep = tmp_path / "deep" / "nested" / "path" + deep.mkdir(parents=True) + (deep / ".cursorrules").write_text("Cursor rules for nested path") + + return tmp_path + + +class TestSubdirectoryHintTracker: + """Unit tests for SubdirectoryHintTracker.""" + + def test_working_dir_not_loaded(self, project): + """Working dir is pre-marked as loaded (startup handles it).""" + tracker = SubdirectoryHintTracker(working_dir=str(project)) + # Reading a file in the root should NOT trigger hints + result = tracker.check_tool_call("read_file", {"path": str(project / "AGENTS.md")}) + assert result is None + + def test_discovers_agents_md_via_ancestor_walk(self, project): + """Reading backend/src/main.py discovers backend/AGENTS.md via ancestor walk.""" + tracker = SubdirectoryHintTracker(working_dir=str(project)) + result = tracker.check_tool_call( + "read_file", {"path": str(project / "backend" / "src" / "main.py")} + ) + # backend/src/ has no hints, but ancestor walk finds backend/AGENTS.md + assert result is not None + assert "Backend-specific instructions" in result + # Second read in same subtree should not re-trigger + result2 = tracker.check_tool_call( + "read_file", {"path": str(project / "backend" / "AGENTS.md")} + ) + assert result2 is None # backend/ already loaded + + def test_discovers_claude_md(self, project): + """Frontend CLAUDE.md should be discovered.""" + tracker = SubdirectoryHintTracker(working_dir=str(project)) + result = tracker.check_tool_call( + "read_file", {"path": str(project / "frontend" / "index.ts")} + ) + assert result is not None + assert "Frontend rules" in result + + def test_no_duplicate_loading(self, project): + """Same directory should not be loaded twice.""" + tracker = SubdirectoryHintTracker(working_dir=str(project)) + result1 = tracker.check_tool_call( + "read_file", {"path": str(project / "frontend" / "a.ts")} + ) + assert result1 is not None + + result2 = tracker.check_tool_call( + "read_file", {"path": str(project / "frontend" / "b.ts")} + ) + assert result2 is None # already loaded + + def test_no_hints_in_empty_directory(self, project): + """Directories without hint files return None.""" + tracker = SubdirectoryHintTracker(working_dir=str(project)) + result = tracker.check_tool_call( + "read_file", {"path": str(project / "docs" / "README.md")} + ) + assert result is None + + def test_terminal_command_path_extraction(self, project): + """Paths extracted from terminal commands.""" + tracker = SubdirectoryHintTracker(working_dir=str(project)) + result = tracker.check_tool_call( + "terminal", {"command": f"cat {project / 'frontend' / 'index.ts'}"} + ) + assert result is not None + assert "Frontend rules" in result + + def test_terminal_cd_command(self, project): + """cd into a directory with hints.""" + tracker = SubdirectoryHintTracker(working_dir=str(project)) + result = tracker.check_tool_call( + "terminal", {"command": f"cd {project / 'backend'} && ls"} + ) + assert result is not None + assert "Backend-specific instructions" in result + + def test_relative_path(self, project): + """Relative paths resolved against working_dir.""" + tracker = SubdirectoryHintTracker(working_dir=str(project)) + result = tracker.check_tool_call( + "read_file", {"path": "frontend/index.ts"} + ) + assert result is not None + assert "Frontend rules" in result + + def test_outside_working_dir_still_checked(self, tmp_path, project): + """Paths outside working_dir are still checked for hints.""" + other_project = tmp_path / "other" + other_project.mkdir() + (other_project / "AGENTS.md").write_text("Other project rules") + tracker = SubdirectoryHintTracker(working_dir=str(project)) + result = tracker.check_tool_call( + "read_file", {"path": str(other_project / "file.py")} + ) + assert result is not None + assert "Other project rules" in result + + def test_workdir_arg(self, project): + """The workdir argument from terminal tool is checked.""" + tracker = SubdirectoryHintTracker(working_dir=str(project)) + result = tracker.check_tool_call( + "terminal", {"command": "ls", "workdir": str(project / "frontend")} + ) + assert result is not None + assert "Frontend rules" in result + + def test_deeply_nested_cursorrules(self, project): + """Deeply nested .cursorrules should be discovered.""" + tracker = SubdirectoryHintTracker(working_dir=str(project)) + result = tracker.check_tool_call( + "read_file", {"path": str(project / "deep" / "nested" / "path" / "file.py")} + ) + assert result is not None + assert "Cursor rules for nested path" in result + + def test_hint_format_includes_path(self, project): + """Discovered hints should indicate which file they came from.""" + tracker = SubdirectoryHintTracker(working_dir=str(project)) + result = tracker.check_tool_call( + "read_file", {"path": str(project / "backend" / "file.py")} + ) + assert result is not None + assert "Subdirectory context discovered:" in result + assert "AGENTS.md" in result + + def test_truncation_of_large_hints(self, tmp_path): + """Hint files over the limit are truncated.""" + sub = tmp_path / "bigdir" + sub.mkdir() + (sub / "AGENTS.md").write_text("x" * 20_000) + + tracker = SubdirectoryHintTracker(working_dir=str(tmp_path)) + result = tracker.check_tool_call( + "read_file", {"path": str(sub / "file.py")} + ) + assert result is not None + assert "truncated" in result.lower() + # Should be capped + assert len(result) < 20_000 + + def test_empty_args(self, project): + """Empty args should not crash.""" + tracker = SubdirectoryHintTracker(working_dir=str(project)) + assert tracker.check_tool_call("read_file", {}) is None + assert tracker.check_tool_call("terminal", {"command": ""}) is None + + def test_url_in_command_ignored(self, project): + """URLs in shell commands should not be treated as paths.""" + tracker = SubdirectoryHintTracker(working_dir=str(project)) + result = tracker.check_tool_call( + "terminal", {"command": "curl https://example.com/frontend/api"} + ) + assert result is None diff --git a/website/docs/developer-guide/prompt-assembly.md b/website/docs/developer-guide/prompt-assembly.md index 858ac38ec1..047117fa7e 100644 --- a/website/docs/developer-guide/prompt-assembly.md +++ b/website/docs/developer-guide/prompt-assembly.md @@ -218,7 +218,7 @@ Local memory and user profile data are injected as frozen snapshots at session s `agent/prompt_builder.py` scans and sanitizes project context files using a **priority system** — only one type is loaded (first match wins): 1. `.hermes.md` / `HERMES.md` (walks to git root) -2. `AGENTS.md` (recursive directory walk) +2. `AGENTS.md` (CWD at startup; subdirectories discovered progressively during the session via `agent/subdirectory_hints.py`) 3. `CLAUDE.md` (CWD only) 4. `.cursorrules` / `.cursor/rules/*.mdc` (CWD only) diff --git a/website/docs/user-guide/features/context-files.md b/website/docs/user-guide/features/context-files.md index 380d453cae..64b9720f62 100644 --- a/website/docs/user-guide/features/context-files.md +++ b/website/docs/user-guide/features/context-files.md @@ -13,8 +13,8 @@ Hermes Agent automatically discovers and loads context files that shape how it b | File | Purpose | Discovery | |------|---------|-----------| | **.hermes.md** / **HERMES.md** | Project instructions (highest priority) | Walks to git root | -| **AGENTS.md** | Project instructions, conventions, architecture | Recursive (walks subdirectories) | -| **CLAUDE.md** | Claude Code context files (also detected) | CWD only | +| **AGENTS.md** | Project instructions, conventions, architecture | CWD at startup + subdirectories progressively | +| **CLAUDE.md** | Claude Code context files (also detected) | CWD at startup + subdirectories progressively | | **SOUL.md** | Global personality and tone customization for this Hermes instance | `HERMES_HOME/SOUL.md` only | | **.cursorrules** | Cursor IDE coding conventions | CWD only | | **.cursor/rules/*.mdc** | Cursor IDE rule modules | CWD only | @@ -27,25 +27,29 @@ Only **one** project context type is loaded per session (first match wins): `.he `AGENTS.md` is the primary project context file. It tells the agent how your project is structured, what conventions to follow, and any special instructions. -### Hierarchical Discovery +### Progressive Subdirectory Discovery -Hermes walks the directory tree starting from the working directory and loads **all** `AGENTS.md` files found, sorted by depth. This supports monorepo-style setups: +At session start, Hermes loads the `AGENTS.md` from your working directory into the system prompt. As the agent navigates into subdirectories during the session (via `read_file`, `terminal`, `search_files`, etc.), it **progressively discovers** context files in those directories and injects them into the conversation at the moment they become relevant. ``` my-project/ -├── AGENTS.md ← Top-level project context +├── AGENTS.md ← Loaded at startup (system prompt) ├── frontend/ -│ └── AGENTS.md ← Frontend-specific instructions +│ └── AGENTS.md ← Discovered when agent reads frontend/ files ├── backend/ -│ └── AGENTS.md ← Backend-specific instructions +│ └── AGENTS.md ← Discovered when agent reads backend/ files └── shared/ - └── AGENTS.md ← Shared library conventions + └── AGENTS.md ← Discovered when agent reads shared/ files ``` -All four files are concatenated into a single context block with relative path headers. +This approach has two advantages over loading everything at startup: +- **No system prompt bloat** — subdirectory hints only appear when needed +- **Prompt cache preservation** — the system prompt stays stable across turns + +Each subdirectory is checked at most once per session. The discovery also walks up parent directories, so reading `backend/src/main.py` will discover `backend/AGENTS.md` even if `backend/src/` has no context file of its own. :::info -Directories that are skipped during the walk: `.`-prefixed dirs, `node_modules`, `__pycache__`, `venv`, `.venv`. +Subdirectory context files go through the same [security scan](#security-prompt-injection-protection) as startup context files. Malicious files are blocked. ::: ### Example AGENTS.md @@ -98,15 +102,28 @@ This means your existing Cursor conventions automatically apply when using Herme ## How Context Files Are Loaded +### At startup (system prompt) + Context files are loaded by `build_context_files_prompt()` in `agent/prompt_builder.py`: -1. **At session start** — the function scans the working directory +1. **Scan working directory** — checks for `.hermes.md` → `AGENTS.md` → `CLAUDE.md` → `.cursorrules` (first match wins) 2. **Content is read** — each file is read as UTF-8 text 3. **Security scan** — content is checked for prompt injection patterns 4. **Truncation** — files exceeding 20,000 characters are head/tail truncated (70% head, 20% tail, with a marker in the middle) 5. **Assembly** — all sections are combined under a `# Project Context` header 6. **Injection** — the assembled content is added to the system prompt +### During the session (progressive discovery) + +`SubdirectoryHintTracker` in `agent/subdirectory_hints.py` watches tool call arguments for file paths: + +1. **Path extraction** — after each tool call, file paths are extracted from arguments (`path`, `workdir`, shell commands) +2. **Ancestor walk** — the directory and up to 5 parent directories are checked (stopping at already-visited directories) +3. **Hint loading** — if an `AGENTS.md`, `CLAUDE.md`, or `.cursorrules` is found, it's loaded (first match per directory) +4. **Security scan** — same prompt injection scan as startup files +5. **Truncation** — capped at 8,000 characters per file +6. **Injection** — appended to the tool result, so the model sees it in context naturally + The final prompt section looks roughly like: ```text