From 25f0cecf5e21bfe87aa27ffe8a8cb8734af2b1ca Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 21 Jun 2026 17:05:58 -0700 Subject: [PATCH] Port from nearai/ironclaw#5029: graceful char-budget truncation for read_file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit read_file previously hard-rejected any read whose formatted output exceeded the ~100K char safety limit, returning an error with zero content. A file with few but very long lines (logs, wide CSV rows, minified data) sails past the line-count limit and then trips the char guard, so the model gets nothing and must guess a smaller limit — wasting a full round-trip. Now the read is trimmed to the last complete line that fits the budget and returns the partial content plus truncated_by="bytes" and a next_offset, so the model paginates forward instead of starting over. A single line larger than the whole budget is clamped on a code-point boundary (never empty) and the cursor still advances. Applies at both read paths (normal + extracted documents). Adapted from IronClaw's Rust dual line/byte cap to hermes's Python tool-layer char guard, which is the single uniform chokepoint over the gutter-rendered content for every backend. --- tests/tools/test_file_read_guards.py | 107 +++++++++++++++++++++----- tools/file_tools.py | 110 +++++++++++++++++++++------ 2 files changed, 177 insertions(+), 40 deletions(-) diff --git a/tests/tools/test_file_read_guards.py b/tests/tools/test_file_read_guards.py index 861ebc4936f..90b7f957c0d 100644 --- a/tests/tools/test_file_read_guards.py +++ b/tests/tools/test_file_read_guards.py @@ -226,7 +226,8 @@ class TestDevicePathBlocking(unittest.TestCase): # --------------------------------------------------------------------------- class TestCharacterCountGuard(unittest.TestCase): - """Large reads should be rejected with guidance to use offset/limit.""" + """Oversized reads are truncated on a line boundary (nearai/ironclaw#5029), + not rejected — the model gets the head of the file plus a next_offset.""" def setUp(self): _read_tracker.clear() @@ -235,28 +236,54 @@ class TestCharacterCountGuard(unittest.TestCase): _read_tracker.clear() @patch("tools.file_tools._get_file_ops") - @patch("tools.file_tools._get_max_read_chars", return_value=_DEFAULT_MAX_READ_CHARS) - def test_oversized_read_rejected(self, _mock_limit, mock_ops): - """A read that returns >max chars is rejected.""" - big_content = "x" * (_DEFAULT_MAX_READ_CHARS + 1) + @patch("tools.file_tools._get_max_read_chars", return_value=1000) + def test_oversized_multiline_read_truncated_with_continuation(self, _mock_limit, mock_ops): + """A read whose many lines exceed the char budget is trimmed to the + last complete line and offers a next_offset, instead of returning an + error with no content.""" + # 50 lines of 100 chars each = ~5050 chars, well over the 1000 budget. + big_content = "\n".join(f"{i}|" + "z" * 98 for i in range(1, 51)) mock_ops.return_value = _make_fake_ops( content=big_content, - total_lines=5000, - file_size=len(big_content) + 100, # bigger than content + total_lines=50, + file_size=len(big_content), ) result = json.loads(read_file_tool("/tmp/huge.txt", task_id="big")) - self.assertIn("error", result) - self.assertIn("safety limit", result["error"]) - self.assertIn("offset and limit", result["error"]) - self.assertIn("total_lines", result) + # No hard rejection — content is present. + self.assertNotIn("error", result) + self.assertIn("content", result) + self.assertTrue(result["content"]) + # Truncation metadata for the model to paginate. + self.assertTrue(result["truncated"]) + self.assertEqual(result["truncated_by"], "bytes") + self.assertIn("next_offset", result) + self.assertGreater(result["next_offset"], 1) + # Body fits the budget (allowing for redaction not growing it). + self.assertLessEqual(len(result["content"]), 1000) + self.assertIn("offset", result["hint"]) @patch("tools.file_tools._get_file_ops") - def test_small_read_not_rejected(self, mock_ops): - """Normal-sized reads pass through fine.""" + @patch("tools.file_tools._get_max_read_chars", return_value=1000) + def test_single_oversized_line_clamped_not_empty(self, _mock_limit, mock_ops): + """A single line larger than the whole budget is clamped (never empty) + and the cursor still advances by one line.""" + big_content = "1|" + "q" * 5000 # one line, no newline, > budget + mock_ops.return_value = _make_fake_ops( + content=big_content, total_lines=1, file_size=len(big_content), + ) + result = json.loads(read_file_tool("/tmp/oneline.txt", task_id="oneline")) + self.assertNotIn("error", result) + self.assertTrue(result["content"]) # not empty + self.assertEqual(result["next_offset"], 2) # advanced past line 1 + + @patch("tools.file_tools._get_file_ops") + def test_small_read_not_truncated(self, mock_ops): + """Normal-sized reads pass through fine with no truncation flag.""" mock_ops.return_value = _make_fake_ops(content="short\n", file_size=6) result = json.loads(read_file_tool("/tmp/small.txt", task_id="small")) self.assertNotIn("error", result) self.assertIn("content", result) + self.assertNotEqual(result.get("truncated_by"), "bytes") @patch("tools.file_tools._get_file_ops") @patch("tools.file_tools._get_max_read_chars", return_value=_DEFAULT_MAX_READ_CHARS) @@ -271,6 +298,49 @@ class TestCharacterCountGuard(unittest.TestCase): self.assertIn("content", result) +class TestTruncateToCharBudget(unittest.TestCase): + """Unit tests for the line-boundary char-budget trimmer.""" + + def _fn(self): + from tools.file_tools import _truncate_to_char_budget + return _truncate_to_char_budget + + def test_fits_unchanged(self): + fn = self._fn() + text = "1|a\n2|b\n3|c" + out, lines, trunc = fn(text, 1000) + self.assertEqual(out, text) + self.assertEqual(lines, 3) + self.assertFalse(trunc) + + def test_trims_on_line_boundary(self): + fn = self._fn() + # 3 lines of 10 chars; budget fits ~2 lines. + text = "\n".join("x" * 10 for _ in range(5)) # 5 lines, 54 chars + out, lines, trunc = fn(text, 25) + self.assertTrue(trunc) + # Output ends on a complete line (no partial line at the tail). + self.assertFalse(out.endswith("x" * 3) and len(out.split("\n")[-1]) != 10) + self.assertEqual(lines, out.count("\n") + 1) + self.assertLessEqual(len(out), 25) + + def test_single_line_over_budget_clamped(self): + fn = self._fn() + text = "y" * 500 # single line, no newline + out, lines, trunc = fn(text, 100) + self.assertTrue(trunc) + self.assertEqual(lines, 1) + self.assertEqual(len(out), 100) # clamped to budget + self.assertNotEqual(out, "") # never empty + + def test_empty_content(self): + fn = self._fn() + out, lines, trunc = fn("", 100) + self.assertEqual(out, "") + self.assertEqual(lines, 0) + self.assertFalse(trunc) + + # --------------------------------------------------------------------------- # File deduplication # --------------------------------------------------------------------------- @@ -711,12 +781,15 @@ class TestConfigOverride(unittest.TestCase): @patch("tools.file_tools._get_file_ops") @patch("hermes_cli.config.load_config", return_value={"file_read_max_chars": 50}) def test_custom_config_lowers_limit(self, _mock_cfg, mock_ops): - """A config value of 50 should reject reads over 50 chars.""" + """A config value of 50 should trigger truncation for reads over 50 chars, + with the configured limit reflected in the continuation hint.""" mock_ops.return_value = _make_fake_ops(content="x" * 60, file_size=60) result = json.loads(read_file_tool("/tmp/cfgtest.txt", task_id="cfg1")) - self.assertIn("error", result) - self.assertIn("safety limit", result["error"]) - self.assertIn("50", result["error"]) # should show the configured limit + self.assertNotIn("error", result) + self.assertTrue(result["truncated"]) + self.assertEqual(result["truncated_by"], "bytes") + self.assertIn("50", result["hint"]) # should show the configured limit + self.assertLessEqual(len(result["content"]), 50) @patch("tools.file_tools._get_file_ops") @patch("hermes_cli.config.load_config", return_value={"file_read_max_chars": 500_000}) diff --git a/tools/file_tools.py b/tools/file_tools.py index 299799fcce8..f9047a11c9f 100644 --- a/tools/file_tools.py +++ b/tools/file_tools.py @@ -82,6 +82,51 @@ def _get_max_read_chars() -> int: _max_read_chars_cached = _DEFAULT_MAX_READ_CHARS return _max_read_chars_cached + +def _truncate_to_char_budget(content: str, max_chars: int) -> tuple[str, int, bool]: + """Trim line-numbered ``read_file`` content to fit a char budget. + + Ported in spirit from nearai/ironclaw#5029 (dual line/byte cap on + ``read_file``). Where hermes previously hard-rejected an oversized read + (forcing the model to guess a smaller ``limit`` and burn a round-trip + returning nothing), this trims the content to the last *complete line* + that fits within ``max_chars`` and reports how many lines were kept so + the caller can offer a ``next_offset`` continuation. + + ``content`` is the gutter-rendered text (``LINE_NUM|CONTENT`` joined by + ``\\n``). Individual lines are already clamped to ``get_max_line_length()`` + upstream, so a single line never blows the whole budget on its own; the + overflow this handles is the *accumulation* of many lines under the + line-count limit (logs, wide CSV rows, minified data). + + Returns ``(kept_text, lines_kept, truncated)``. When ``content`` already + fits, returns it unchanged with ``truncated=False``. If not even the + first line fits, that single line is clamped on a code-point boundary + (Python ``str`` slicing never splits a code point) so the read never + returns empty and the cursor can still advance. + """ + if len(content) <= max_chars: + return content, (content.count("\n") + 1 if content else 0), False + + lines = content.split("\n") + kept: list[str] = [] + running = 0 + for line in lines: + # +1 for the "\n" that rejoins this line to the previous one. + addition = len(line) + (1 if kept else 0) + if running + addition > max_chars: + break + kept.append(line) + running += addition + + if not kept: + # First line alone exceeds the budget. Clamp on a code-point + # boundary rather than emitting nothing. + kept.append(lines[0][:max_chars]) + + return "\n".join(kept), len(kept), True + + # If the total file size exceeds this AND the caller didn't specify a narrow # range (limit <= 200), we include a hint encouraging targeted reads. _LARGE_FILE_HINT_BYTES = 512_000 # 512 KB @@ -1177,17 +1222,24 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 500, task_id: str = content_len = len(result_dict["content"]) max_chars = _get_max_read_chars() if content_len > max_chars: - return json.dumps({ - "error": ( - f"Read produced {content_len:,} characters which exceeds " - f"the safety limit ({max_chars:,} chars). " - "Use offset and limit to read a smaller range. " - f"The document has {total_lines} lines of extracted text." - ), - "path": path, - "total_lines": total_lines, - "file_size": result_dict["file_size"], - }, ensure_ascii=False) + # Graceful char-budget truncation (nearai/ironclaw#5029): + # trim to the last complete line that fits and offer a + # next_offset rather than rejecting the whole extraction. + trimmed, lines_kept, _ = _truncate_to_char_budget( + result_dict["content"], max_chars + ) + next_offset = offset + lines_kept + shown_end = offset + lines_kept - 1 + result_dict["content"] = trimmed + result_dict["truncated"] = True + result_dict["truncated_by"] = "bytes" + result_dict["next_offset"] = next_offset + result_dict["hint"] = ( + f"Output truncated at the {max_chars:,}-char read budget " + f"after {lines_kept} line(s) (showing lines {offset}-" + f"{shown_end} of {total_lines}). Use offset={next_offset} " + "to continue." + ) if result_dict["content"]: result_dict["content"] = redact_sensitive_text(result_dict["content"], file_read=True) return json.dumps(result_dict, ensure_ascii=False) @@ -1290,18 +1342,30 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 500, task_id: str = file_size = result_dict.get("file_size", 0) max_chars = _get_max_read_chars() if content_len > max_chars: + # Graceful char-budget truncation (ported from nearai/ironclaw#5029). + # Instead of rejecting the whole read — which forces the model to + # guess a smaller `limit` and wastes a round-trip returning nothing + # — trim to the last complete line that fits and offer a + # `next_offset` so the model can paginate forward. This rescues the + # "few but very long lines" case (logs, wide CSVs, minified data) + # that sails past the line-count `limit` but blows the char budget. total_lines = result_dict.get("total_lines", "unknown") - return json.dumps({ - "error": ( - f"Read produced {content_len:,} characters which exceeds " - f"the safety limit ({max_chars:,} chars). " - "Use offset and limit to read a smaller range. " - f"The file has {total_lines} lines total." - ), - "path": path, - "total_lines": total_lines, - "file_size": file_size, - }, ensure_ascii=False) + trimmed, lines_kept, _ = _truncate_to_char_budget( + result.content or "", max_chars + ) + next_offset = offset + lines_kept + shown_end = offset + lines_kept - 1 + result.content = trimmed + result_dict["content"] = trimmed + result_dict["truncated"] = True + result_dict["truncated_by"] = "bytes" + result_dict["next_offset"] = next_offset + result_dict["hint"] = ( + f"Output truncated at the {max_chars:,}-char read budget after " + f"{lines_kept} line(s) (showing lines {offset}-{shown_end} of " + f"{total_lines}). Use offset={next_offset} to continue." + ) + content_len = len(trimmed) # ── Redact secrets (after guard check to skip oversized content) ── if result.content: @@ -1936,7 +2000,7 @@ def _check_file_reqs(): READ_FILE_SCHEMA = { "name": "read_file", - "description": "Read a text file with line numbers and pagination. Use this instead of cat/head/tail in terminal. Output format: 'LINE_NUM|CONTENT'. Suggests similar filenames if not found. Use offset and limit for large files. Reads exceeding ~100K characters are rejected; use offset and limit to read specific sections of large files. Jupyter notebooks (.ipynb), Word documents (.docx), and Excel workbooks (.xlsx) are auto-extracted to readable text. NOTE: Cannot read images or other binary files — use vision_analyze for images.", + "description": "Read a text file with line numbers and pagination. Use this instead of cat/head/tail in terminal. Output format: 'LINE_NUM|CONTENT'. Suggests similar filenames if not found. Use offset and limit for large files. Reads exceeding ~100K characters are truncated on a line boundary and return a next_offset; continue with offset to read the rest. Jupyter notebooks (.ipynb), Word documents (.docx), and Excel workbooks (.xlsx) are auto-extracted to readable text. NOTE: Cannot read images or other binary files — use vision_analyze for images.", "parameters": { "type": "object", "properties": {