From eaa35ae68ffad153b63eddee95208f4af71af6ec Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Thu, 23 Jul 2026 08:21:23 -0700 Subject: [PATCH] fix(gateway): balance code fences on every remaining chunk-split path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Widen #48476's fence guarantees to the two splitters that still emitted fence-broken chunks: * GatewayStreamConsumer._split_text_chunks (fallback final send): close the orphaned ``` at each chunk boundary and reopen it — with the original language tag — on the next chunk, mirroring BasePlatformAdapter.truncate_message's contract. Headroom is reserved so balanced chunks stay within the platform limit. * Slack block_kit._split_text (3000-char section chunking): same close/reopen balancing for mrkdwn section text carrying fences. With these, every chunk boundary — non-streaming send (truncate_message), streaming overflow (_truncate_for_stream via adapter.truncate_message per #45938), fallback final (_split_text_chunks), final-send balance (ensure_closed_code_fences), and Block Kit section splits — delivers fence-balanced chunks. Regression tests probe each path with fenced fixtures, assert per-chunk balance, limit compliance, language-tag reopening, and prose passthrough. --- gateway/stream_consumer.py | 57 +++++++++++++++++-- plugins/platforms/slack/block_kit.py | 29 ++++++++-- tests/gateway/test_code_fence_tracking.py | 67 +++++++++++++++++++++++ tests/gateway/test_slack_block_kit.py | 31 +++++++++++ 4 files changed, 176 insertions(+), 8 deletions(-) diff --git a/gateway/stream_consumer.py b/gateway/stream_consumer.py index d24812c73740..e60e9117031f 100644 --- a/gateway/stream_consumer.py +++ b/gateway/stream_consumer.py @@ -1175,19 +1175,68 @@ class GatewayStreamConsumer: return final_text[len(prefix):].lstrip() return final_text + @staticmethod + def _balance_fences_across_chunks(chunks: "list[str]") -> "list[str]": + """Close orphaned ``` fences at each chunk boundary and reopen on the next. + + When a split lands inside a triple-backtick code block, the head chunk + would render everything after the orphaned fence as code, and the tail + chunk's content would lose its code formatting. Mirror + ``BasePlatformAdapter.truncate_message``'s contract: close the fence at + the end of the chunk and reopen it (with the original language tag) at + the start of the next one, so EVERY delivered chunk is fence-balanced + on its own. + """ + if len(chunks) <= 1: + return chunks + out: "list[str]" = [] + carry_lang: "Optional[str]" = None + for chunk in chunks: + prefix = f"```{carry_lang}\n" if carry_lang is not None else "" + in_code = carry_lang is not None + lang = carry_lang or "" + for line in chunk.split("\n"): + stripped = line.strip() + if stripped.startswith("```"): + if in_code: + in_code = False + lang = "" + else: + in_code = True + tag = stripped[3:].strip() + lang = tag.split()[0] if tag else "" + body = prefix + chunk + if in_code: + body += "\n```" + carry_lang = lang + else: + carry_lang = None + out.append(body) + return out + @staticmethod def _split_text_chunks( text: str, limit: int, len_fn: "Callable[[str], int]" = len, ) -> list[str]: - """Split text into reasonably sized chunks for fallback sends.""" + """Split text into reasonably sized chunks for fallback sends. + + Chunks are fence-balanced: a split inside a ``` code block closes the + fence on the head chunk and reopens it on the tail, so no chunk leaves + the rest of a message rendering as one giant code block. + """ if len_fn(text) <= limit: return [text] + # Reserve headroom for the close/reopen fence markers the balancing + # pass may add, so balanced chunks stay within the platform limit. + split_limit = limit + if "```" in text: + split_limit = max(limit - 16, limit // 2, 1) chunks: list[str] = [] remaining = text - while len_fn(remaining) > limit: - _cp_budget = _custom_unit_to_cp(remaining, limit, len_fn) + while len_fn(remaining) > split_limit: + _cp_budget = _custom_unit_to_cp(remaining, split_limit, len_fn) split_at = remaining.rfind("\n", 0, _cp_budget) if split_at < _cp_budget // 2: split_at = _cp_budget @@ -1195,7 +1244,7 @@ class GatewayStreamConsumer: remaining = remaining[split_at:].lstrip("\n") if remaining: chunks.append(remaining) - return chunks + return GatewayStreamConsumer._balance_fences_across_chunks(chunks) def _truncate_for_stream( self, diff --git a/plugins/platforms/slack/block_kit.py b/plugins/platforms/slack/block_kit.py index 5f8dc4d0c1b1..c2767208ccc6 100644 --- a/plugins/platforms/slack/block_kit.py +++ b/plugins/platforms/slack/block_kit.py @@ -536,19 +536,40 @@ def render_blocks( def _split_text(text: str, limit: int) -> List[str]: - """Split ``text`` into <= ``limit``-char chunks on line, then hard, boundaries.""" + """Split ``text`` into <= ``limit``-char chunks on line, then hard, boundaries. + + Chunks are fence-balanced: when a split lands inside a ``` code span that + survived into section text (the renderer normally routes fenced blocks to + ``rich_text_preformatted``, but mrkdwn text can still carry fences), the + fence is closed at the end of the chunk and reopened on the next so each + section renders correctly on its own. + """ if len(text) <= limit: return [text] + # Reserve headroom for the close/reopen markers the balancing pass adds. + split_limit = max(limit - 8, limit // 2, 1) if "```" in text else limit out: List[str] = [] remaining = text - while len(remaining) > limit: - cut = remaining.rfind("\n", 0, limit) + while len(remaining) > split_limit: + cut = remaining.rfind("\n", 0, split_limit) if cut <= 0: - cut = limit + cut = split_limit out.append(remaining[:cut]) remaining = remaining[cut:].lstrip("\n") if remaining: out.append(remaining) + if len(out) > 1 and "```" in text: + balanced: List[str] = [] + reopen = False + for chunk in out: + if reopen: + chunk = "```\n" + chunk + odd = chunk.count("```") % 2 == 1 + if odd: + chunk += "\n```" + reopen = odd + balanced.append(chunk) + out = balanced return out diff --git a/tests/gateway/test_code_fence_tracking.py b/tests/gateway/test_code_fence_tracking.py index eeb7815c44c4..b43224d49838 100644 --- a/tests/gateway/test_code_fence_tracking.py +++ b/tests/gateway/test_code_fence_tracking.py @@ -590,3 +590,70 @@ class TestFallbackFinalFenceGap: ) assert _odd_fences(result[0]), "Unclosed fence passes through" + + +# ═══════════════════════════════════════════════════════════════════════════ +# M. Widened: every chunk boundary is fence-balanced (C11 salvage) +# ═══════════════════════════════════════════════════════════════════════════ + +class TestSplitTextChunksFenceBalanced: + """_split_text_chunks now closes orphaned fences at each boundary and + reopens them on the next chunk (mirrors truncate_message's contract), + so the fallback-final path can never leave a chunk rendering the rest + of the message as one giant code block.""" + + def test_every_chunk_balanced_bare_fence(self): + long = "\n".join(f"code{i}" for i in range(30)) + text = f"```\n{long}\n```" + chunks = GatewayStreamConsumer._split_text_chunks(text, 80) + assert len(chunks) >= 2 + _assert_balanced(chunks, "fallback chunk") + + def test_every_chunk_balanced_lang_fence_reopens_with_tag(self): + long = "\n".join(f"print({i})" for i in range(40)) + text = f"```python\n{long}\n```" + chunks = GatewayStreamConsumer._split_text_chunks(text, 90) + assert len(chunks) >= 2 + _assert_balanced(chunks, "fallback chunk") + # Continuation chunks reopen with the original language tag + for chunk in chunks[1:]: + assert chunk.startswith("```python"), ( + f"continuation should reopen with ```python: {chunk[:40]!r}" + ) + + def test_prose_only_split_unchanged(self): + """No fences → behaviour identical to the plain splitter.""" + text = "\n".join(f"line {i}" for i in range(50)) + chunks = GatewayStreamConsumer._split_text_chunks(text, 60) + assert len(chunks) >= 2 + assert "```" not in "".join(chunks) + # Round-trips the content (modulo the newline trimming at cuts) + assert "".join(c.replace("\n", "") for c in chunks) == text.replace("\n", "") + + def test_unclosed_input_final_chunk_closed(self): + """Input truncated mid-block (finish_reason=length) → last chunk + still balanced.""" + long = "\n".join(f"row{i}" for i in range(40)) + text = f"```\n{long}" # never closed + chunks = GatewayStreamConsumer._split_text_chunks(text, 80) + assert len(chunks) >= 2 + _assert_balanced(chunks, "fallback chunk") + + def test_balanced_chunks_respect_limit(self): + long = "\n".join(f"code{i}" for i in range(30)) + text = f"```\n{long}\n```" + limit = 80 + chunks = GatewayStreamConsumer._split_text_chunks(text, limit) + for chunk in chunks: + assert len(chunk) <= limit, ( + f"balanced chunk exceeds limit: {len(chunk)} > {limit}" + ) + + def test_multiple_blocks_alternating(self): + text = ( + "intro\n```\n" + "\n".join("a" * 10 for _ in range(10)) + "\n```\n" + "middle prose\n```js\n" + "\n".join("b" * 10 for _ in range(10)) + "\n```\nend" + ) + chunks = GatewayStreamConsumer._split_text_chunks(text, 70) + assert len(chunks) >= 2 + _assert_balanced(chunks, "fallback chunk") diff --git a/tests/gateway/test_slack_block_kit.py b/tests/gateway/test_slack_block_kit.py index d745ccdefde7..6ecd976eaf31 100644 --- a/tests/gateway/test_slack_block_kit.py +++ b/tests/gateway/test_slack_block_kit.py @@ -429,3 +429,34 @@ class TestSanitizeBlocks: def test_never_raises_on_garbage(self): assert sanitize_blocks([{"no_type": True}, "not-a-dict", 42]) is None + + +class TestSplitTextFenceBalanced: + """_split_text closes/reopens ``` fences at section chunk boundaries.""" + + def test_fenced_split_every_chunk_balanced(self): + from plugins.platforms.slack.block_kit import _split_text + + text = "```\n" + "\n".join("y" * 20 for _ in range(30)) + "\n```" + chunks = _split_text(text, 100) + assert len(chunks) >= 2 + for i, chunk in enumerate(chunks): + assert chunk.count("```") % 2 == 0, ( + f"chunk {i} has unbalanced fences: {chunk[:60]!r}" + ) + + def test_fenced_split_respects_limit(self): + from plugins.platforms.slack.block_kit import _split_text + + text = "```\n" + "\n".join("y" * 20 for _ in range(30)) + "\n```" + limit = 100 + for chunk in _split_text(text, limit): + assert len(chunk) <= limit + + def test_prose_split_unchanged(self): + from plugins.platforms.slack.block_kit import _split_text + + text = "\n".join(f"line {i}" for i in range(60)) + chunks = _split_text(text, 80) + assert len(chunks) >= 2 + assert all("```" not in c for c in chunks)