From c82f4636f2a57c8e1ad98c5224a2f0652c6c0f70 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:32:10 -0700 Subject: [PATCH] fix(gateway): deliver MEDIA tags with sentence-final punctuation and inline-code wrapping Two remaining formatting variants that silently killed file delivery: - A trailing sentence period (MEDIA:/x/data.csv.) failed the boundary lookahead, so the tag neither extracted nor stripped. The period is now accepted as a boundary only when followed by whitespace/EOL, keeping multi-part extensions (.tar.gz) intact. - A whole tag wrapped in inline code (`MEDIA:/x/data.csv`) was masked as a prose example (#35695). Models routinely format paths as inline code, eating real deliveries. Inline-code tags now deliver when the path validates on disk; non-existent example paths stay masked and fenced code blocks remain fully masked. Adds a regression matrix covering both plus the salvaged contributor fixes (emphasis wrap, glued tags, glued [[as_document]], dedupe, unknown-extension and extensionless delivery). --- gateway/platforms/base.py | 20 ++- .../test_media_tag_formatting_variants.py | 126 ++++++++++++++++++ 2 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 tests/gateway/test_media_tag_formatting_variants.py diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py index 2568d474977..78fb93c5348 100644 --- a/gateway/platforms/base.py +++ b/gateway/platforms/base.py @@ -1525,11 +1525,18 @@ _MEDIA_EXT_ALTERNATION = "|".join( # followed by stray text don't merge into one invalid path. The trailing # lookahead also accepts ``MEDIA:`` as a boundary, so the next tag stops # the current match cleanly (#68773). +# +# Sentence-final punctuation: a ``.`` is accepted as a boundary only when +# followed by whitespace / EOL (``\.(?=\s|$)``) so ``MEDIA:/x/data.csv.`` +# at the end of a sentence still extracts ``data.csv``. The whitespace +# guard keeps multi-part extensions intact — for ``archive.tar.gz`` the +# ``.`` after ``tar`` is followed by ``g``, so the match must extend to +# ``.gz`` instead of stopping early at ``.tar``. MEDIA_TAG_CLEANUP_RE = re.compile( r'''[`"'*_]{0,3}MEDIA:\s*''' r'''(?P`[^`\n]+?`|"[^"\n]+?"|'[^'\n]+?'|''' r'''(?:~/|/|[A-Za-z]:[/\\])\S+?(?:[^\S\n]+\S+?)*?\.(?:''' + _MEDIA_EXT_ALTERNATION + r'''))''' - r'''(?=[\s`"'*_,;:)\]}\[]|MEDIA:|$)[`"'*_]{0,3}''', + r'''(?=[\s`"'*_,;:)\]}\[]|MEDIA:|\.(?:\s|$)|$)[`"'*_]{0,3}\.?''', re.IGNORECASE, ) @@ -3831,6 +3838,17 @@ class BasePlatformAdapter(ABC): prefix = content[max(0, start - 20):start] if re.search(r'MEDIA:\s*$', prefix): continue # This is a MEDIA path quote, not inline code + # A whole tag wrapped in inline code (`MEDIA:/path.csv`) is a real + # delivery directive, not a prose example — models routinely format + # file paths as inline code. Deliver it IF the path validates + # (exists on disk, not denylisted). Prose examples with + # non-existent paths stay masked (#35695), and fenced code blocks + # are always masked regardless. + inner = m.group(0)[1:-1].strip() + if inner.upper().startswith("MEDIA:"): + candidate = _normalize_media_tag_path(inner[6:]) + if candidate and validate_media_delivery_path(candidate): + continue # Real deliverable tag in inline code — keep it scannable spans.append((start, m.end())) # Blockquote lines: > at line start diff --git a/tests/gateway/test_media_tag_formatting_variants.py b/tests/gateway/test_media_tag_formatting_variants.py new file mode 100644 index 00000000000..1e3ccecf97e --- /dev/null +++ b/tests/gateway/test_media_tag_formatting_variants.py @@ -0,0 +1,126 @@ +"""Regression tests: MEDIA tag formatting variants that previously broke delivery. + +Covers the two follow-up fixes layered on top of the salvaged contributor PRs: + +1. Sentence-final punctuation — ``MEDIA:/x/data.csv.`` at the end of a + sentence must extract ``data.csv`` (the trailing ``.`` is a boundary, not + part of the path), while multi-part extensions (``archive.tar.gz``) + remain intact. + +2. Inline-code-wrapped tags — a whole ``MEDIA:`` tag inside inline backticks + (`` `MEDIA:/path.csv` ``) is a real delivery directive when the path + validates on disk; prose examples with non-existent paths stay masked + (#35695), and fenced code blocks are always masked. +""" + +import os + +import pytest + +from gateway.platforms.base import BasePlatformAdapter + + +@pytest.fixture() +def real_file(tmp_path): + p = tmp_path / "data.csv" + p.write_text("x,y\n1,2\n") + return str(p) + + +@pytest.fixture() +def real_targz(tmp_path): + p = tmp_path / "archive.tar.gz" + p.write_bytes(b"\x1f\x8b") + return str(p) + + +class TestTrailingPunctuation: + def test_sentence_final_period_extracts_path(self, real_file): + media, cleaned = BasePlatformAdapter.extract_media( + f"Saved your data. MEDIA:{real_file}." + ) + assert [p for p, _ in media] == [real_file] + assert "MEDIA:" not in cleaned + + def test_period_then_more_prose(self, real_file): + media, cleaned = BasePlatformAdapter.extract_media( + f"Done: MEDIA:{real_file}. Enjoy!" + ) + assert [p for p, _ in media] == [real_file] + assert "Enjoy!" in cleaned + + def test_multipart_extension_not_truncated(self, real_targz): + media, _ = BasePlatformAdapter.extract_media(f"MEDIA:{real_targz}") + assert [p for p, _ in media] == [real_targz] + + def test_multipart_extension_with_trailing_period(self, real_targz): + media, _ = BasePlatformAdapter.extract_media(f"MEDIA:{real_targz}.") + assert [p for p, _ in media] == [real_targz] + + +class TestInlineCodeWrappedTags: + def test_real_path_in_inline_code_delivers(self, real_file): + media, cleaned = BasePlatformAdapter.extract_media( + f"Here is your file `MEDIA:{real_file}`" + ) + assert [p for p, _ in media] == [real_file] + assert "MEDIA:" not in cleaned + + def test_nonexistent_path_in_inline_code_stays_masked(self): + text = "Use the format `MEDIA:/nonexistent/example.csv` to attach files." + media, cleaned = BasePlatformAdapter.extract_media(text) + assert media == [] + assert "`MEDIA:/nonexistent/example.csv`" in cleaned + + def test_fenced_code_block_always_masked(self, real_file): + text = f"```\nMEDIA:{real_file}\n```" + media, cleaned = BasePlatformAdapter.extract_media(text) + assert media == [] + assert real_file in cleaned + + def test_inline_code_non_media_untouched(self, real_file): + text = f"Run `ls -la` then see MEDIA:{real_file}" + media, cleaned = BasePlatformAdapter.extract_media(text) + assert [p for p, _ in media] == [real_file] + assert "`ls -la`" in cleaned + + +class TestEmphasisAndDedupeIntegration: + """End-to-end matrix over the salvaged contributor fixes.""" + + def test_bold_wrapped_tag_delivers(self, real_file): + media, cleaned = BasePlatformAdapter.extract_media(f"**MEDIA:{real_file}**") + assert [p for p, _ in media] == [real_file] + assert "MEDIA:" not in cleaned + + def test_two_tags_one_line_both_deliver(self, tmp_path): + a = tmp_path / "a.csv" + b = tmp_path / "b.csv" + a.write_text("1") + b.write_text("2") + media, _ = BasePlatformAdapter.extract_media(f"MEDIA:{a} MEDIA:{b}") + assert [p for p, _ in media] == [str(a), str(b)] + + def test_duplicate_tags_deliver_once(self, real_file): + media, _ = BasePlatformAdapter.extract_media( + f"MEDIA:{real_file} and again MEDIA:{real_file}" + ) + assert [p for p, _ in media] == [real_file] + + def test_glued_as_document_delivers(self, real_file): + media, _ = BasePlatformAdapter.extract_media( + f"MEDIA:{real_file}[[as_document]]" + ) + assert [p for p, _ in media] == [real_file] + + def test_unknown_extension_real_file_delivers(self, tmp_path): + p = tmp_path / "script.py" + p.write_text("print('hi')\n") + media, _ = BasePlatformAdapter.extract_media(f"MEDIA:{p}") + assert [os.path.realpath(x) for x, _ in media] == [os.path.realpath(str(p))] + + def test_extensionless_real_file_delivers(self, tmp_path): + p = tmp_path / "Caddyfile" + p.write_text("localhost\n") + media, _ = BasePlatformAdapter.extract_media(f"MEDIA:{p}") + assert [os.path.realpath(x) for x, _ in media] == [os.path.realpath(str(p))]