fix(photon): preserve text in mixed iMessage attachments (salvage #46513) (#46818)

* fix(photon): preserve text in mixed iMessage attachments

When an iMessage bubble carried both text and an attachment, spectrum-ts'
inbound mapper returned only buildAttachmentMessage(...), dropping the user's
typed text before Hermes could see it. The Photon adapter then had no 'group'
content path, so the text was lost entirely.

- adapter.py: handle a new 'group' content type that flattens text + attachment
  items, preserving the typed text alongside cached media (extracted shared
  _normalize_binary_payload helper).
- sidecar: emit 'group' content in normalizeContent, and ship
  patch-spectrum-mixed-attachments.mjs which patches spectrum-ts' pinned mapper
  (at npm postinstall AND at sidecar startup, so existing installs self-heal).

Windows robustness fixes on top of the original PR:
- The patcher's CLI guard used 'import.meta.url === file://${argv[1]}', which
  never matches on Windows (file:/// + drive letter) — it silently no-opped.
  Switched to pathToFileURL(argv[1]).href.
- The patcher matched \n-joined strings, so a CRLF checkout (Windows git
  autocrlf) defeated every replacement. It now normalizes CRLF->LF for matching
  and restores the original EOL style on write.

Co-authored-by: Yuhang Lin <yuhanglin@YuhangdeMac-mini.local>

* chore: map YuhangLin contributor email for attribution (#46513)

---------

Co-authored-by: Yuhang Lin <yuhanglin@YuhangdeMac-mini.local>
Co-authored-by: Teknium <127238744+teknium1@users.noreply.github.com>
This commit is contained in:
Austin Pickett 2026-06-17 17:14:24 -04:00 committed by GitHub
parent 7fbb8c9df5
commit fd674af47f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 491 additions and 37 deletions

View file

@ -131,10 +131,13 @@ All env vars are documented in `plugin.yaml`. The most important:
the bytes (`content.read()`) and base64-inlines them on the NDJSON event; the
adapter caches them to the shared media cache and populates `media_urls` /
`media_types`, so the agent sees the real image/file or can transcribe the
voice note — parity with the BlueBubbles iMessage channel. Media larger than
`PHOTON_MAX_INLINE_ATTACHMENT_BYTES` (default 20 MB), or any byte read that
fails, falls back to a text marker (`[Photon attachment received: …]` or
`[Photon voice received: …]`) so the agent still knows something arrived.
voice note — parity with the BlueBubbles iMessage channel. Mixed iMessage
bubbles that contain both text and attachments are normalized as a grouped
payload so the user's typed text is preserved alongside the cached media.
Media larger than `PHOTON_MAX_INLINE_ATTACHMENT_BYTES` (default 20 MB), or
any byte read that fails, falls back to a text marker (`[Photon attachment
received: …]` or `[Photon voice received: …]`) so the agent still knows
something arrived.
- **Outbound attachments are supported.** Images, voice notes, video, and
documents are sent via `space.send(attachment(...))` /
`space.send(voice(...))` through the sidecar's `/send-attachment`

View file

@ -508,6 +508,38 @@ class PhotonAdapter(BasePlatformAdapter):
media_urls: List[str] = []
media_types: List[str] = []
def _normalize_binary_payload(
payload: Dict[str, Any]
) -> tuple[str, MessageType, List[str], List[str]]:
is_voice = payload.get("type") == "voice"
name = payload.get("name") or ("voice" if is_voice else "(unnamed)")
mime = payload.get("mimeType") or ""
mtype = MessageType.VOICE if is_voice else _attachment_message_type(mime)
cached = _cache_inbound_attachment(
payload, name, mime, force_audio=is_voice
)
if cached:
return (
"(voice)" if is_voice else "(attachment)",
mtype,
[cached],
[mime or ("audio/mp4" if is_voice else "application/octet-stream")],
)
label = "voice" if is_voice else "attachment"
duration = payload.get("duration")
duration_text = (
f", duration: {duration}s"
if isinstance(duration, (int, float))
else ""
)
return (
f"[Photon {label} received: {name} "
f"({mime or 'unknown MIME'}{duration_text})]",
mtype,
[],
[],
)
ctype = content.get("type")
if ctype == "reaction":
# Route only tapbacks on messages WE sent — those are implicitly
@ -551,37 +583,40 @@ class PhotonAdapter(BasePlatformAdapter):
text = content.get("text") or ""
mtype = MessageType.TEXT
elif ctype in {"attachment", "voice"}:
is_voice = ctype == "voice"
name = content.get("name") or ("voice" if is_voice else "(unnamed)")
mime = content.get("mimeType") or ""
mtype = MessageType.VOICE if is_voice else _attachment_message_type(mime)
cached = _cache_inbound_attachment(
content, name, mime, force_audio=is_voice
)
if cached:
media_urls.append(cached)
media_types.append(
mime or ("audio/mp4" if is_voice else "application/octet-stream")
)
# The real bytes are attached, so the agent sees the media
# itself — a short marker is enough text, and it keeps group
# mention-gating consistent with plain messages.
text = "(voice)" if is_voice else "(attachment)"
else:
# No bytes (over the sidecar cap, a failed read, or a caching
# failure) — fall back to a metadata marker so the agent still
# knows something arrived.
label = "voice" if is_voice else "attachment"
duration = content.get("duration")
duration_text = (
f", duration: {duration}s"
if isinstance(duration, (int, float))
else ""
)
text = (
f"[Photon {label} received: {name} "
f"({mime or 'unknown MIME'}{duration_text})]"
)
text, mtype, media_urls, media_types = _normalize_binary_payload(content)
elif ctype == "group":
text_parts: List[str] = []
mtype = MessageType.TEXT
for item in content.get("items") or []:
if not isinstance(item, dict):
continue
item_content = item.get("content") or {}
if not isinstance(item_content, dict):
continue
item_type = item_content.get("type")
if item_type == "text":
item_text = item_content.get("text") or ""
if item_text:
text_parts.append(item_text)
continue
if item_type in {"attachment", "voice"}:
marker, item_mtype, item_urls, item_types = _normalize_binary_payload(
item_content
)
if mtype == MessageType.TEXT:
mtype = item_mtype
media_urls.extend(item_urls)
media_types.extend(item_types)
if not item_urls:
text_parts.append(marker)
continue
if item_type:
text_parts.append(f"[Photon content type not handled: {item_type}]")
if media_urls and mtype == MessageType.TEXT:
mtype = MessageType.DOCUMENT
text = "\n".join(part for part in text_parts if part).strip()
if not text:
text = "(attachment)" if media_urls else "[Photon empty group received]"
else:
text = f"[Photon content type not handled: {ctype}]"
mtype = MessageType.TEXT
@ -729,6 +764,28 @@ class PhotonAdapter(BasePlatformAdapter):
# never runs — can't leave it orphaned on the port.
env["PHOTON_SIDECAR_WATCH_STDIN"] = "1"
try:
patch = subprocess.run( # noqa: S603
[
self._node_bin,
str(_SIDECAR_DIR / "patch-spectrum-mixed-attachments.mjs"),
str(_SIDECAR_DIR),
],
capture_output=True,
text=True,
timeout=10,
check=False,
)
if patch.returncode != 0:
raise RuntimeError((patch.stderr or patch.stdout or "").strip())
if patch.stderr.strip():
logger.debug("[photon] %s", patch.stderr.strip())
except Exception as exc:
logger.warning(
"[photon] failed to apply Spectrum mixed attachment patch: %s",
exc,
)
self._sidecar_proc = subprocess.Popen( # noqa: S603
[self._node_bin, str(_SIDECAR_DIR / "index.mjs")],
stdin=subprocess.PIPE,

View file

@ -57,6 +57,7 @@
import http from "node:http";
import crypto from "node:crypto";
import { once } from "node:events";
import { patchSpectrumTs } from "./patch-spectrum-mixed-attachments.mjs";
const projectId = process.env.PHOTON_PROJECT_ID;
const projectSecret = process.env.PHOTON_PROJECT_SECRET;
@ -89,7 +90,26 @@ if (!projectId || !projectSecret || !sharedToken) {
}
// Lazy-load spectrum-ts so a missing install fails with a clear message
// instead of a cryptic module-resolution error during import.
// instead of a cryptic module-resolution error during import. Apply Hermes'
// pinned-sdk compatibility patch first so existing installs self-heal at
// runtime, not only during npm postinstall.
try {
const patchResult = patchSpectrumTs();
if (patchResult.patched) {
console.error(
`photon-sidecar: spectrum mixed attachment patch applied: ${patchResult.file}`
);
}
} catch (e) {
console.error(
"photon-sidecar: spectrum mixed attachment patch failed. " +
"Run `npm install` inside plugins/platforms/photon/sidecar/ or " +
"upgrade the Photon sidecar patch for the pinned spectrum-ts version. " +
"Original error: " +
(e && e.stack ? e.stack : String(e))
);
process.exit(3);
}
let Spectrum,
imessage,
attachment,
@ -273,6 +293,16 @@ async function normalizeContent(content) {
if (content.type === "attachment" || content.type === "voice") {
return await normalizeBinaryContent(content);
}
if (content.type === "group") {
const items = [];
for (const item of Array.isArray(content.items) ? content.items : []) {
items.push({
id: item && typeof item === "object" ? item.id ?? null : null,
content: await normalizeContent(item?.content),
});
}
return { type: "group", items };
}
if (content.type === "reaction") {
return {
type: "reaction",

View file

@ -7,6 +7,7 @@
"": {
"name": "@hermes-agent/photon-sidecar",
"version": "0.3.0",
"hasInstallScript": true,
"dependencies": {
"spectrum-ts": "3.1.0"
},

View file

@ -6,7 +6,8 @@
"type": "module",
"main": "index.mjs",
"scripts": {
"start": "node index.mjs"
"start": "node index.mjs",
"postinstall": "node patch-spectrum-mixed-attachments.mjs"
},
"engines": {
"node": ">=18.17"

View file

@ -0,0 +1,155 @@
#!/usr/bin/env node
// Patch spectrum-ts' iMessage inbound mapper until upstream preserves mixed
// text + attachment Apple events. The current spectrum-ts mapper returns only
// buildAttachmentMessage(...) whenever attachments are present, which drops
// event.message.content.text before Hermes can see it.
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
const MARKER = "Hermes patch: Preserve mixed text + attachment iMessage payloads";
function scriptDir() {
return path.dirname(fileURLToPath(import.meta.url));
}
function replaceOnce(source, from, to, label) {
const count = source.split(from).length - 1;
if (count !== 1) {
throw new Error(`expected exactly one ${label} match, found ${count}`);
}
return source.replace(from, to);
}
function replaceFirst(source, from, to, label) {
if (!source.includes(from)) {
throw new Error(`expected at least one ${label} match, found 0`);
}
return source.replace(from, to);
}
function addTextChildSnippet(messageExpr) {
return `if (text2) {\n items.unshift({\n ...base,\n id: formatChildId(0, messageGuidStr),\n content: asText(text2),\n partIndex: 0,\n parentId: messageGuidStr\n });\n }`;
}
function patchRebuild(source) {
source = replaceOnce(
source,
` const attachments = messageAttachments(message);\n if (attachments.length === 1) {`,
` const attachments = messageAttachments(message);\n const text2 = message.content.text;\n if (attachments.length === 1) {`,
"rebuild text capture"
);
source = replaceOnce(
source,
` return buildAttachmentMessage(client, base, info, messageGuidStr, 0);`,
` const msg2 = await buildAttachmentMessage(\n client,\n base,\n info,\n text2 ? formatChildId(1, messageGuidStr) : messageGuidStr,\n text2 ? 1 : 0,\n text2 ? messageGuidStr : void 0\n );\n if (text2) {\n const textMsg = {\n ...base,\n id: formatChildId(0, messageGuidStr),\n content: asText(text2),\n partIndex: 0,\n parentId: messageGuidStr\n };\n return {\n ...base,\n id: messageGuidStr,\n content: asProviderGroup([textMsg, msg2])\n };\n }\n return msg2;`,
"rebuild single attachment"
);
source = replaceFirst(
source,
` formatChildId(i, messageGuidStr),\n i,\n messageGuidStr`,
` formatChildId(text2 ? i + 1 : i, messageGuidStr),\n text2 ? i + 1 : i,\n messageGuidStr`,
"rebuild multi attachment child index"
);
source = replaceFirst(
source,
` return {\n ...base,\n id: messageGuidStr,\n content: asProviderGroup(items)\n };\n }\n if (getBalloonBundleId(message) === URL_BALLOON_BUNDLE_ID) {`,
` ${addTextChildSnippet("message")}\n return {\n ...base,\n id: messageGuidStr,\n content: asProviderGroup(items)\n };\n }\n if (getBalloonBundleId(message) === URL_BALLOON_BUNDLE_ID) {`,
"rebuild multi attachment text child"
);
source = replaceFirst(
source,
` const text2 = message.content.text;\n return {\n ...base,`,
` return {\n ...base,`,
"rebuild duplicate text declaration"
);
return source;
}
function patchInbound(source) {
source = replaceOnce(
source,
` const attachments = messageAttachments(event.message);\n if (attachments.length === 1) {`,
` const attachments = messageAttachments(event.message);\n const text2 = event.message.content.text;\n if (attachments.length === 1) {`,
"inbound text capture"
);
source = replaceOnce(
source,
` messageGuidStr,\n 0\n );\n cacheMessage(cache, msg2);\n return [msg2];`,
` text2 ? formatChildId(1, messageGuidStr) : messageGuidStr,\n text2 ? 1 : 0,\n text2 ? messageGuidStr : void 0\n );\n if (text2) {\n const textMsg = {\n ...base,\n id: formatChildId(0, messageGuidStr),\n content: asText(text2),\n partIndex: 0,\n parentId: messageGuidStr\n };\n const parent = {\n ...base,\n id: messageGuidStr,\n content: asProviderGroup([textMsg, msg2])\n };\n cacheMessage(cache, parent);\n return [parent];\n }\n cacheMessage(cache, msg2);\n return [msg2];`,
"inbound single attachment"
);
source = replaceOnce(
source,
` formatChildId(i, messageGuidStr),\n i,\n messageGuidStr`,
` formatChildId(text2 ? i + 1 : i, messageGuidStr),\n text2 ? i + 1 : i,\n messageGuidStr`,
"inbound multi attachment child index"
);
source = replaceOnce(
source,
` const parent = {\n ...base,\n id: messageGuidStr,\n content: asProviderGroup(items)\n };`,
` ${addTextChildSnippet("event.message")}\n const parent = {\n ...base,\n id: messageGuidStr,\n content: asProviderGroup(items)\n };`,
"inbound multi attachment text child"
);
source = replaceOnce(
source,
` const text2 = event.message.content.text;\n const msg = {`,
` const msg = {`,
"inbound duplicate text declaration"
);
return source;
}
export function patchSpectrumTs(root = scriptDir()) {
const dist = path.join(root, "node_modules", "spectrum-ts", "dist");
if (!fs.existsSync(dist)) {
throw new Error(`spectrum-ts dist not found: ${dist}`);
}
const files = fs.readdirSync(dist)
.filter((name) => name.endsWith(".js"))
.map((name) => path.join(dist, name));
for (const file of files) {
const raw = fs.readFileSync(file, "utf8");
if (raw.includes(MARKER)) {
return { patched: false, file, reason: "already patched" };
}
// Normalize to LF for matching so the patch works regardless of the
// checkout's line-ending style (Windows git autocrlf produces CRLF,
// which would otherwise defeat the \n-based search strings). The
// original EOL style is restored on write.
const CR = String.fromCharCode(13);
const CRLF = CR + "\n";
const usedCRLF = raw.includes(CRLF);
const original = usedCRLF ? raw.split(CRLF).join("\n") : raw;
if (!original.includes("var toInboundMessages = async") ||
!original.includes("var rebuildFromAppleMessage = async")) {
continue;
}
let patched = original;
patched = patchRebuild(patched);
patched = patchInbound(patched);
patched = `// ${MARKER}\n${patched}`;
if (usedCRLF) {
patched = patched.split("\n").join(CRLF);
}
fs.writeFileSync(file, patched, "utf8");
return { patched: true, file };
}
throw new Error("could not find spectrum-ts iMessage inbound chunk to patch");
}
const _invokedDirectly =
process.argv[1] &&
import.meta.url === pathToFileURL(process.argv[1]).href;
if (_invokedDirectly) {
try {
const root = process.argv[2] ? path.resolve(process.argv[2]) : scriptDir();
const result = patchSpectrumTs(root);
const action = result.patched ? "patched" : "ok";
console.error(`photon-sidecar: spectrum mixed attachment patch ${action}: ${result.file}`);
} catch (err) {
console.error(`photon-sidecar: spectrum mixed attachment patch failed: ${err?.stack || err}`);
process.exit(1);
}
}

View file

@ -45,6 +45,7 @@ ACP_REGISTRY_MANIFEST = REPO_ROOT / "acp_registry" / "agent.json"
# Auto-extracted from noreply emails + manual overrides
AUTHOR_MAP = {
"yuhanglin@YuhangdeMac-mini.local": "1960697431",
"despitemeguru@gmail.com": "definitelynotguru",
"chaslui@outlook.com": "ChasLui",
"rio.jeong@thebytesize.ai": "rio-jeong",

View file

@ -168,6 +168,56 @@ async def test_dispatch_attachment_downloads_image(
cached.unlink(missing_ok=True)
@pytest.mark.asyncio
async def test_dispatch_group_preserves_text_and_attachment(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Spectrum group content from a mixed text+image iMessage must not drop text."""
adapter = _make_adapter(monkeypatch)
captured = _capture(adapter, monkeypatch)
raw = base64.b64decode(_PNG_1X1_B64)
event = _attachment_event(
{},
msg_id="spc-msg-mixed",
)
event["content"] = {
"type": "group",
"items": [
{
"id": "p:0/spc-msg-mixed",
"content": {"type": "text", "text": "请分析这张图的重点"},
},
{
"id": "p:1/spc-msg-mixed",
"content": {
"type": "attachment",
"name": "photo.png",
"mimeType": "image/png",
"size": len(raw),
"data": _PNG_1X1_B64,
"encoding": "base64",
},
},
],
}
await adapter._dispatch_inbound(event)
assert len(captured) == 1
ev = captured[0]
assert ev.text == "请分析这张图的重点"
assert ev.message_type == MessageType.PHOTO
assert ev.media_types == ["image/png"]
assert len(ev.media_urls) == 1
cached = Path(ev.media_urls[0])
try:
assert cached.is_file()
assert cached.read_bytes() == raw
finally:
cached.unlink(missing_ok=True)
@pytest.mark.asyncio
async def test_dispatch_voice_downloads_audio(
monkeypatch: pytest.MonkeyPatch,

View file

@ -0,0 +1,156 @@
"""Regression tests for Hermes' Spectrum mixed text+attachment workaround."""
from __future__ import annotations
import subprocess
import textwrap
from pathlib import Path
_PATCHER = Path("plugins/platforms/photon/sidecar/patch-spectrum-mixed-attachments.mjs")
def test_sidecar_applies_spectrum_patch_before_importing_sdk() -> None:
"""Existing installs should self-heal at runtime, not only during npm postinstall."""
index = Path("plugins/platforms/photon/sidecar/index.mjs").read_text(encoding="utf-8")
assert "import { patchSpectrumTs }" in index
assert "patchSpectrumTs();" in index
assert index.index("patchSpectrumTs();") < index.index('await import("spectrum-ts")')
def test_spectrum_patch_preserves_text_when_single_attachment(tmp_path: Path) -> None:
"""The sidecar dependency patch must turn text+one attachment into group content."""
dist = tmp_path / "node_modules" / "spectrum-ts" / "dist"
dist.mkdir(parents=True)
chunk = dist / "chunk-test.js"
chunk.write_text(
textwrap.dedent(
"""
var rebuildFromAppleMessage = async (client, message, phone, chatGuidHint) => {
const messageGuidStr = message.guid;
const timestamp = message.dateCreated ?? /* @__PURE__ */ new Date();
const base = buildMessageBase(message, chatGuidHint, timestamp, phone);
const attachments = messageAttachments(message);
if (attachments.length === 1) {
const info = attachments[0];
if (!info) {
throw new Error("Unreachable: attachments.length === 1 but no element");
}
return buildAttachmentMessage(client, base, info, messageGuidStr, 0);
}
if (attachments.length > 1) {
const items = [];
for (let i = 0; i < attachments.length; i++) {
const info = attachments[i];
if (!info) {
continue;
}
items.push(
await buildAttachmentMessage(
client,
base,
info,
formatChildId(i, messageGuidStr),
i,
messageGuidStr
)
);
}
return {
...base,
id: messageGuidStr,
content: asProviderGroup(items)
};
}
if (getBalloonBundleId(message) === URL_BALLOON_BUNDLE_ID) {
return toRichlinkMessage(message, base, messageGuidStr);
}
const text2 = message.content.text;
return {
...base,
id: messageGuidStr,
content: text2 ? asText(text2) : asCustom(message)
};
};
var toInboundMessages = async (client, cache, event, phone) => {
const base = buildMessageBase(
event.message,
event.chatGuid,
event.occurredAt,
phone
);
const messageGuidStr = event.message.guid;
if (getBalloonBundleId(event.message) === URL_BALLOON_BUNDLE_ID) {
const msg2 = toRichlinkMessage(event.message, base, messageGuidStr);
cacheMessage(cache, msg2);
return [msg2];
}
const attachments = messageAttachments(event.message);
if (attachments.length === 1) {
const info = attachments[0];
if (!info) {
throw new Error("Unreachable: attachments.length === 1 but no element");
}
const msg2 = await buildAttachmentMessage(
client,
base,
info,
messageGuidStr,
0
);
cacheMessage(cache, msg2);
return [msg2];
}
if (attachments.length > 1) {
const items = [];
for (let i = 0; i < attachments.length; i++) {
const info = attachments[i];
if (!info) {
continue;
}
items.push(
await buildAttachmentMessage(
client,
base,
info,
formatChildId(i, messageGuidStr),
i,
messageGuidStr
)
);
}
const parent = {
...base,
id: messageGuidStr,
content: asProviderGroup(items)
};
cacheMessage(cache, parent);
return [parent];
}
const text2 = event.message.content.text;
const msg = {
...base,
id: messageGuidStr,
content: text2 ? asText(text2) : asCustom(event.message)
};
cacheMessage(cache, msg);
return [msg];
};
"""
),
encoding="utf-8",
)
result = subprocess.run(
["node", str(_PATCHER), str(tmp_path)],
cwd=Path.cwd(),
text=True,
capture_output=True,
check=False,
)
assert result.returncode == 0, result.stderr
patched = chunk.read_text(encoding="utf-8")
assert "Preserve mixed text + attachment iMessage payloads" in patched
assert "content: asProviderGroup([textMsg, msg2])" in patched
assert "content: asProviderGroup(items)" in patched
assert "formatChildId(text2 ? i + 1 : i, messageGuidStr)" in patched