From 653a95f9fa55b4524452dc27279101fcda4976f3 Mon Sep 17 00:00:00 2001 From: Kshitij Kapoor Date: Sun, 19 Jul 2026 13:28:28 +0530 Subject: [PATCH] fix(state): widen surrogate scrub to remaining raw-str bind sites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the cherry-picked fix: the same UnicodeEncodeError bind failure was live at sibling sites the PR didn't cover — - api_content sidecar (append_message, _insert_message_rows, set_latest_user_api_content): bound raw; a surrogate in the composed api_content aborted the whole row/UPDATE. Scrubbing is wire-accurate: the conversation loop already scrubs every outgoing payload, so the scrubbed form IS what was sent. - tool_name (both INSERT sites): raw bind. - sanitize_title: session titles from LLM title generation or /title could carry surrogates; scrub before validation. E2E-verified each site raised UnicodeEncodeError on main and persists after this commit. Tests added for all five paths. --- hermes_state.py | 19 +++++++++++++------ tests/test_hermes_state.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/hermes_state.py b/hermes_state.py index b828890cf23..398b1442d6f 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -3304,6 +3304,10 @@ class SessionDB: if not title: return None + # Lone surrogates cannot be bound by sqlite3 (UnicodeEncodeError at + # UTF-8 encode time) — scrub them like every other write path here. + title = _sanitize_surrogates(title) + # Remove ASCII control characters (0x00-0x1F, 0x7F) but keep # whitespace chars (\t=0x09, \n=0x0A, \r=0x0D) so they can be # normalized to spaces by the whitespace collapsing step below @@ -4181,7 +4185,10 @@ class SessionDB: ``api_content`` is the exact content string sent to the API for this message when it differs from ``content`` (ephemeral memory/plugin injections, persist overrides). It is a byte-fidelity sidecar for - prompt-cache-stable replay — stored verbatim, never sanitized. + prompt-cache-stable replay — stored as sent, except lone surrogates + (which sqlite3 cannot bind and which the conversation loop scrubs + from every outgoing payload anyway, so the scrubbed form IS the + wire bytes). """ # Serialize structured fields to JSON before entering the write txn reasoning_details_json = ( @@ -4229,7 +4236,7 @@ class SessionDB: stored_content, tool_call_id, tool_calls_json, - tool_name, + _scrub_surrogates(tool_name), effect_disposition, message_timestamp, token_count, @@ -4242,7 +4249,7 @@ class SessionDB: platform_message_id, 1 if observed else 0, 1, - api_content if isinstance(api_content, str) else None, + _scrub_surrogates(api_content) if isinstance(api_content, str) else None, ), ) msg_id = cursor.lastrowid @@ -4325,7 +4332,7 @@ class SessionDB: self._encode_content(msg.get("content")), msg.get("tool_call_id"), tool_calls_json, - msg.get("tool_name"), + _scrub_surrogates(msg.get("tool_name")), msg.get("effect_disposition"), message_timestamp, msg.get("token_count"), @@ -4338,7 +4345,7 @@ class SessionDB: platform_msg_id, 1 if msg.get("observed") else 0, 1, - api_content if isinstance(api_content, str) else None, + _scrub_surrogates(api_content) if isinstance(api_content, str) else None, ), ) inserted += 1 @@ -4489,7 +4496,7 @@ class SessionDB: "WHERE session_id = ? AND role = 'user' AND active = 1 " "ORDER BY id DESC LIMIT 1" ") AND content IS ?", - (api_content, session_id, encoded), + (_scrub_surrogates(api_content), session_id, encoded), ) return cursor.rowcount diff --git a/tests/test_hermes_state.py b/tests/test_hermes_state.py index 023155f4775..414b0d69c60 100644 --- a/tests/test_hermes_state.py +++ b/tests/test_hermes_state.py @@ -6278,3 +6278,33 @@ class TestLoneSurrogatePersistence: benign = "Ünïcödé ok — 日本語 🎉 emoji fine" db.append_message("s1", "assistant", benign) assert db.get_messages("s1")[0]["content"] == benign + + # -- sibling raw-str bind sites (follow-up widening of the same bug class) + + def test_append_message_survives_lone_surrogate_api_content(self, db): + db.create_session("s1", source="cli") + db.append_message("s1", "user", "clean", api_content=self.DIRTY) + assert db.get_messages("s1")[0]["api_content"] == "scraped \ufffd price" + + def test_append_message_survives_lone_surrogate_tool_name(self, db): + db.create_session("s1", source="cli") + db.append_message("s1", "tool", "ok", tool_name="web\ud835search") + assert len(db.get_messages("s1")) == 1 + + def test_replace_messages_survives_lone_surrogate_api_content(self, db): + db.create_session("s1", source="cli") + db.replace_messages( + "s1", [{"role": "user", "content": "u1", "api_content": self.DIRTY}] + ) + assert db.get_messages("s1")[0]["api_content"] == "scraped \ufffd price" + + def test_set_latest_user_api_content_survives_lone_surrogate(self, db): + db.create_session("s1", source="cli") + db.append_message("s1", "user", "turn text") + assert db.set_latest_user_api_content("s1", "turn text", self.DIRTY) == 1 + + def test_session_title_survives_lone_surrogate(self, db): + db.create_session("s1", source="cli") + assert db.set_session_title("s1", "title \ud835 bad") is True + assert db.get_session("s1")["title"] == "title \ufffd bad" +