From c2e4d6a0e58dd5f0ad80565388327db8e9999fd8 Mon Sep 17 00:00:00 2001 From: Teknium Date: Thu, 16 Apr 2026 17:11:11 -0700 Subject: [PATCH] feat(sessions): add --sanitize flag to sessions export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port from anomalyco/opencode#22489: redact user/model content from session exports before sharing for bug reports or training data. Adds hermes_state.sanitize_session_export() which returns a deep-copied session with: - Message content, reasoning, and reasoning_details replaced with [redacted::] tokens - Tool-call arguments redacted (tool id, type, and function name preserved) - Session title and system_prompt redacted - All structural/metric fields preserved: ids, timestamps, token counts, tool names, finish reasons, model info, cost data, message counts Wired into 'hermes sessions export --sanitize' (applies to both --session-id and full exports). The flag is opt-in — default behaviour is unchanged. User sees '(sanitized)' suffix on the export summary when the flag is active. 5 new tests covering content redaction, reasoning/tool-call redaction, empty-value preservation, input immutability, and reasoning_details block structure. E2E verified: raw export still leaks sk-proj-* API keys and usernames, sanitized export replaces them with redaction tokens while preserving model names, tool names, and tool call ids. Authored-by: Hermes Agent (autonomous weekly OpenCode PR scout) --- hermes_cli/main.py | 31 +++++++- hermes_state.py | 150 ++++++++++++++++++++++++++++++++++++ tests/test_hermes_state.py | 151 +++++++++++++++++++++++++++++++++++++ 3 files changed, 328 insertions(+), 4 deletions(-) diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 243bad599..0b9c44e8f 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -5928,6 +5928,13 @@ Examples: sessions_export.add_argument("output", help="Output JSONL file path (use - for stdout)") sessions_export.add_argument("--source", help="Filter by source") sessions_export.add_argument("--session-id", help="Export a specific session") + sessions_export.add_argument( + "--sanitize", + action="store_true", + help="Redact user/model content (message text, reasoning, tool args/output, titles, " + "system prompt) before export. Structure and metrics are preserved. " + "Use when sharing exports for bug reports or training data.", + ) sessions_delete = sessions_subparsers.add_parser("delete", help="Delete a specific session") sessions_delete.add_argument("session_id", help="Session ID to delete") @@ -5997,6 +6004,19 @@ Examples: print(f"{preview:<50} {last_active:<13} {s['source']:<6} {sid}") elif action == "export": + sanitize = getattr(args, "sanitize", False) + if sanitize: + try: + from hermes_state import sanitize_session_export as _sanitize_fn + except Exception: + _sanitize_fn = None + print("Warning: sanitize_session_export unavailable — exporting raw data.") + else: + _sanitize_fn = None + + def _maybe_sanitize(d): + return _sanitize_fn(d) if _sanitize_fn else d + if args.session_id: resolved_session_id = db.resolve_session_id(args.session_id) if not resolved_session_id: @@ -6006,6 +6026,7 @@ Examples: if not data: print(f"Session '{args.session_id}' not found.") return + data = _maybe_sanitize(data) line = _json.dumps(data, ensure_ascii=False) + "\n" if args.output == "-": import sys @@ -6013,18 +6034,20 @@ Examples: else: with open(args.output, "w", encoding="utf-8") as f: f.write(line) - print(f"Exported 1 session to {args.output}") + suffix = " (sanitized)" if sanitize and _sanitize_fn else "" + print(f"Exported 1 session to {args.output}{suffix}") else: sessions = db.export_all(source=args.source) if args.output == "-": import sys for s in sessions: - sys.stdout.write(_json.dumps(s, ensure_ascii=False) + "\n") + sys.stdout.write(_json.dumps(_maybe_sanitize(s), ensure_ascii=False) + "\n") else: with open(args.output, "w", encoding="utf-8") as f: for s in sessions: - f.write(_json.dumps(s, ensure_ascii=False) + "\n") - print(f"Exported {len(sessions)} sessions to {args.output}") + f.write(_json.dumps(_maybe_sanitize(s), ensure_ascii=False) + "\n") + suffix = " (sanitized)" if sanitize and _sanitize_fn else "" + print(f"Exported {len(sessions)} sessions to {args.output}{suffix}") elif action == "delete": resolved_session_id = db.resolve_session_id(args.session_id) diff --git a/hermes_state.py b/hermes_state.py index 5e563666e..d1dd7516b 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -1160,6 +1160,23 @@ class SessionDB: results.append({**session, "messages": messages}) return results + # --------------------------------------------------------------- + # Export sanitization + # --------------------------------------------------------------- + # + # When users share session exports for debugging or training, the + # raw JSON contains every user message, tool output, and reasoning + # trace — which often includes file contents, command output, env + # variables, paths, and other confidential information. + # + # ``sanitize_session_export`` produces a deep copy of the export + # with all content fields replaced by opaque ``[redacted::]`` + # tokens. Structural metadata (IDs, roles, timestamps, token counts, + # tool names, finish reasons, model info, cost data) is preserved + # so that the shape of a conversation is still analysable. + # + # Inspired by anomalyco/opencode#22489 (opencode's ``export --sanitize``). + def clear_messages(self, session_id: str) -> None: """Delete all messages for a session and reset its counters.""" def _do(conn): @@ -1236,3 +1253,136 @@ class SessionDB: return len(session_ids) return self._execute_write(_do) + + +# ========================================================================= +# Session export sanitization +# ========================================================================= +# +# Ported from anomalyco/opencode#22489 — users often want to share a +# session export for bug reports, feature requests, or training data +# collection, but the raw export contains every user prompt, tool +# output, file content, and reasoning trace. ``sanitize_session_export`` +# replaces content fields with opaque tokens while preserving the +# conversation's structure and metrics. + +# Message-level content fields that are always redacted on a message. +_REDACT_MSG_STRING_FIELDS = ( + "content", + "reasoning", +) + +# Session-level fields that can contain user-facing text. +_REDACT_SESSION_STRING_FIELDS = ( + "system_prompt", + "title", +) + + +def _redact_token(kind: str, id_: Any, value: Any) -> Any: + """Produce an opaque redaction token. Preserves empty/None values.""" + if value in (None, "", b""): + return value + return f"[redacted:{kind}:{id_}]" + + +def _redact_tool_call(call: Any, msg_id: Any, index: int) -> Any: + """Redact arguments inside a tool_call while preserving structure (id, name).""" + if not isinstance(call, dict): + return call + out = dict(call) + tcid = out.get("id") or f"{msg_id}-{index}" + fn = out.get("function") + if isinstance(fn, dict): + new_fn = dict(fn) + if "arguments" in new_fn and new_fn["arguments"] not in (None, "", "{}"): + new_fn["arguments"] = _redact_token("tool-input", tcid, new_fn["arguments"]) + out["function"] = new_fn + # Some schemas put args at the top level rather than under ``function``. + if "arguments" in out and out["arguments"] not in (None, "", "{}"): + out["arguments"] = _redact_token("tool-input", tcid, out["arguments"]) + return out + + +def _redact_reasoning_details(details: Any, msg_id: Any) -> Any: + """Redact text inside OpenAI / Anthropic reasoning_details blocks. + + ``reasoning_details`` is a list of dicts with shapes like:: + + {"type": "reasoning.text", "text": "..."} + {"type": "reasoning.encrypted", "data": "..."} + {"type": "reasoning.summary", "summary": "..."} + + We preserve the block type/structure and redact the inner payload. + """ + if not isinstance(details, list): + return details + out = [] + for idx, block in enumerate(details): + if not isinstance(block, dict): + out.append(block) + continue + new_block = dict(block) + for key in ("text", "data", "summary", "content"): + if key in new_block and new_block[key] not in (None, ""): + new_block[key] = _redact_token(f"reasoning-{key}", f"{msg_id}-{idx}", new_block[key]) + out.append(new_block) + return out + + +def _redact_message(msg: Dict[str, Any]) -> Dict[str, Any]: + """Return a sanitized copy of a single message row.""" + if not isinstance(msg, dict): + return msg + msg_id = msg.get("id", "msg") + out = dict(msg) + + # Plain string content fields. + for field in _REDACT_MSG_STRING_FIELDS: + if field in out and out[field] not in (None, ""): + out[field] = _redact_token(field.replace("_", "-"), msg_id, out[field]) + + # Tool calls: keep structure (id, name) but redact arguments. + tcs = out.get("tool_calls") + if isinstance(tcs, list): + out["tool_calls"] = [_redact_tool_call(tc, msg_id, i) for i, tc in enumerate(tcs)] + + # Reasoning details: preserve block structure, redact text/data. + if "reasoning_details" in out: + out["reasoning_details"] = _redact_reasoning_details(out["reasoning_details"], msg_id) + + # Codex reasoning items follow the same shape as reasoning_details. + if "codex_reasoning_items" in out: + out["codex_reasoning_items"] = _redact_reasoning_details(out["codex_reasoning_items"], msg_id) + + return out + + +def sanitize_session_export(session: Dict[str, Any]) -> Dict[str, Any]: + """Return a deep-sanitized copy of a session export. + + All user-facing content (message text, reasoning, tool arguments and + outputs, system prompt, title) is replaced by ``[redacted::]`` + tokens. Structural metadata (ids, timestamps, token counts, tool names, + model/provider info, cost data, finish reasons) is preserved so the + export remains useful for debugging schema issues, analysing tool-use + patterns, or counting sessions without leaking confidential data. + + The input dict is not mutated. + """ + if not isinstance(session, dict): + return session + sid = session.get("id", "session") + out = dict(session) + + # Session-level text fields (title, system prompt). + for field in _REDACT_SESSION_STRING_FIELDS: + if field in out and out[field] not in (None, ""): + out[field] = _redact_token(field.replace("_", "-"), sid, out[field]) + + # Messages list: sanitize each row. + msgs = out.get("messages") + if isinstance(msgs, list): + out["messages"] = [_redact_message(m) for m in msgs] + + return out diff --git a/tests/test_hermes_state.py b/tests/test_hermes_state.py index 5f9a16a52..35479fef9 100644 --- a/tests/test_hermes_state.py +++ b/tests/test_hermes_state.py @@ -1,5 +1,6 @@ """Tests for hermes_state.py — SessionDB SQLite CRUD, FTS5 search, export.""" +import json import time import pytest from pathlib import Path @@ -609,6 +610,156 @@ class TestDeleteAndExport: assert exports[0]["source"] == "cli" +# ========================================================================= +# Export sanitization (ported from anomalyco/opencode#22489) +# ========================================================================= + +class TestSanitizeSessionExport: + """Validate that sanitize_session_export redacts user content while + preserving structural metadata useful for analysis.""" + + def test_redacts_message_content(self, db): + from hermes_state import sanitize_session_export + + db.create_session(session_id="s1", source="cli", model="test", system_prompt="secret prompt") + db.set_session_title("s1", "my confidential task") + db.append_message("s1", role="user", content="what is my password?") + db.append_message("s1", role="assistant", content="Here's your secret: XYZ") + + raw = db.export_session("s1") + sanitized = sanitize_session_export(raw) + + # Structural / metric fields are preserved. + assert sanitized["id"] == "s1" + assert sanitized["source"] == "cli" + assert sanitized["model"] == "test" + assert len(sanitized["messages"]) == 2 + for msg in sanitized["messages"]: + assert "role" in msg + assert msg["role"] in ("user", "assistant") + assert "id" in msg + assert "timestamp" in msg + + # Content is redacted. + assert "password" not in json.dumps(sanitized) + assert "XYZ" not in json.dumps(sanitized) + assert "confidential" not in json.dumps(sanitized) + assert "secret prompt" not in json.dumps(sanitized) + for msg in sanitized["messages"]: + assert msg["content"].startswith("[redacted:content:") + + # Title and system_prompt are redacted. + assert sanitized["title"].startswith("[redacted:title:") + assert sanitized["system_prompt"].startswith("[redacted:system-prompt:") + + def test_redacts_reasoning_and_tool_calls(self, db): + from hermes_state import sanitize_session_export + + db.create_session(session_id="s1", source="cli") + db.append_message( + "s1", + role="assistant", + content="let me search", + reasoning="user asked about their private API key", + tool_calls=[{ + "id": "tc_1", + "type": "function", + "function": { + "name": "terminal", + "arguments": '{"command": "cat /etc/passwd"}', + }, + }], + ) + db.append_message( + "s1", + role="tool", + content="root:x:0:0:root:/root:/bin/bash", + tool_call_id="tc_1", + tool_name="terminal", + ) + + raw = db.export_session("s1") + sanitized = sanitize_session_export(raw) + dumped = json.dumps(sanitized) + + # No leaked content. + assert "private API key" not in dumped + assert "/etc/passwd" not in dumped + assert "root:x:0:0" not in dumped + assert "cat" not in dumped # the command body should not leak + + # Tool call structure preserved (id, type, function name). + asst = sanitized["messages"][0] + assert asst["tool_calls"][0]["id"] == "tc_1" + assert asst["tool_calls"][0]["type"] == "function" + assert asst["tool_calls"][0]["function"]["name"] == "terminal" + assert asst["tool_calls"][0]["function"]["arguments"].startswith("[redacted:tool-input:") + + # Reasoning field redacted but present. + assert asst["reasoning"].startswith("[redacted:reasoning:") + + # Tool response metadata preserved (tool_call_id, tool_name). + tool_msg = sanitized["messages"][1] + assert tool_msg["tool_call_id"] == "tc_1" + assert tool_msg["tool_name"] == "terminal" + assert tool_msg["content"].startswith("[redacted:content:") + + def test_preserves_empty_values(self, db): + """Empty/None content should pass through untouched so consumers + don't treat sanitization as 'there was hidden data here'.""" + from hermes_state import sanitize_session_export + + db.create_session(session_id="s1", source="cli") + db.append_message("s1", role="user", content="") + raw = db.export_session("s1") + sanitized = sanitize_session_export(raw) + + # Empty content stays empty (not a fake redaction token). + assert sanitized["messages"][0]["content"] in ("", None) + + def test_does_not_mutate_input(self, db): + from hermes_state import sanitize_session_export + + db.create_session(session_id="s1", source="cli") + db.append_message("s1", role="user", content="original text") + raw = db.export_session("s1") + original_content = raw["messages"][0]["content"] + + sanitize_session_export(raw) + + # Original dict is unchanged. + assert raw["messages"][0]["content"] == original_content + + def test_redacts_reasoning_details_blocks(self): + """reasoning_details is a list of typed blocks — preserve type, redact payload.""" + from hermes_state import sanitize_session_export + + session = { + "id": "s1", + "source": "cli", + "messages": [{ + "id": "m1", + "role": "assistant", + "content": "done", + "reasoning_details": [ + {"type": "reasoning.text", "text": "sensitive internal thought"}, + {"type": "reasoning.encrypted", "data": "encrypted_blob_XYZ"}, + ], + }], + } + sanitized = sanitize_session_export(session) + dumped = json.dumps(sanitized) + + assert "sensitive internal thought" not in dumped + assert "encrypted_blob_XYZ" not in dumped + # Block types preserved. + blocks = sanitized["messages"][0]["reasoning_details"] + assert blocks[0]["type"] == "reasoning.text" + assert blocks[0]["text"].startswith("[redacted:reasoning-text:") + assert blocks[1]["type"] == "reasoning.encrypted" + assert blocks[1]["data"].startswith("[redacted:reasoning-data:") + + # ========================================================================= # Prune # =========================================================================