diff --git a/hermes_state.py b/hermes_state.py index 8c3b3079ceb..d8da7719dfe 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -43,6 +43,35 @@ from hermes_cli.sqlite_runtime import ( ) from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar +from hermes_state_common import ( # noqa: F401 (re-exported for back-compat) + _BRANCH_CHILD_SQL, + _COMPRESSION_CHILD_SQL, + _FTS_CJK_TRIGGERS, + _FTS_TRIGGERS, + _LISTABLE_CHILD_SQL, + _PREVIEW_RAW_SELECT, + _ephemeral_child_sql, + _shape_preview, + DEFERRED_INDEX_SQL, + FTS_CJK_STALE_KEY, + FTS_SQL, + FTS_STORAGE_VERSION, + FTS_TRIGRAM_SQL, + LEGACY_FTS_SQL, + LEGACY_FTS_TRIGRAM_SQL, + MAX_FTS5_QUERY_CHARS, + SCHEMA_SQL, + SCHEMA_VERSION, + _PREVIEW_CONTENT_SQL, + _PREVIEW_HEAD_CHARS, + _PREVIEW_MAX_CHARS, + _PREVIEW_SCAFFOLD_WINDOW, + _PREVIEW_SCAFFOLDED_SQL, +) +from hermes_state_portability import SessionPortabilityMixin +from hermes_state_schema import SessionSchemaMixin +from hermes_state_search import SessionSearchMixin + try: # Hard dependency, but tolerate scaffold-phase imports before pip install. import psutil except ImportError: # pragma: no cover - stripped/scaffold installs only @@ -156,83 +185,6 @@ def _workspace_key_clause(key: str) -> Tuple[str, List[str]]: ) -# Session preview = the head of the first user message, shown wherever a -# session has no title (sidebar rows, pickers, exports, the desktop's -# `sessionTitle` fallback). -# -# A /skill invocation expands into a message that embeds the whole skill body, -# so the plain head of it previews the SKILL's opening prose as if the user had -# written it. Scaffolded rows therefore carry a wider excerpt so -# ``_shape_preview`` can hand it to ``describe_skill_invocation`` and recover -# ``/work — fix the title leak``: the whole message while it stays under the -# budget, and head + tail (where the typed instruction lands) once it doesn't. -_PREVIEW_HEAD_CHARS = 63 -_PREVIEW_SCAFFOLD_WINDOW = 400 -_PREVIEW_MAX_CHARS = 60 - -_PREVIEW_CONTENT_SQL = "REPLACE(REPLACE(m.content, X'0A', ' '), X'0D', ' ')" -_PREVIEW_SCAFFOLDED_SQL = f"m.content LIKE '{SKILL_SCAFFOLD_SQL_LIKE}'" - -# The shared ``_preview_raw`` SELECT expression, interpolated by every listing -# query. A scaffolded row gets a wider excerpt: the whole message while it fits -# the budget, else head + tail (where the typed instruction lands) spliced -# around SKILL_EXCERPT_JOINT. -_PREVIEW_RAW_SELECT = ( - f"CASE WHEN {_PREVIEW_SCAFFOLDED_SQL}" - f" AND LENGTH(m.content) > {_PREVIEW_SCAFFOLD_WINDOW * 2}" - f" THEN SUBSTR({_PREVIEW_CONTENT_SQL}, 1, {_PREVIEW_SCAFFOLD_WINDOW})" - f" || '{SKILL_EXCERPT_JOINT}'" - f" || SUBSTR({_PREVIEW_CONTENT_SQL}, -{_PREVIEW_SCAFFOLD_WINDOW})" - f" WHEN {_PREVIEW_SCAFFOLDED_SQL}" - f" THEN SUBSTR({_PREVIEW_CONTENT_SQL}, 1, {_PREVIEW_SCAFFOLD_WINDOW * 2})" - f" ELSE SUBSTR({_PREVIEW_CONTENT_SQL}, 1, {_PREVIEW_HEAD_CHARS}) END" -) - - -def _shape_preview(raw: Any) -> str: - """Turn a ``_preview_raw`` column into the short preview callers show.""" - text = str(raw or "").strip() - if not text: - return "" - described = describe_skill_invocation(text) - text = described if described is not None else text.split(SKILL_EXCERPT_JOINT)[0] - if len(text) > _PREVIEW_MAX_CHARS: - return text[:_PREVIEW_MAX_CHARS] + "..." - return text - - -# A child session counts as a /branch (kept visible, never cascade-deleted) if -# it carries the stable marker OR the legacy end_reason heuristic holds. -_BRANCH_CHILD_SQL = ( - "json_extract(COALESCE({a}.model_config, '{{}}'), '$._branched_from') IS NOT NULL" - " OR EXISTS (SELECT 1 FROM sessions p" - " WHERE p.id = {a}.parent_session_id" - " AND p.end_reason = 'branched'" - " AND {a}.started_at >= p.ended_at)" -) - -_COMPRESSION_CHILD_SQL = ( - "EXISTS (SELECT 1 FROM sessions p" - " WHERE p.id = {a}.parent_session_id" - " AND p.end_reason = 'compression')" -) - -# Rows that surface in pickers: roots + branch children (subagent runs and -# compression continuations stay hidden). -_LISTABLE_CHILD_SQL = f"(s.parent_session_id IS NULL OR {_BRANCH_CHILD_SQL.format(a='s')})" - - -def _ephemeral_child_sql(alias: str = "s") -> str: - """Subagent runs (cascade-delete targets), not branches or compression tips.""" - branch = _BRANCH_CHILD_SQL.format(a=alias) - compression = _COMPRESSION_CHILD_SQL.format(a=alias) - return ( - f"({alias}.parent_session_id IS NOT NULL" - f" AND NOT ({branch})" - f" AND NOT ({compression}))" - ) - - def _collect_delegate_child_ids(conn, parent_ids: List[str]) -> List[str]: """Delegate-subagent ids to cascade-delete with *parent_ids*. @@ -283,23 +235,6 @@ T = TypeVar("T") DEFAULT_DB_PATH = get_hermes_home() / "state.db" -SCHEMA_VERSION = 23 - -# FTS storage-layout version, tracked INDEPENDENTLY of SCHEMA_VERSION in the -# state_meta key ``fts_storage_version``. The main schema version advances -# freely on open (so future migrations always land); the FTS *layout* only -# reaches the current version when a DB is either born fresh or explicitly -# optimized via ``hermes sessions optimize-storage``. A legacy DB sits at -# layout 0 (marker absent) with a working inline index until the user opts in. -# 1 = v23 external-content layout (content/tool_name/tool_calls, -# tool-row-excluded trigram) -FTS_STORAGE_VERSION = 1 - -# Cap on user-controlled FTS5 query input before regex/sanitizer processing. -# Search queries do not need to be arbitrarily large, and bounding them keeps -# sanitizer/runtime behavior predictable under adversarial input. -MAX_FTS5_QUERY_CHARS = 2_048 - # --------------------------------------------------------------------------- # WAL-compatibility fallback # --------------------------------------------------------------------------- @@ -350,16 +285,6 @@ _wal_fallback_warned_lock = threading.Lock() _wal_reset_bug_warned_paths: set[str] = set() _wal_reset_bug_warned_lock = threading.Lock() -_FTS_TRIGGERS = ( - "messages_fts_insert", - "messages_fts_delete", - "messages_fts_update", - "messages_fts_trigram_insert", - "messages_fts_trigram_delete", - "messages_fts_trigram_update", -) - - def _set_last_init_error(msg: Optional[str]) -> None: """Record (or clear) the most recent state.db init failure. @@ -1233,315 +1158,6 @@ def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, A return report -SCHEMA_SQL = """ -CREATE TABLE IF NOT EXISTS schema_version ( - version INTEGER NOT NULL -); - -CREATE TABLE IF NOT EXISTS sessions ( - id TEXT PRIMARY KEY, - source TEXT NOT NULL, - user_id TEXT, - session_key TEXT, - chat_id TEXT, - chat_type TEXT, - thread_id TEXT, - display_name TEXT, - origin_json TEXT, - expiry_finalized INTEGER DEFAULT 0, - model TEXT, - model_config TEXT, - system_prompt TEXT, - parent_session_id TEXT, - started_at REAL NOT NULL, - ended_at REAL, - end_reason TEXT, - message_count INTEGER DEFAULT 0, - tool_call_count INTEGER DEFAULT 0, - input_tokens INTEGER DEFAULT 0, - output_tokens INTEGER DEFAULT 0, - cache_read_tokens INTEGER DEFAULT 0, - cache_write_tokens INTEGER DEFAULT 0, - reasoning_tokens INTEGER DEFAULT 0, - cwd TEXT, - git_branch TEXT, - git_repo_root TEXT, - billing_provider TEXT, - billing_base_url TEXT, - billing_mode TEXT, - estimated_cost_usd REAL, - actual_cost_usd REAL, - cost_status TEXT, - cost_source TEXT, - pricing_version TEXT, - title TEXT, - api_call_count INTEGER DEFAULT 0, - handoff_state TEXT, - handoff_platform TEXT, - handoff_error TEXT, - compression_failure_cooldown_until REAL, - compression_failure_error TEXT, - compression_fallback_streak INTEGER NOT NULL DEFAULT 0, - compression_ineffective_count INTEGER NOT NULL DEFAULT 0, - profile_name TEXT, - rewind_count INTEGER NOT NULL DEFAULT 0, - archived INTEGER NOT NULL DEFAULT 0, - pinned INTEGER NOT NULL DEFAULT 0, - FOREIGN KEY (parent_session_id) REFERENCES sessions(id) -); - -CREATE TABLE IF NOT EXISTS messages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - session_id TEXT NOT NULL REFERENCES sessions(id), - role TEXT NOT NULL, - content TEXT, - tool_call_id TEXT, - tool_calls TEXT, - tool_name TEXT, - effect_disposition TEXT, - timestamp REAL NOT NULL, - token_count INTEGER, - finish_reason TEXT, - reasoning TEXT, - reasoning_content TEXT, - reasoning_details TEXT, - codex_reasoning_items TEXT, - codex_message_items TEXT, - platform_message_id TEXT, - observed INTEGER DEFAULT 0, - active INTEGER NOT NULL DEFAULT 1, - compacted INTEGER NOT NULL DEFAULT 0, - api_content TEXT, - display_kind TEXT, - display_metadata TEXT -); - -CREATE TABLE IF NOT EXISTS session_model_usage ( - session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, - model TEXT NOT NULL, - billing_provider TEXT NOT NULL DEFAULT '', - billing_base_url TEXT NOT NULL DEFAULT '', - billing_mode TEXT NOT NULL DEFAULT '', - task TEXT NOT NULL DEFAULT '', - api_call_count INTEGER NOT NULL DEFAULT 0, - input_tokens INTEGER NOT NULL DEFAULT 0, - output_tokens INTEGER NOT NULL DEFAULT 0, - cache_read_tokens INTEGER NOT NULL DEFAULT 0, - cache_write_tokens INTEGER NOT NULL DEFAULT 0, - reasoning_tokens INTEGER NOT NULL DEFAULT 0, - estimated_cost_usd REAL NOT NULL DEFAULT 0, - actual_cost_usd REAL NOT NULL DEFAULT 0, - cost_status TEXT, - cost_source TEXT, - first_seen REAL, - last_seen REAL, - PRIMARY KEY (session_id, model, billing_provider, billing_base_url, billing_mode, task) -); - -CREATE TABLE IF NOT EXISTS state_meta ( - key TEXT PRIMARY KEY, - value TEXT -); - -CREATE TABLE IF NOT EXISTS gateway_routing ( - scope TEXT NOT NULL DEFAULT '', - session_key TEXT NOT NULL, - entry_json TEXT NOT NULL, - updated_at REAL NOT NULL, - PRIMARY KEY (scope, session_key) -); - -CREATE TABLE IF NOT EXISTS compression_locks ( - session_id TEXT PRIMARY KEY, - holder TEXT NOT NULL, - acquired_at REAL NOT NULL, - expires_at REAL NOT NULL -); - -CREATE TABLE IF NOT EXISTS async_delegations ( - delegation_id TEXT PRIMARY KEY, - origin_session TEXT NOT NULL, - origin_ui_session_id TEXT NOT NULL DEFAULT '', - parent_session_id TEXT, - state TEXT NOT NULL, - dispatched_at REAL NOT NULL, - completed_at REAL, - updated_at REAL NOT NULL, - event_json TEXT, - result_json TEXT, - delivery_state TEXT NOT NULL DEFAULT 'pending', - delivery_attempts INTEGER NOT NULL DEFAULT 0, - delivered_at REAL, - owner_pid INTEGER, - owner_started_at INTEGER, - task_json TEXT, - delivery_claim TEXT, - delivery_claimed_at REAL -); - -CREATE INDEX IF NOT EXISTS idx_sessions_source ON sessions(source); -CREATE INDEX IF NOT EXISTS idx_sessions_source_id ON sessions(source, id); -CREATE INDEX IF NOT EXISTS idx_sessions_parent ON sessions(parent_session_id); -CREATE INDEX IF NOT EXISTS idx_sessions_started ON sessions(started_at DESC); -CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_id, timestamp); -CREATE INDEX IF NOT EXISTS idx_compression_locks_expires ON compression_locks(expires_at); -CREATE INDEX IF NOT EXISTS idx_session_model_usage_session ON session_model_usage(session_id); -CREATE INDEX IF NOT EXISTS idx_session_model_usage_model ON session_model_usage(model); -CREATE INDEX IF NOT EXISTS idx_async_delegations_delivery - ON async_delegations(delivery_state, completed_at); -""" - -# Indexes that reference columns added in later schema versions must be -# created AFTER _reconcile_columns() has had a chance to ADD them on -# existing databases. SCHEMA_SQL above is run by sqlite executescript -# which would otherwise fail on legacy DBs ("no such column: active"). -DEFERRED_INDEX_SQL = """ -CREATE INDEX IF NOT EXISTS idx_messages_session_active - ON messages(session_id, active, timestamp); -CREATE INDEX IF NOT EXISTS idx_messages_active_null - ON messages(active) WHERE active IS NULL; -CREATE INDEX IF NOT EXISTS idx_sessions_session_key - ON sessions(session_key, started_at DESC); -CREATE INDEX IF NOT EXISTS idx_sessions_gateway_peer - ON sessions(source, user_id, chat_id, chat_type, thread_id, started_at DESC); -CREATE INDEX IF NOT EXISTS idx_sessions_handoff_state - ON sessions(handoff_state, started_at); -""" - -# ── Deferred FTS rebuild bookkeeping (schema v23) ── -# While a background index rebuild is pending, two state_meta keys define -# which message rows are currently IN the FTS indexes: -# -# fts_rebuild_high_water H — MAX(messages.id) at the moment the old -# indexes were dropped -# fts_rebuild_progress P — highest id the chunked backfill has indexed -# -# A row is indexed iff id <= P (backfilled) OR id > H (inserted after -# the drop; ids are AUTOINCREMENT so new rows are always > H and the insert -# triggers index them live). Rows in (P, H] are not yet indexed. -# -# Every trigger below gates on that same predicate: firing an FTS5 -# external-content 'delete' for a row that is NOT in the index corrupts the -# index, and skipping it for a row that IS indexed leaves a stale entry. -# When no rebuild is pending both keys are absent and COALESCE turns the -# predicate into a tautology (id > -1 OR id <= -1), i.e. normal operation. -# The two state_meta PK probes per write are negligible next to the FTS -# insert itself. -FTS_SQL = """ -CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( - content, - tool_name, - tool_calls, - content='messages', - content_rowid='id' -); - -CREATE TRIGGER IF NOT EXISTS messages_fts_insert AFTER INSERT ON messages -WHEN (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_high_water'), -1) - OR new.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_progress'), -1)) -BEGIN - INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) - VALUES (new.id, new.content, new.tool_name, new.tool_calls); -END; - -CREATE TRIGGER IF NOT EXISTS messages_fts_delete AFTER DELETE ON messages -WHEN (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_high_water'), -1) - OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_progress'), -1)) -BEGIN - INSERT INTO messages_fts(messages_fts, rowid, content, tool_name, tool_calls) - VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls); -END; - -CREATE TRIGGER IF NOT EXISTS messages_fts_update AFTER UPDATE ON messages -WHEN (old.content IS NOT new.content - OR old.tool_name IS NOT new.tool_name - OR old.tool_calls IS NOT new.tool_calls) - AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_high_water'), -1) - OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_progress'), -1)) -BEGIN - INSERT INTO messages_fts(messages_fts, rowid, content, tool_name, tool_calls) - VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls); - INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) - VALUES (new.id, new.content, new.tool_name, new.tool_calls); -END; -""" - -# Trigram FTS5 table for CJK substring search. The default unicode61 -# tokenizer splits CJK characters into individual tokens, breaking phrase -# matching. The trigram tokenizer creates overlapping 3-byte sequences so -# substring queries work natively for any script (CJK, Thai, etc.). -# -# The trigram index is the most expensive index in state.db (~2.6x the size -# of the text it covers), and ``role='tool'`` rows are ~90% of message bytes -# while being almost entirely machine noise (base64 payloads, file dumps, -# delegation transcripts). The index therefore reads through -# ``messages_fts_trigram_src``, a view that excludes tool rows — they stay -# fully stored in ``messages`` and fully searchable via the standard -# ``messages_fts`` index; they just don't get trigram (CJK substring) -# treatment. ``search_messages`` routes CJK queries that filter on -# ``role='tool'`` to the LIKE fallback for the same reason. -FTS_TRIGRAM_SQL = """ -CREATE VIEW IF NOT EXISTS messages_fts_trigram_src AS - SELECT id, role, content, tool_name, tool_calls - FROM messages - WHERE role <> 'tool'; - -CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5( - content, - tool_name, - tool_calls, - content='messages_fts_trigram_src', - content_rowid='id', - tokenize='trigram' -); - -CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_insert AFTER INSERT ON messages -WHEN new.role <> 'tool' - AND (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_high_water'), -1) - OR new.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_progress'), -1)) -BEGIN - INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) - VALUES (new.id, new.content, new.tool_name, new.tool_calls); -END; - -CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_delete AFTER DELETE ON messages -WHEN old.role <> 'tool' - AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_high_water'), -1) - OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_progress'), -1)) -BEGIN - INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content, tool_name, tool_calls) - VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls); -END; - -CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_update AFTER UPDATE ON messages -WHEN (old.content IS NOT new.content - OR old.tool_name IS NOT new.tool_name - OR old.tool_calls IS NOT new.tool_calls - OR old.role IS NOT new.role) - AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_high_water'), -1) - OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta - WHERE key = 'fts_rebuild_progress'), -1)) -BEGIN - INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content, tool_name, tool_calls) - SELECT 'delete', old.id, old.content, old.tool_name, old.tool_calls - WHERE old.role <> 'tool'; - INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) - SELECT new.id, new.content, new.tool_name, new.tool_calls - WHERE new.role <> 'tool'; -END; -""" - # ── CJK-bigram FTS index (replaces the trigram index when available) ──── # # The trigram tokenizer needs >=3 chars per query term, so 1-2 char CJK @@ -1634,19 +1250,6 @@ BEGIN END; """ -_FTS_CJK_TRIGGERS = ( - "messages_fts_cjk_insert", - "messages_fts_cjk_delete", - "messages_fts_cjk_update", -) - -# state_meta breadcrumb set when a tokenizer-less process had to drop the -# cjk triggers to keep message writes alive: rows written from that moment -# on are missing from the cjk index, so it must not serve reads until -# `hermes sessions optimize-storage` rebuilds it on a capable host. -FTS_CJK_STALE_KEY = "fts_cjk_stale" - - def fts5_cjk_so_path() -> Path: """Location of the cjk_unicode61 loadable extension.""" env = os.getenv("HERMES_FTS5_CJK_SO") @@ -1687,70 +1290,6 @@ def load_fts5_cjk_extension(conn: sqlite3.Connection) -> bool: return False - -# ── Legacy (v22 / inline-content) FTS DDL ────────────────────────────── -# Used ONLY to keep an existing pre-v23 install's search working and its -# triggers repairable UNTIL the user opts into `hermes db optimize`. This is -# the exact inline shape v11..v22 shipped: each virtual table stores its own -# copy of ``content || tool_name || tool_calls`` and the trigram table indexes -# every row (including role='tool'). We never CREATE these on a fresh install — -# fresh installs are born on the v23 external-content schema above. These -# constants exist so a legacy DB is never accidentally handed the v23 DDL -# (which would create the external-content trigram source VIEW and leave the -# DB in a mixed, broken state). `optimize_fts_storage()` is what migrates a -# legacy DB to the v23 shape. -LEGACY_FTS_SQL = """ -CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( - content -); - -CREATE TRIGGER IF NOT EXISTS messages_fts_insert AFTER INSERT ON messages BEGIN - INSERT INTO messages_fts(rowid, content) VALUES ( - new.id, - COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') - ); -END; - -CREATE TRIGGER IF NOT EXISTS messages_fts_delete AFTER DELETE ON messages BEGIN - DELETE FROM messages_fts WHERE rowid = old.id; -END; - -CREATE TRIGGER IF NOT EXISTS messages_fts_update AFTER UPDATE ON messages BEGIN - DELETE FROM messages_fts WHERE rowid = old.id; - INSERT INTO messages_fts(rowid, content) VALUES ( - new.id, - COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') - ); -END; -""" - -LEGACY_FTS_TRIGRAM_SQL = """ -CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5( - content, - tokenize='trigram' -); - -CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_insert AFTER INSERT ON messages BEGIN - INSERT INTO messages_fts_trigram(rowid, content) VALUES ( - new.id, - COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') - ); -END; - -CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_delete AFTER DELETE ON messages BEGIN - DELETE FROM messages_fts_trigram WHERE rowid = old.id; -END; - -CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_update AFTER UPDATE ON messages BEGIN - DELETE FROM messages_fts_trigram WHERE rowid = old.id; - INSERT INTO messages_fts_trigram(rowid, content) VALUES ( - new.id, - COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') - ); -END; -""" - - class CompressionSessionClosedError(RuntimeError): """A durable write targeted a parent already closed by compression.""" @@ -1954,7 +1493,7 @@ def quarantine_zeroed_state_db(path: Path) -> Optional[Path]: handle.close() -class SessionDB: +class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin): """ SQLite-backed session storage with FTS5 search. @@ -2335,17 +1874,6 @@ class SessionDB: exc, ) - def _sqlite_supports_fts5(self, cursor: sqlite3.Cursor) -> bool: - try: - cursor.execute("CREATE VIRTUAL TABLE temp._hermes_fts5_probe USING fts5(x)") - cursor.execute("DROP TABLE temp._hermes_fts5_probe") - return True - except sqlite3.OperationalError as exc: - if not self._is_fts5_unavailable_error(exc): - raise - self._warn_fts5_unavailable(exc) - return False - def _ensure_fts_cjk_schema(self, cursor) -> None: """Create / repair / self-heal the CJK-bigram index surface. @@ -2472,90 +2000,6 @@ class SessionDB: except sqlite3.OperationalError: pass - @staticmethod - def _fts_trigger_count(cursor: sqlite3.Cursor) -> int: - placeholders = ",".join("?" for _ in _FTS_TRIGGERS) - row = cursor.execute( - f"SELECT COUNT(*) FROM sqlite_master " - f"WHERE type = 'trigger' AND name IN ({placeholders})", - _FTS_TRIGGERS, - ).fetchone() - return int(row[0] if not isinstance(row, sqlite3.Row) else row[0]) - - @staticmethod - def _rebuild_fts_indexes( - cursor: sqlite3.Cursor, - *, - include_trigram: bool = True, - ) -> None: - # Both FTS tables are external-content (v23+): the special 'rebuild' - # command wipes the inverted index and repopulates it from the - # content source (messages for the standard index, the tool-row- - # excluding messages_fts_trigram_src view for the trigram index). - cursor.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')") - if include_trigram: - cursor.execute( - "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('rebuild')" - ) - # 'rebuild' indexes EVERY row, so any deferred-backfill markers are - # now satisfied — clear them, otherwise the background worker would - # re-insert rows the rebuild already covered (duplicate entries). - cursor.execute( - "DELETE FROM state_meta WHERE key IN " - "('fts_rebuild_high_water', 'fts_rebuild_progress')" - ) - - @staticmethod - def _rebuild_legacy_fts_indexes( - cursor: sqlite3.Cursor, - *, - include_trigram: bool = True, - ) -> None: - """Rebuild the LEGACY inline FTS indexes (pre-v23) from messages. - - Used only to repair a legacy DB whose triggers degraded under an - earlier no-FTS5 runtime. Inline tables have no external-content - 'rebuild' source, so we DELETE + reinsert the concatenated content - the legacy triggers produced. Never touches the v23 shape. - """ - cursor.execute("DELETE FROM messages_fts") - cursor.execute( - "INSERT INTO messages_fts(rowid, content) " - "SELECT id, " - "COALESCE(content, '') || ' ' || " - "COALESCE(tool_name, '') || ' ' || " - "COALESCE(tool_calls, '') " - "FROM messages" - ) - if not include_trigram: - return - cursor.execute("DELETE FROM messages_fts_trigram") - cursor.execute( - "INSERT INTO messages_fts_trigram(rowid, content) " - "SELECT id, " - "COALESCE(content, '') || ' ' || " - "COALESCE(tool_name, '') || ' ' || " - "COALESCE(tool_calls, '') " - "FROM messages" - ) - - def _fts_table_probe(self, cursor: sqlite3.Cursor, table_name: str) -> Optional[bool]: - try: - cursor.execute(f"SELECT * FROM {table_name} LIMIT 0") - return True - except sqlite3.OperationalError as exc: - if self._is_fts5_unavailable_error(exc): - # Only disable FTS entirely when the whole module is missing. - # A missing trigram tokenizer only affects trigram searches. - if self._is_trigram_unavailable_error(exc): - self._warn_trigram_unavailable(exc) - else: - self._warn_fts5_unavailable(exc) - return None - if "no such table" in str(exc).lower(): - return False - raise - def _ensure_fts_schema( self, cursor: sqlite3.Cursor, @@ -2748,20 +2192,6 @@ class SessionDB: except Exception as exc: logger.warning("WAL checkpoint (PASSIVE) failed: %s", exc) - def _try_incremental_merge_fts(self) -> None: - """Run one bounded FTS5 merge pass without failing the completed write.""" - if not self._fts_enabled: - return - try: - self._merge_fts_incrementally( - max_pages=self._FTS_MERGE_MAX_PAGES_PER_INDEX - ) - except sqlite3.Error as exc: - # Routine maintenance is best effort, but unexpected SQLite errors - # must remain visible instead of being silently mistaken for an - # optional missing index. - logger.warning("FTS incremental merge failed: %s", exc) - def close(self): """Close the database connection. @@ -2836,74 +2266,6 @@ class SessionDB: _FTS_REBUILD_DUTY_FACTOR = 4.0 # sleep >= 4x chunk cost (≤20% duty) _FTS_REBUILD_MIN_PAUSE = 0.2 # seconds — floor between chunks - def fts_rebuild_status(self) -> Optional[Dict[str, Any]]: - """Return deferred-rebuild progress, or None when no rebuild pending. - - Shape: {"pending": True, "total": , - "indexed": , "percent": <0-100 int>}. - Consumed by search_messages() notes and by status surfaces - (dashboard/desktop can poll this to render a progress indicator). - - Reads state_meta directly via _read_ctx instead of calling - get_meta() (which takes self._lock) so search_messages doesn't - block on the writer lock when checking rebuild status. - """ - with self._read_ctx() as conn: - row = conn.execute( - "SELECT key, value FROM state_meta WHERE key IN (?, ?)", - ("fts_rebuild_high_water", "fts_rebuild_progress"), - ).fetchall() - meta = {r["key"]: r["value"] for r in row} - high_water = meta.get("fts_rebuild_high_water") - if high_water is None: - return None - progress = int(meta.get("fts_rebuild_progress") or 0) - total = int(high_water) - if total <= 0: - return None - pct = min(100, int(100 * progress / total)) - return {"pending": True, "total": total, "indexed": progress, "percent": pct} - - def _fts_rebuild_finish(self) -> None: - """Finalize the deferred rebuild: boundary sweep + clear markers. - - The sweep is cheap insurance against any write that slipped through - the migration-boundary instant (between high_water capture and - trigger activation): re-index any row near the boundary that the - index is missing. docsize has one row per indexed doc, so the - anti-join is exact and runs on a narrow id range. - """ - def _do(conn): - hw_row = conn.execute( - "SELECT value FROM state_meta WHERE key = 'fts_rebuild_high_water'" - ).fetchone() - if hw_row is not None: - hw = int(hw_row[0]) - # Sweep a generous window around the boundary. - lo, hi = hw - 1000, hw + 1000 - conn.execute( - "INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) " - "SELECT m.id, m.content, m.tool_name, m.tool_calls " - "FROM messages m " - "WHERE m.id > ? AND m.id <= ? " - "AND NOT EXISTS (SELECT 1 FROM messages_fts_docsize d WHERE d.id = m.id)", - (lo, hi), - ) - conn.execute( - "INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) " - "SELECT m.id, m.content, m.tool_name, m.tool_calls " - "FROM messages m " - "WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' " - "AND NOT EXISTS (SELECT 1 FROM messages_fts_trigram_docsize d WHERE d.id = m.id)", - (lo, hi), - ) - conn.execute( - "DELETE FROM state_meta WHERE key IN " - "('fts_rebuild_high_water', 'fts_rebuild_progress')" - ) - self._execute_write(_do) - logger.info("Deferred FTS rebuild complete — all messages indexed.") - # Demoted v22 FTS shadow tables awaiting teardown (see the v23 migration: # DROP of a multi-GB FTS vtable blocks for minutes, so the migration # demotes the vtable definitions out of sqlite_master and renames the @@ -2911,116 +2273,6 @@ class SessionDB: # worker empties them in bounded chunks, then drops them cheaply). _FTS_TRASH_PREFIX = "fts_v22_trash_" - def _fts_teardown_trash_step(self) -> bool: - """Tear down one chunk of a demoted v22 FTS shadow table. - - The trash tables are PLAIN tables (their vtable parent was demoted - away during the migration), so chunked DELETE + final DROP involve - no FTS5 machinery at all. Returns True while teardown work remains. - """ - with self._lock: - trash = [ - r[0] for r in self._conn.execute( - "SELECT name FROM sqlite_master WHERE type = 'table' " - "AND name LIKE ? ESCAPE '\\'", - (self._FTS_TRASH_PREFIX.replace("_", "\\_") + "%",), - ).fetchall() - ] - if not trash: - return False - - tbl = trash[0] - - def _do(conn): - pk_cols = [ - r[1] for r in conn.execute(f"PRAGMA table_info({tbl})") - if r[5] > 0 - ] - key = ", ".join(pk_cols) if pk_cols else "rowid" - cur = conn.execute( - f"DELETE FROM {tbl} WHERE ({key}) IN " - f"(SELECT {key} FROM {tbl} LIMIT {self._FTS_REBUILD_CHUNK_ROWS})" - ) - if cur.rowcount == 0: - # Empty — the DROP is cheap now. - conn.execute(f"DROP TABLE IF EXISTS {tbl}") - logger.info("Old FTS shadow table %s torn down.", tbl) - return True # re-check: more trash tables / chunks may remain - - try: - return bool(self._execute_write(_do)) - except sqlite3.OperationalError as exc: - logger.debug("FTS trash teardown chunk failed (will retry): %s", exc) - return True - - def fts_rebuild_step(self) -> bool: - """Backfill one chunk of the deferred FTS rebuild. - - Returns True when more work remains, False when the rebuild is - complete (or none is pending). Safe to call from any process at any - time; chunks are claimed atomically inside the write transaction, so - concurrent callers interleave instead of duplicating rows. - """ - if not self._fts_enabled: - return False - high_water_raw = self.get_meta("fts_rebuild_high_water") - if high_water_raw is None: - return False - high_water = int(high_water_raw) - include_trigram = self._trigram_available - chunk = self._FTS_REBUILD_CHUNK_ROWS - - def _do(conn): - # Re-read progress inside the write transaction (BEGIN IMMEDIATE - # is already held by _execute_write) — this is the claim: two - # workers can't read the same progress value concurrently. - row = conn.execute( - "SELECT value FROM state_meta WHERE key = 'fts_rebuild_progress'" - ).fetchone() - if row is None: - return False # finished (or cleared) by another process - progress = int(row[0]) - if progress >= high_water: - return False - - # The chunk upper bound is an id, not a row count, so gaps from - # deleted rows don't shrink chunks below the claimed range. - upper = min(progress + chunk, high_water) - conn.execute( - "INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) " - "SELECT id, content, tool_name, tool_calls FROM messages " - "WHERE id > ? AND id <= ?", - (progress, upper), - ) - if include_trigram: - conn.execute( - "INSERT INTO messages_fts_trigram" - "(rowid, content, tool_name, tool_calls) " - "SELECT id, content, tool_name, tool_calls FROM messages " - "WHERE id > ? AND id <= ? AND role <> 'tool'", - (progress, upper), - ) - # Publish progress in the same transaction as the rows it - # covers — crash-atomic: either both land or neither does. - conn.execute( - "UPDATE state_meta SET value = ? " - "WHERE key = 'fts_rebuild_progress'", - (str(upper),), - ) - return upper < high_water - - try: - more = self._execute_write(_do) - except sqlite3.OperationalError as exc: - logger.debug("FTS rebuild chunk failed (will retry): %s", exc) - return True # transient (lock contention) — caller retries - if more is False: - status = self.fts_rebuild_status() - if status is not None and status["indexed"] >= status["total"]: - self._fts_rebuild_finish() - return False - return bool(more) - # ── CJK-bigram index backfill (dedicated marker pair) ── # # Same chunk engine as the main deferred rebuild, but on the @@ -3028,135 +2280,6 @@ class SessionDB: # an already-optimized v23 DB gaining the cjk index) never gates the # complete ``messages_fts`` / trigram triggers. - def fts_cjk_rebuild_status(self) -> Optional[Dict[str, Any]]: - """CJK-index backfill progress, or None when none is pending.""" - with self._read_ctx() as conn: - row = conn.execute( - "SELECT key, value FROM state_meta WHERE key IN (?, ?)", - ("fts_cjk_rebuild_high_water", "fts_cjk_rebuild_progress"), - ).fetchall() - meta = {r["key"]: r["value"] for r in row} - high_water = meta.get("fts_cjk_rebuild_high_water") - if high_water is None: - return None - progress = int(meta.get("fts_cjk_rebuild_progress") or 0) - total = int(high_water) - if total <= 0: - return None - pct = min(100, int(100 * progress / total)) - return {"pending": True, "total": total, "indexed": progress, "percent": pct} - - def fts_cjk_rebuild_step(self) -> bool: - """Backfill one chunk of the CJK index. True while work remains.""" - if not self._fts_enabled or not self._fts_cjk_loaded: - return False - high_water_raw = self.get_meta("fts_cjk_rebuild_high_water") - if high_water_raw is None: - return False - high_water = int(high_water_raw) - chunk = self._FTS_REBUILD_CHUNK_ROWS - - def _do(conn): - row = conn.execute( - "SELECT value FROM state_meta " - "WHERE key = 'fts_cjk_rebuild_progress'" - ).fetchone() - if row is None: - return False # finished (or cleared) by another process - progress = int(row[0]) - if progress >= high_water: - return False - upper = min(progress + chunk, high_water) - conn.execute( - "INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) " - "SELECT id, content, tool_name, tool_calls FROM messages " - "WHERE id > ? AND id <= ? AND role <> 'tool'", - (progress, upper), - ) - conn.execute( - "UPDATE state_meta SET value = ? " - "WHERE key = 'fts_cjk_rebuild_progress'", - (str(upper),), - ) - return upper < high_water - - try: - more = self._execute_write(_do) - except sqlite3.OperationalError as exc: - logger.debug("CJK FTS rebuild chunk failed (will retry): %s", exc) - return True - if more is False: - status = self.fts_cjk_rebuild_status() - if status is not None and status["indexed"] >= status["total"]: - self._fts_cjk_rebuild_finish() - return False - return bool(more) - - def _fts_cjk_rebuild_finish(self) -> None: - """Boundary sweep + clear the cjk markers; index becomes servable.""" - def _do(conn): - hw_row = conn.execute( - "SELECT value FROM state_meta " - "WHERE key = 'fts_cjk_rebuild_high_water'" - ).fetchone() - if hw_row is not None: - hw = int(hw_row[0]) - lo, hi = hw - 1000, hw + 1000 - conn.execute( - "INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) " - "SELECT m.id, m.content, m.tool_name, m.tool_calls " - "FROM messages m " - "WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' " - "AND NOT EXISTS (SELECT 1 FROM messages_fts_cjk_docsize d WHERE d.id = m.id)", - (lo, hi), - ) - conn.execute( - "DELETE FROM state_meta WHERE key IN " - "('fts_cjk_rebuild_high_water', 'fts_cjk_rebuild_progress')" - ) - self._execute_write(_do) - self._fts_cjk_available = True - logger.info("CJK FTS index backfill complete — serving CJK search.") - - def _fts_cjk_reset_if_stale(self) -> None: - """Rebuild path for a stale cjk index (triggers were dropped). - - The gap's extent is unknown, so the only safe recovery is a from- - scratch rebuild: drop the table + triggers, clear the breadcrumb, - recreate via ``_ensure_fts_cjk_schema`` (which sets fresh backfill - markers on a populated DB). Called from ``optimize_fts_storage`` on - a tokenizer-capable host; no-op when not stale. - """ - if not self._fts_cjk_loaded: - return - - def _do(conn): - stale = conn.execute( - "SELECT 1 FROM state_meta WHERE key = ?", - (FTS_CJK_STALE_KEY,), - ).fetchone() - if not stale: - return False - for trig in _FTS_CJK_TRIGGERS: - conn.execute(f"DROP TRIGGER IF EXISTS {trig}") - conn.execute("DROP TABLE IF EXISTS messages_fts_cjk") - conn.execute("DROP VIEW IF EXISTS messages_fts_cjk_src") - conn.execute( - "DELETE FROM state_meta WHERE key IN " - f"('{FTS_CJK_STALE_KEY}', 'fts_cjk_rebuild_high_water', " - "'fts_cjk_rebuild_progress')" - ) - return True - was_stale = self._execute_write(_do) - if was_stale: - # Recreate outside the write transaction — _ensure_fts_cjk_schema - # uses executescript(), which implicitly commits any pending - # transaction and must not run inside _execute_write's BEGIN - # IMMEDIATE. Sets fresh backfill markers on a populated DB. - with self._lock: - self._ensure_fts_cjk_schema(self._conn) - self._conn.commit() - # ── Opt-in v23 FTS storage optimization (`hermes sessions optimize-storage`) ── # # This is the ONLY path that migrates an existing legacy (v22 inline) DB @@ -3167,40 +2290,6 @@ class SessionDB: # teardown — with progress callbacks, a disk preflight in the CLI # wrapper, a VACUUM at the end, and a defensive schema_version bump. - def fts_optimize_available(self) -> bool: - """True when `optimize_fts_storage()` has work to do: either this DB - is a legacy inline-FTS install that can be optimized to the v23 - external-content schema, or a previous optimize run was interrupted - (legacy vtables already demoted, but backfill markers and/or trash - tables remain) and re-running would resume it, or the CJK-bigram - index needs a backfill/rebuild on this tokenizer-capable host. - False for fresh and fully-optimized installs (and when FTS5 is - unavailable).""" - if not self._fts_enabled or self.read_only: - return False - with self._lock: - if self._db_has_legacy_inline_fts(self._conn): - return True - # Interrupted optimize: demotion already removed the legacy - # vtables (so the check above is False), but the transition is - # unfinished until the backfill markers are cleared and the - # demoted trash tables are torn down. Search stays complete - # through the gap supplement meanwhile; re-running resumes. - if self._conn.execute( - "SELECT 1 FROM state_meta " - "WHERE key = 'fts_rebuild_high_water' LIMIT 1" - ).fetchone(): - return True - # CJK-bigram index work — only offerable when THIS process can - # tokenize: a pending backfill (markers set at creation on a - # populated DB) or a stale index awaiting a from-scratch rebuild. - if self._fts_cjk_loaded and self._conn.execute( - "SELECT 1 FROM state_meta WHERE key IN " - f"('fts_cjk_rebuild_high_water', '{FTS_CJK_STALE_KEY}') LIMIT 1" - ).fetchone(): - return True - return self._has_fts_trash(self._conn) - def _has_fts_trash(self, conn) -> bool: """True when demoted v22 shadow tables are still awaiting teardown. Caller must hold ``self._lock`` (or pass a migration-time cursor).""" @@ -3210,804 +2299,6 @@ class SessionDB: (self._FTS_TRASH_PREFIX.replace("_", "\\_") + "%",), ).fetchone()) - def _demote_legacy_fts_to_trash(self) -> int: - """Demote the legacy inline FTS vtables and stage their shadow tables - for chunked teardown. Returns MAX(messages.id) as the rebuild high - water. O(1) schema surgery — the heavy delete is deferred to the - chunked teardown, exactly as the validated auto path did.""" - def _do(conn): - self._drop_fts_triggers(conn) - conn.execute("DROP VIEW IF EXISTS messages_fts_trigram_src") - had = bool(conn.execute( - "SELECT 1 FROM sqlite_master WHERE type = 'table' " - "AND name IN ('messages_fts', 'messages_fts_trigram') " - "AND sql LIKE 'CREATE VIRTUAL TABLE%' LIMIT 1" - ).fetchone()) - if had: - conn.execute("PRAGMA writable_schema=ON") - conn.execute( - "DELETE FROM sqlite_master WHERE type = 'table' " - "AND name IN ('messages_fts', 'messages_fts_trigram') " - "AND sql LIKE 'CREATE VIRTUAL TABLE%'" - ) - conn.execute("PRAGMA writable_schema=RESET") - shadows = [ - r[0] for r in conn.execute( - "SELECT name FROM sqlite_master WHERE type = 'table' " - "AND (name LIKE 'messages_fts_%' ESCAPE '\\' " - "OR name LIKE 'messages_fts_trigram_%' ESCAPE '\\')" - ).fetchall() - ] - for sh in shadows: - conn.execute(f"ALTER TABLE {sh} RENAME TO fts_v22_trash_{sh}") - # Create the new v23 empty schema + set the backfill markers. - self._ensure_fts_schema(conn, "messages_fts", FTS_SQL) - self._ensure_fts_schema(conn, "messages_fts_trigram", FTS_TRIGRAM_SQL) - hw = conn.execute("SELECT COALESCE(MAX(id), 0) FROM messages").fetchone()[0] - for k, v in ( - ("fts_rebuild_high_water", str(hw)), - ("fts_rebuild_progress", "0"), - ): - conn.execute( - "INSERT INTO state_meta (key, value) VALUES (?, ?) " - "ON CONFLICT(key) DO UPDATE SET value = excluded.value", - (k, v), - ) - conn.execute("DELETE FROM state_meta WHERE key = 'fts_optimize_available'") - return hw - return int(self._execute_write(_do)) - - def optimize_fts_storage( - self, - *, - progress_cb: Optional[Callable[[Dict[str, Any]], None]] = None, - vacuum: bool = True, - ) -> Dict[str, Any]: - """Migrate a legacy v22 inline-FTS DB to the v23 external-content - schema, foreground and to completion. Safe to re-run: if a previous - attempt was interrupted it resumes from the progress marker. - - ``progress_cb`` receives {"phase", "percent", "indexed", "total"} - dicts for a CLI progress bar. Returns a summary dict. - - The trigram tokenizer being unavailable is not fatal — the base index - is still rebuilt (CJK falls back to LIKE), mirroring normal startup. - """ - if not self._fts_enabled: - return {"ok": False, "reason": "fts5_unavailable"} - if self.read_only: - return {"ok": False, "reason": "read_only"} - - # Only demote if we're actually still on the legacy shape. If a prior - # run already demoted (markers/trash present), skip straight to - # finishing the backfill + teardown — this is what makes re-running - # after an interruption safe. - with self._lock: - legacy = self._db_has_legacy_inline_fts(self._conn) - pending = self.get_meta("fts_rebuild_high_water") is not None - if legacy and not pending: - self._demote_legacy_fts_to_trash() - - # A stale CJK index (triggers dropped by a tokenizer-less process) - # can only be recovered from scratch — reset it now so the cjk - # backfill phase below rebuilds it. No-op without the tokenizer. - self._fts_cjk_reset_if_stale() - # An optimized v23 DB gaining the cjk index for the first time (no - # legacy work left, tokenizer newly installed): ensure the table + - # markers exist so the backfill phase has work to claim. - if self._fts_cjk_loaded: - with self._lock: - self._ensure_fts_cjk_schema(self._conn) - self._conn.commit() - - def _emit(phase: str) -> None: - if progress_cb is None: - return - st = self.fts_rebuild_status() - if st is None: - st = self.fts_cjk_rebuild_status() - progress_cb({ - "phase": phase, - "percent": st["percent"] if st else 100, - "indexed": st["indexed"] if st else 0, - "total": st["total"] if st else 0, - }) - - def _pause(chunk_seconds: float) -> None: - """Inter-chunk throttle (see the chunk-engine note above). - - The chunk methods themselves never sleep, so this loop is the - single place the duty cycle is enforced: without it, back-to-back - BEGIN IMMEDIATE chunks starve any live gateway/CLI process - sharing the DB out of its lock retries (the measured ~85% - write-lock ownership that froze concurrent sessions). - """ - time.sleep(max( - self._FTS_REBUILD_MIN_PAUSE, - chunk_seconds * self._FTS_REBUILD_DUTY_FACTOR, - )) - - # Phase 1: backfill (foreground, throttled between chunks so a live - # gateway sharing the DB stays responsive). - _emit("backfill") - while True: - _t0 = time.monotonic() - if not self.fts_rebuild_step(): - break - _emit("backfill") - _pause(time.monotonic() - _t0) - _emit("backfill") - - # Phase 1b: backfill the CJK-bigram index (its own marker pair; a - # no-op when the tokenizer isn't loadable or nothing is pending). - while True: - _t0 = time.monotonic() - if not self.fts_cjk_rebuild_step(): - break - _emit("backfill") - _pause(time.monotonic() - _t0) - - # Phase 2: tear down the demoted legacy shadow tables in chunks. - _emit("teardown") - while True: - _t0 = time.monotonic() - if not self._fts_teardown_trash_step(): - break - _emit("teardown") - _pause(time.monotonic() - _t0) - - # Phase 3: reclaim freed pages to the OS. - vacuum_ok = None - if vacuum: - _emit("vacuum") - try: - with self._lock: - self._conn.execute("VACUUM") - vacuum_ok = True - except sqlite3.OperationalError as exc: - # Most common cause: not enough free disk for VACUUM's temp - # copy. The optimization still succeeded; space just isn't - # reclaimed until a later VACUUM. Non-fatal. - logger.warning("VACUUM after FTS optimize failed: %s", exc) - vacuum_ok = False - # Best-effort: fold the WAL back into the main file so the on-disk - # size settles now rather than at close(). NOTE this is REFUSED - # (SQLITE_BUSY) while any other connection holds a WAL read-mark — - # e.g. a live gateway sharing the DB — so it is not sufficient on - # its own. Callers must therefore NOT size the result by stat()ing - # the file; use :meth:`logical_size_bytes`, which is truthful - # immediately regardless of readers. - try: - with self._lock: - self._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") - except Exception as exc: - logger.debug( - "WAL checkpoint (TRUNCATE) after optimize VACUUM failed: %s", - exc, - ) - - # Phase 4: stamp the FTS storage layout as current, clear the "available" - # flag, and advance schema_version if it was somehow still behind (the - # main version normally advances on open now, but bump defensively so a - # DB opened only by pre-decoupling code still settles). The FTS-layout - # marker is the source of truth for "is this DB optimized". - def _settle(conn): - conn.execute( - "INSERT INTO state_meta (key, value) VALUES ('fts_storage_version', ?) " - "ON CONFLICT(key) DO UPDATE SET value = excluded.value", - (str(FTS_STORAGE_VERSION),), - ) - conn.execute("DELETE FROM state_meta WHERE key = 'fts_optimize_available'") - conn.execute( - "UPDATE schema_version SET version = ? WHERE version < ?", - (SCHEMA_VERSION, SCHEMA_VERSION), - ) - self._execute_write(_settle) - _emit("done") - logger.info( - "FTS storage optimization complete (layout v%d).", FTS_STORAGE_VERSION - ) - return {"ok": True, "vacuumed": vacuum_ok} - - @staticmethod - def _parse_schema_columns(schema_sql: str) -> Dict[str, Dict[str, str]]: - """Extract expected columns per table from SCHEMA_SQL. - - Uses an in-memory SQLite database to parse the SQL — SQLite itself - handles all syntax (DEFAULT expressions with commas, inline - REFERENCES, CHECK constraints, etc.) so there are zero regex - edge cases. The in-memory DB is opened, the schema DDL is - executed, and PRAGMA table_info extracts the column metadata. - - Adding a column to SCHEMA_SQL is all that's needed; the - reconciliation loop picks it up automatically. - """ - ref = sqlite3.connect(":memory:") - try: - ref.executescript(schema_sql) - table_columns: Dict[str, Dict[str, str]] = {} - for (tbl,) in ref.execute( - "SELECT name FROM sqlite_master " - "WHERE type='table' AND name NOT LIKE 'sqlite_%'" - ).fetchall(): - cols: Dict[str, str] = {} - for row in ref.execute( - f'PRAGMA table_info("{tbl}")' - ).fetchall(): - # row: (cid, name, type, notnull, dflt_value, pk) - col_name = row[1] - col_type = row[2] or "" - notnull = row[3] - default = row[4] - pk = row[5] - # Reconstruct the type expression for ALTER TABLE ADD COLUMN - parts = [col_type] if col_type else [] - if notnull and not pk: - parts.append("NOT NULL") - if default is not None: - parts.append(f"DEFAULT {default}") - cols[col_name] = " ".join(parts) - table_columns[tbl] = cols - return table_columns - finally: - ref.close() - - def _reconcile_columns(self, cursor: sqlite3.Cursor) -> None: - """Ensure live tables have every column declared in SCHEMA_SQL. - - Follows the Beets/sqlite-utils pattern: the CREATE TABLE definition - in SCHEMA_SQL is the single source of truth for the desired schema. - On every startup this method diffs the live columns (via PRAGMA - table_info) against the declared columns, and ADDs any that are - missing. - - This makes column additions a declarative operation — just add - the column to SCHEMA_SQL and it appears on the next startup. - Version-gated migration blocks are no longer needed for ADD COLUMN. - """ - expected = self._parse_schema_columns(SCHEMA_SQL) - for table_name, declared_cols in expected.items(): - # Get current columns from the live table - try: - rows = cursor.execute( - f'PRAGMA table_info("{table_name}")' - ).fetchall() - except sqlite3.OperationalError: - continue # Table doesn't exist yet (shouldn't happen after executescript) - live_cols = set() - for row in rows: - # PRAGMA table_info returns (cid, name, type, notnull, dflt_value, pk) - name = row[1] if isinstance(row, (tuple, list)) else row["name"] - live_cols.add(name) - - for col_name, col_type in declared_cols.items(): - if col_name not in live_cols: - safe_name = col_name.replace('"', '""') - try: - cursor.execute( - f'ALTER TABLE "{table_name}" ADD COLUMN "{safe_name}" {col_type}' - ) - except sqlite3.OperationalError as exc: - # Expected: "duplicate column name" from a race or - # re-run. Unexpected: "Cannot add a NOT NULL column - # with default value NULL" from a schema mistake. - # Log at DEBUG so it's visible in agent.log. - logger.debug( - "reconcile %s.%s: %s", table_name, col_name, exc, - ) - - def _heal_gateway_routing_pk(self, cursor: sqlite3.Cursor) -> None: - """Rebuild ``gateway_routing`` when its PRIMARY KEY predates scoping. - - Early builds of the routing-index migration (#59203) created the - table with ``session_key TEXT PRIMARY KEY`` and no ``scope`` column. - ``_reconcile_columns()`` ADDs the missing ``scope`` column on those - databases, but SQLite cannot ALTER a primary key, so the shipped - composite ``PRIMARY KEY (scope, session_key)`` never lands. On such - tables every write path is broken: - - * ``save_gateway_routing_entry`` fails with "ON CONFLICT clause does - not match any PRIMARY KEY or UNIQUE constraint" (its upsert targets - the composite key), and - * ``replace_gateway_routing_entries`` fails with "UNIQUE constraint - failed: gateway_routing.session_key" whenever the same session_key - exists under a different scope — the exact isolation the composite - key exists to provide. - - Each failed save logs a warning and falls back to sessions.json, - so a legacy-shaped table produces endless per-save warning spam. - Rebuild it once, preserving rows. On a session_key collision across - scopes (possible while the PK was wrong) the newest row wins. - """ - try: - rows = cursor.execute( - 'PRAGMA table_info("gateway_routing")' - ).fetchall() - except sqlite3.OperationalError: - return - if not rows: - return - - def _col(row, idx, name): - return row[idx] if isinstance(row, (tuple, list)) else row[name] - - pk_cols = [ - _col(r, 1, "name") - for r in sorted( - (r for r in rows if _col(r, 5, "pk")), - key=lambda r: _col(r, 5, "pk"), - ) - ] - if pk_cols == ["scope", "session_key"]: - return - - logger.info( - "gateway_routing has legacy primary key %r; rebuilding with " - "composite (scope, session_key) key", - pk_cols, - ) - cursor.execute( - "ALTER TABLE gateway_routing RENAME TO gateway_routing_legacy_pk" - ) - cursor.execute( - """CREATE TABLE gateway_routing ( - scope TEXT NOT NULL DEFAULT '', - session_key TEXT NOT NULL, - entry_json TEXT NOT NULL, - updated_at REAL NOT NULL, - PRIMARY KEY (scope, session_key) -)""" - ) - # INSERT OR REPLACE + updated_at ordering: if the broken PK ever let - # two scopes race over one session_key, keep the newest row per - # (scope, session_key) pair. - cursor.execute( - "INSERT OR REPLACE INTO gateway_routing " - "(scope, session_key, entry_json, updated_at) " - "SELECT COALESCE(scope, ''), session_key, entry_json, updated_at " - "FROM gateway_routing_legacy_pk ORDER BY updated_at ASC" - ) - cursor.execute("DROP TABLE gateway_routing_legacy_pk") - - def _init_schema(self): - """Create tables and FTS if they don't exist, reconcile columns. - - Schema management follows the declarative reconciliation pattern - (Beets, sqlite-utils): SCHEMA_SQL is the single source of truth. - On existing databases, _reconcile_columns() diffs live columns - against SCHEMA_SQL and ADDs any missing ones. This eliminates - the version-gated migration chain for column additions, making - it impossible for reordered or inserted migrations to skip columns. - - The schema_version table is retained for future data migrations - (transforming existing rows) which cannot be handled declaratively. - """ - cursor = self._conn.cursor() - - cursor.executescript(SCHEMA_SQL) - - # ── Declarative column reconciliation ────────────────────────── - # Diff live tables against SCHEMA_SQL and ADD any missing columns. - # This is idempotent and self-healing: even if a version-gated - # migration was skipped (e.g. due to version renumbering), the - # column gets created here. - self._reconcile_columns(cursor) - - # Rebuild gateway_routing if it still carries the pre-scope PRIMARY - # KEY (session_key alone). ADD COLUMN cannot fix a PK, so this is - # the one table-shape repair reconciliation can't express. - self._heal_gateway_routing_pk(cursor) - - # Indexes that reference reconciler-added columns must be created - # AFTER _reconcile_columns runs — declaring them in SCHEMA_SQL - # makes the initial executescript fail on legacy DBs (the index's - # WHERE clause references a column that doesn't exist yet). - try: - cursor.execute( - "CREATE INDEX IF NOT EXISTS idx_messages_platform_msg_id " - "ON messages(session_id, platform_message_id) " - "WHERE platform_message_id IS NOT NULL" - ) - except sqlite3.OperationalError as exc: - logger.debug("idx_messages_platform_msg_id create skipped: %s", exc) - - # Deferred indexes that reference the reconciler-added ``active`` - # column (idx_messages_session_active) — same ordering constraint. - cursor.executescript(DEFERRED_INDEX_SQL) - - # Heal NULL ``active`` rows unconditionally on every startup. - # On real-world DBs the reconciler-added ``active`` column can lack - # its NOT NULL DEFAULT 1 (older reconciler builds reconstructed the - # type without the default — see #51646: PRAGMA shows - # (17,'active','INTEGER',0,None,0) in the wild), so INSERTs that - # omitted the column wrote NULL and the ``WHERE active = 1`` - # transcript loaders hid the whole history. The INSERTs now set - # active=1 explicitly; this idempotent repair un-hides rows written - # before the fix. It was previously gated at ``current_version < - # 12`` which never re-ran for already-v12+ databases. - try: - cursor.execute( - "UPDATE messages SET active = 1 WHERE active IS NULL" - ) - except sqlite3.OperationalError: - pass - - fts5_available = self._sqlite_supports_fts5(cursor) - fts_migrations_complete = True - if not fts5_available: - # Existing FTS triggers can still fire on messages INSERT/UPDATE - # even though the current sqlite runtime cannot read the virtual - # tables they target. Drop only the triggers so core persistence - # continues; if a future runtime has FTS5, _ensure_fts_schema() - # recreates them. - self._drop_fts_triggers(cursor) - - # ── Schema version bookkeeping ───────────────────────────────── - # Bump to current so future data migrations (if any) can gate on - # version. No version-gated column additions remain. - cursor.execute("SELECT version FROM schema_version LIMIT 1") - row = cursor.fetchone() - if row is None: - cursor.execute( - "INSERT INTO schema_version (version) VALUES (?)", - (SCHEMA_VERSION,), - ) - else: - current_version = row["version"] if isinstance(row, sqlite3.Row) else row[0] - # Data migrations that can't be expressed declaratively (row - # backfills, index changes tied to a specific version step) stay - # in a version-gated chain. Column additions are handled by - # _reconcile_columns() above and no longer need entries here. - if current_version < 10 and SCHEMA_VERSION == 10: - # v10: trigram FTS5 table for CJK/substring search. The - # virtual table + triggers are created unconditionally via - # FTS_TRIGRAM_SQL below, but existing rows need a one-time - # backfill into the FTS index. - # - # Only run this when v10 itself is the target schema. Current - # v11+ code drops and rebuilds both FTS tables below, so doing - # the v10-only trigram backfill first only burns startup time - # and WAL space before v11 throws the work away. - if fts5_available: - _fts_trigram_exists = self._fts_table_probe( - cursor, "messages_fts_trigram" - ) - if _fts_trigram_exists is False: - if self._ensure_fts_schema( - cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL - ): - cursor.execute( - "INSERT INTO messages_fts_trigram(rowid, content) " - "SELECT id, content FROM messages WHERE content IS NOT NULL" - ) - else: - fts_migrations_complete = False - elif _fts_trigram_exists is None: - fts_migrations_complete = False - else: - fts_migrations_complete = False - if current_version < 11 and SCHEMA_VERSION < 23: - # v11 (SUPERSEDED by v23): re-index FTS5 tables to cover - # tool_name + tool_calls in inline mode (#16751). v23 drops - # and rebuilds both FTS tables in external-content form, so - # running the v11 inline backfill first would only burn - # startup time and WAL space before v23 throws the work - # away — and its inline INSERT shape no longer matches the - # current external-content FTS_SQL anyway. Kept only for - # source archaeology; unreachable while SCHEMA_VERSION >= 23. - pass - if current_version < 16: - # v16: tag delegate subagent rows so pickers stay clean after - # parent deletes that used to orphan them (parent_session_id → NULL). - try: - cursor.execute( - "UPDATE sessions SET model_config = json_set(" - "COALESCE(model_config, '{}'), '$._delegate_from', parent_session_id) " - f"WHERE parent_session_id IS NOT NULL " - "AND json_extract(COALESCE(model_config, '{}'), '$._delegate_from') IS NULL " - f"AND {_ephemeral_child_sql('sessions')}" - ) - cursor.execute( - "UPDATE sessions SET model_config = json_set(" - "COALESCE(model_config, '{}'), '$._delegate_from', '__orphaned__') " - "WHERE parent_session_id IS NULL " - "AND json_extract(COALESCE(model_config, '{}'), '$._delegate_from') IS NULL " - "AND json_extract(COALESCE(model_config, '{}'), '$._branched_from') IS NULL " - "AND title IS NULL " - "AND message_count <= 25 " - "AND EXISTS (SELECT 1 FROM messages m " - " WHERE m.session_id = sessions.id AND m.role = 'tool') " - "AND NOT EXISTS (SELECT 1 FROM sessions ch " - " WHERE ch.parent_session_id = sessions.id)" - ) - except sqlite3.OperationalError: - pass - if current_version < 18: - # v18: gateway metadata consolidation (#9006). Backfill - # display_name / origin_json / expiry_finalized from - # sessions.json so pre-migration gateway sessions are - # discoverable from state.db without the JSON index. - try: - self._backfill_gateway_metadata_from_sessions_json(cursor) - except Exception as exc: - # Backfill is best-effort: sessions.json may be absent, - # corrupted, or partially stale. Missing metadata simply - # means consumers fall back to sessions.json for those - # rows until the gateway rewrites them. - logger.debug("v18 gateway metadata backfill skipped: %s", exc) - if current_version < 20: - # v20: per-model usage attribution (issue #51607). Going - # forward update_token_counts() records each API call into - # session_model_usage keyed by the live model, but existing - # sessions only have their aggregate totals on the sessions - # row. Seed one usage row per historical session from those - # aggregates so insights reads uniformly from the new table. - # INSERT OR IGNORE keeps it idempotent: if newer code already - # wrote a (session_id, model, provider) row for a session, the - # PK conflict skips the stale aggregate rather than doubling it. - try: - cursor.execute( - """INSERT OR IGNORE INTO session_model_usage ( - session_id, model, billing_provider, - billing_base_url, billing_mode, - api_call_count, input_tokens, - output_tokens, cache_read_tokens, - cache_write_tokens, reasoning_tokens, - estimated_cost_usd, actual_cost_usd, - cost_status, cost_source, first_seen, last_seen - ) - SELECT id, COALESCE(model, 'unknown'), - COALESCE(billing_provider, ''), - COALESCE(billing_base_url, ''), - COALESCE(billing_mode, ''), - COALESCE(api_call_count, 0), - COALESCE(input_tokens, 0), - COALESCE(output_tokens, 0), - COALESCE(cache_read_tokens, 0), - COALESCE(cache_write_tokens, 0), - COALESCE(reasoning_tokens, 0), - COALESCE(estimated_cost_usd, 0), - COALESCE(actual_cost_usd, 0), - cost_status, cost_source, - started_at, COALESCE(ended_at, started_at) - FROM sessions - WHERE COALESCE(input_tokens, 0) - + COALESCE(output_tokens, 0) - + COALESCE(cache_read_tokens, 0) - + COALESCE(cache_write_tokens, 0) - + COALESCE(reasoning_tokens, 0) > 0""" - ) - except sqlite3.OperationalError: - pass - if current_version < 22: - # v22: task-dimension usage attribution (issue #23270). - # session_model_usage gains a ``task`` column ('' = main agent - # loop; 'vision'/'compression'/'title_generation'/... = - # auxiliary calls) so aux model spend is visible in analytics. - # The column participates in the PRIMARY KEY and SQLite cannot - # ALTER a PK, so rebuild the table. The reconciler will have - # already ADDed the plain column on legacy DBs (harmless); - # the rebuild bakes it into the PK properly. Existing rows are - # main-loop accounting by definition → task=''. - try: - legacy_pk = cursor.execute( - "SELECT COUNT(*) FROM pragma_table_info('session_model_usage') " - "WHERE name = 'task' AND pk > 0" - ).fetchone()[0] - if not legacy_pk: - cursor.execute("ALTER TABLE session_model_usage RENAME TO session_model_usage_v21") - cursor.execute( - """CREATE TABLE session_model_usage ( - session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, - model TEXT NOT NULL, - billing_provider TEXT NOT NULL DEFAULT '', - billing_base_url TEXT NOT NULL DEFAULT '', - billing_mode TEXT NOT NULL DEFAULT '', - task TEXT NOT NULL DEFAULT '', - api_call_count INTEGER NOT NULL DEFAULT 0, - input_tokens INTEGER NOT NULL DEFAULT 0, - output_tokens INTEGER NOT NULL DEFAULT 0, - cache_read_tokens INTEGER NOT NULL DEFAULT 0, - cache_write_tokens INTEGER NOT NULL DEFAULT 0, - reasoning_tokens INTEGER NOT NULL DEFAULT 0, - estimated_cost_usd REAL NOT NULL DEFAULT 0, - actual_cost_usd REAL NOT NULL DEFAULT 0, - cost_status TEXT, - cost_source TEXT, - first_seen REAL, - last_seen REAL, - PRIMARY KEY (session_id, model, billing_provider, billing_base_url, billing_mode, task) - )""" - ) - cursor.execute( - """INSERT INTO session_model_usage ( - session_id, model, billing_provider, billing_base_url, - billing_mode, task, api_call_count, input_tokens, - output_tokens, cache_read_tokens, cache_write_tokens, - reasoning_tokens, estimated_cost_usd, actual_cost_usd, - cost_status, cost_source, first_seen, last_seen - ) - SELECT session_id, model, billing_provider, billing_base_url, - billing_mode, '', api_call_count, input_tokens, - output_tokens, cache_read_tokens, cache_write_tokens, - reasoning_tokens, estimated_cost_usd, actual_cost_usd, - cost_status, cost_source, first_seen, last_seen - FROM session_model_usage_v21""" - ) - cursor.execute("DROP TABLE session_model_usage_v21") - cursor.execute( - "CREATE INDEX IF NOT EXISTS idx_session_model_usage_session " - "ON session_model_usage(session_id)" - ) - cursor.execute( - "CREATE INDEX IF NOT EXISTS idx_session_model_usage_model " - "ON session_model_usage(model)" - ) - except sqlite3.OperationalError as exc: - logger.debug("v22 session_model_usage rebuild skipped: %s", exc) - if current_version < 23: - # v23: FTS storage redesign (issues #22478, #43690, #55233). - # The v11 inline-mode FTS tables each store a full private - # copy of every message (content || tool_name || tool_calls), - # and the trigram index additionally covers role='tool' rows - # (~90% of message bytes: base64 payloads, file dumps) at - # ~2.6x amplification — together ~75% of state.db on heavy - # installs (observed: 18.9 GB of a 25 GB DB). - # - # OPT-IN, NOT AUTOMATIC. The transition (demote old vtables → - # new external-content schema → backfill → teardown → VACUUM) - # is disk-heavy (transient ~2x file size to fully reclaim via - # VACUUM) and long (~1-2h background on a 25 GB DB). Doing it - # silently on every big user's next open — with a completeness - # guarantee that depends on the process staying alive long - # enough — is the wrong default. So on an EXISTING install we - # touch nothing here: the v22 inline FTS keeps working exactly - # as before, and we only record a flag advertising that the - # optimization is available. `hermes sessions optimize-storage` - # performs the whole transition as one deliberate, disk-checked, - # progress-reported foreground operation. - # - # DECOUPLED VERSIONING. Crucially, this does NOT hold back the - # main schema_version. The FTS storage LAYOUT is tracked by an - # independent `fts_storage_version` marker (see - # _fts_storage_version / SETTLE below), so schema_version - # advances to SCHEMA_VERSION here like every other migration — - # future v24+ migrations land automatically for legacy-FTS - # users too. Only the FTS *layout* waits for opt-in. - if fts5_available and self._db_has_legacy_inline_fts(cursor): - self.set_meta("fts_optimize_available", "1", cursor=cursor) - - # The FTS storage layout is versioned independently of the main - # schema (see the v23 note above). Stamp the current layout so the - # main version can always advance: a fresh/optimized DB is at - # FTS_STORAGE_VERSION; a legacy DB is left at whatever it had - # (absent/0) until `optimize-storage` runs. An INTERRUPTED - # optimize (legacy vtables already demoted, but rebuild markers - # or demoted trash tables still present) is NOT stamped either — - # the marker is the source of truth for "fully optimized", and - # `fts_optimize_available()` keeps offering the resume until the - # transition actually completes. - if ( - fts5_available - and not self._db_has_legacy_inline_fts(cursor) - and cursor.execute( - "SELECT 1 FROM state_meta " - "WHERE key = 'fts_rebuild_high_water' LIMIT 1" - ).fetchone() is None - and not self._has_fts_trash(cursor) - ): - self.set_meta( - "fts_storage_version", str(FTS_STORAGE_VERSION), cursor=cursor - ) - - # Advance schema_version to current for ALL non-FTS-layout - # migrations. This is deliberately NOT gated on the FTS opt-in — - # holding the whole version back would block every future schema - # migration for a user who never optimizes. FTS5 being unavailable - # is the one case we skip (we can't have created the current FTS - # objects, so claiming the current schema would be a lie). - if ( - current_version < SCHEMA_VERSION - and fts_migrations_complete - and fts5_available - ): - cursor.execute( - "UPDATE schema_version SET version = ?", - (SCHEMA_VERSION,), - ) - - # Unique title index — always ensure it exists. Older databases may - # contain duplicate aliases from before the constraint was enforced; - # preserve every session while letting the newest one retain the alias. - title_index_sql = ( - "CREATE UNIQUE INDEX IF NOT EXISTS idx_sessions_title_unique " - "ON sessions(title) WHERE title IS NOT NULL" - ) - try: - cursor.execute(title_index_sql) - except sqlite3.IntegrityError: - # The index is an optimization — its creation must never abort - # opening the database, so the repair itself is also guarded. - try: - cursor.execute( - """UPDATE sessions AS older - SET title = NULL - WHERE title IS NOT NULL - AND EXISTS ( - SELECT 1 FROM sessions AS newer - WHERE newer.title = older.title - AND newer.rowid > older.rowid - )""" - ) - logger.warning( - "Cleared %d duplicate session title(s) while restoring the unique index", - cursor.rowcount, - ) - cursor.execute(title_index_sql) - except sqlite3.Error: - logger.exception( - "Could not repair duplicate session titles; " - "unique title index not created" - ) - except sqlite3.OperationalError: - pass # Index already exists - - if fts5_available: - # FTS5 setup. Run the DDL even when the virtual table exists so - # CREATE TRIGGER IF NOT EXISTS repairs trigger-only degradation from - # an earlier no-FTS5 runtime. - # - # OPT-IN v23 boundary: a legacy v22 install (inline-content FTS, - # not yet opted into `hermes db optimize`) must keep its EXISTING - # inline schema + triggers. Running the v23 external-content DDL - # here would create the trigram source VIEW and leave the DB in a - # mixed inline/external state. So for a legacy DB we only ensure - # its inline triggers exist (via the legacy DDL), and skip the - # v23 view/external tables entirely. Fresh installs and opted-in - # DBs have no legacy inline FTS, so they get the v23 DDL. - if self._db_has_legacy_inline_fts(cursor): - triggers_need_repair = ( - self._fts_trigger_count(cursor) < len(_FTS_TRIGGERS) - ) - self._fts_enabled = self._ensure_fts_schema( - cursor, "messages_fts", LEGACY_FTS_SQL - ) - if self._fts_enabled: - trigram_enabled = self._ensure_fts_schema( - cursor, "messages_fts_trigram", LEGACY_FTS_TRIGRAM_SQL - ) - self._trigram_available = trigram_enabled - if triggers_need_repair: - self._rebuild_legacy_fts_indexes( - cursor, include_trigram=trigram_enabled - ) - else: - triggers_need_repair = ( - self._fts_trigger_count(cursor) < len(_FTS_TRIGGERS) - ) - self._fts_enabled = self._ensure_fts_schema( - cursor, "messages_fts", FTS_SQL - ) - - # Trigram FTS5 for CJK/substring search. This is optional - # relative to the main FTS table; if it cannot be created, - # CJK search falls back to LIKE. - if self._fts_enabled: - trigram_enabled = self._ensure_fts_schema( - cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL - ) - self._trigram_available = trigram_enabled - if triggers_need_repair: - self._rebuild_fts_indexes( - cursor, - include_trigram=trigram_enabled, - ) - # CJK-bigram index (cjk_unicode61). Strictly additive to - # the surfaces above and gated on the loadable tokenizer: - self._ensure_fts_cjk_schema(cursor) - - self._conn.commit() - # ========================================================================= # Session lifecycle # ========================================================================= @@ -4401,55 +2692,6 @@ class SessionDB: return None return str(rows[0]["id"]) - def _backfill_gateway_metadata_from_sessions_json( - self, cursor: sqlite3.Cursor - ) -> None: - """One-time v18 backfill of gateway metadata from sessions.json. - - Existing gateway sessions predate the display_name / origin_json / - expiry_finalized columns; copy what sessions.json knows so consumers - can switch to state.db without losing pre-migration sessions. - Only fills NULL columns — never overwrites data written by newer code. - """ - sessions_file = get_hermes_home() / "sessions" / "sessions.json" - if not sessions_file.exists(): - return - with open(sessions_file, "r", encoding="utf-8") as f: - data = json.load(f) - if not isinstance(data, dict): - return - for key, entry in data.items(): - if str(key).startswith("_") or not isinstance(entry, dict): - continue - session_id = entry.get("session_id") - if not session_id: - continue - origin = entry.get("origin") - cursor.execute( - """UPDATE sessions - SET session_key = COALESCE(session_key, ?), - chat_id = COALESCE(chat_id, ?), - chat_type = COALESCE(chat_type, ?), - thread_id = COALESCE(thread_id, ?), - display_name = COALESCE(display_name, ?), - origin_json = COALESCE(origin_json, ?), - expiry_finalized = CASE - WHEN COALESCE(expiry_finalized, 0) = 0 AND ? = 1 THEN 1 - ELSE expiry_finalized - END - WHERE id = ?""", - ( - entry.get("session_key") or key, - (origin or {}).get("chat_id") if isinstance(origin, dict) else None, - entry.get("chat_type"), - (origin or {}).get("thread_id") if isinstance(origin, dict) else None, - entry.get("display_name"), - json.dumps(origin) if isinstance(origin, dict) else None, - 1 if entry.get("expiry_finalized") or entry.get("memory_flushed") else 0, - str(session_id), - ), - ) - def find_latest_gateway_session_for_peer( self, *, @@ -6435,45 +4677,6 @@ class SessionDB: _SESSION_COMPACT_EXCLUDED = frozenset({"system_prompt"}) _session_compact_cols_sql: Optional[str] = None - @classmethod - def _compact_session_cols(cls) -> str: - """SELECT list for compact_rows: every ``sessions`` column declared in - SCHEMA_SQL except the ``system_prompt`` blob, aliased with the ``s`` - prefix used by list_sessions_rich/_get_session_rich_row queries.""" - if cls._session_compact_cols_sql is None: - declared = cls._parse_schema_columns(SCHEMA_SQL)["sessions"] - cls._session_compact_cols_sql = ", ".join( - f"s.{name}" for name in declared - if name not in cls._SESSION_COMPACT_EXCLUDED - ) - return cls._session_compact_cols_sql - - def distinct_session_cwds(self, include_archived: bool = False) -> List[Dict[str, Any]]: - """Distinct non-empty session cwds with usage stats, for repo discovery. - - Aggregates across ALL session history (not a single page), so the desktop - can surface every git repo the user has worked in — not just the repos - that happen to be in the currently-loaded recents. Children/branches - count: a worktree session is still a real workspace signal. - """ - where = "cwd IS NOT NULL AND TRIM(cwd) != ''" - if not include_archived: - where += " AND archived = 0" - with self._lock: - rows = self._conn.execute( - "SELECT cwd AS cwd, COUNT(*) AS sessions, " - "MAX(COALESCE(ended_at, started_at, 0)) AS last_active " - f"FROM sessions WHERE {where} GROUP BY cwd" - ).fetchall() - return [ - { - "cwd": r["cwd"], - "sessions": int(r["sessions"] or 0), - "last_active": float(r["last_active"] or 0), - } - for r in rows - ] - def list_sessions_rich( self, source: str = None, @@ -6766,157 +4969,6 @@ class SessionDB: return sessions - def list_cron_job_runs( - self, - job_id: str, - limit: int = 20, - offset: int = 0, - ) -> List[Dict[str, Any]]: - """List the run sessions produced by a single cron job, newest first. - - Cron runs are flat, independent sessions whose id is - ``cron_{job_id}_{timestamp}`` (see ``cron/scheduler.run_job``). They are - never compression roots and never branch, so this deliberately skips the - ``list_sessions_rich`` recursive compression-chain CTE / leading-wildcard - ``id_query`` path — that path seeds from *every* ``source='cron'`` row in - the DB and only filters to one job's runs after the scan, so it scales - with the whole cron pile (a heavy history makes the desktop run-history - endpoint time out before it eventually populates). - - Instead this binds to one job with a ``[prefix, prefix_hi)`` range over - the id (an index range scan, not a ``%...%`` substring), filters - ``source='cron'``, and orders by ``started_at DESC``. Work scales with - the requested window, not the total cron history. - - Returns the same enriched row shape as ``list_sessions_rich`` (adds - ``preview`` + ``last_active``) so callers can reuse it. - """ - prefix = f"cron_{job_id}_" - # Half-open upper bound for an index range scan: increment the final - # byte of the prefix so the range covers exactly the ids that start - # with ``prefix`` and nothing else. ``prefix`` always ends in '_', but - # compute it generically rather than hardcoding the successor char. - prefix_hi = prefix[:-1] + chr(ord(prefix[-1]) + 1) - - query = f""" - SELECT s.*, - COALESCE( - (SELECT {_PREVIEW_RAW_SELECT} - FROM messages m - WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL - ORDER BY m.timestamp, m.id LIMIT 1), - '' - ) AS _preview_raw, - COALESCE( - (SELECT MAX(m2.timestamp) FROM messages m2 WHERE m2.session_id = s.id), - s.started_at - ) AS last_active - FROM sessions s - WHERE s.source = 'cron' AND s.id >= ? AND s.id < ? - ORDER BY s.started_at DESC, s.id DESC - LIMIT ? OFFSET ? - """ - with self._lock: - cursor = self._conn.execute(query, (prefix, prefix_hi, limit, offset)) - rows = cursor.fetchall() - - runs: List[Dict[str, Any]] = [] - for row in rows: - s = dict(row) - s["preview"] = _shape_preview(s.pop("_preview_raw", "")) - runs.append(s) - return runs - - def _get_session_rich_row(self, session_id: str, compact_rows: bool = False) -> Optional[Dict[str, Any]]: - """Fetch a single session with the same enriched columns as - ``list_sessions_rich`` (preview + last_active). Returns None if the - session doesn't exist. - - Pass ``compact_rows=True`` to omit the ``system_prompt`` blob (see - ``list_sessions_rich`` for details). - """ - # Same read-your-writes guarantee as list_sessions_rich. - self.flush_token_counts() - _sel = self._compact_session_cols() if compact_rows else "s.*" - query = f""" - SELECT {_sel}, - COALESCE( - (SELECT {_PREVIEW_RAW_SELECT} - FROM messages m - WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL - ORDER BY m.timestamp, m.id LIMIT 1), - '' - ) AS _preview_raw, - COALESCE( - (SELECT MAX(m2.timestamp) FROM messages m2 WHERE m2.session_id = s.id), - s.started_at - ) AS last_active - FROM sessions s - WHERE s.id = ? - """ - with self._lock: - cursor = self._conn.execute(query, (session_id,)) - row = cursor.fetchone() - if not row: - return None - s = dict(row) - s["preview"] = _shape_preview(s.pop("_preview_raw", "")) - return s - - def get_session_rich_row(self, session_id: str, compact_rows: bool = False) -> Optional[Dict[str, Any]]: - """Public wrapper for :meth:`_get_session_rich_row`. - - Exposes the single-session enriched row (same columns as - ``list_sessions_rich``: preview + last_active) for callers outside - this module, e.g. the web server's session-search hydration. - """ - return self._get_session_rich_row(session_id, compact_rows=compact_rows) - - def list_skill_scaffolded_sessions(self, limit: int = 200) -> List[Dict[str, Any]]: - """Titled sessions whose first user turn was a ``/skill`` invocation. - - Those titles were generated from the expanded message, which embeds the - whole skill body — so they describe the skill rather than the request. - Returns ``id``, ``title``, and the full first-turn ``content`` so a - caller can re-derive what the user typed. Newest first. - """ - with self._lock: - rows = self._conn.execute( - """ - SELECT s.id, s.title, m.content - FROM sessions s - JOIN messages m ON m.id = ( - SELECT m2.id FROM messages m2 - WHERE m2.session_id = s.id AND m2.role = 'user' - AND m2.content IS NOT NULL - ORDER BY m2.timestamp, m2.id LIMIT 1 - ) - WHERE s.title IS NOT NULL AND m.content LIKE ? - ORDER BY s.started_at DESC - LIMIT ? - """, - (SKILL_SCAFFOLD_SQL_LIKE, int(limit)), - ).fetchall() - return [dict(row) for row in rows] - - def get_first_assistant_text(self, session_id: str) -> str: - """The session's first assistant reply as plain text ('' when none). - - Pairs with :meth:`list_skill_scaffolded_sessions` so a re-title can feed - the titler the same (request, reply) shape the live path uses. - """ - with self._lock: - row = self._conn.execute( - "SELECT content FROM messages " - "WHERE session_id = ? AND role = 'assistant' AND content IS NOT NULL " - "ORDER BY timestamp, id LIMIT 1", - (session_id,), - ).fetchone() - if not row: - return "" - decoded = self._decode_content(row["content"]) - return decoded if isinstance(decoded, str) else "" - # ========================================================================= # Message storage # ========================================================================= @@ -7606,128 +5658,6 @@ class SessionDB: "messages_after": messages_after, } - def get_anchored_view( - self, - session_id: str, - around_message_id: int, - window: int = 5, - bookend: int = 3, - keep_roles: Optional[Tuple[str, ...]] = ("user", "assistant"), - ) -> Dict[str, Any]: - """Return an anchored window plus session bookends. - - Built on top of ``get_messages_around``. Three slices: - - - ``window``: messages immediately surrounding the anchor. Filtered - to ``keep_roles`` (tool-response noise dropped by default), EXCEPT - the anchor itself is always preserved regardless of role. - - ``bookend_start``: first ``bookend`` user/assistant messages of the - session — but only those whose id is strictly before the window's - first message id. Empty when the window already overlaps the - session head. Empty-content messages (tool-call-only assistant - turns) are skipped so they don't crowd out actual prose openings. - - ``bookend_end``: last ``bookend`` user/assistant messages of the - session, same non-overlap rule at the tail. - - Bookends let an FTS5 hit anywhere in a long session yield the goal - (opening) and the resolution (closing) on a single call — without - loading the whole transcript. - - Returns ``{"window": [], "messages_before": 0, "messages_after": 0, - "bookend_start": [], "bookend_end": []}`` when the anchor isn't in - the session. - - ``keep_roles=None`` disables role filtering (raw window + raw - bookends). - """ - if bookend < 0: - bookend = 0 - - # Reuse the primitive — handles anchor-existence, content decoding, - # tool_calls deserialisation, and boundary counts. - primitive = self.get_messages_around( - session_id, around_message_id, window=window - ) - window_rows = primitive["window"] - if not window_rows: - return { - "window": [], - "messages_before": 0, - "messages_after": 0, - "bookend_start": [], - "bookend_end": [], - } - - # Apply role filter to the window, but never drop the anchor itself. - if keep_roles is not None: - keep_set = set(keep_roles) - filtered_window = [ - m for m in window_rows - if m.get("id") == around_message_id or m.get("role") in keep_set - ] - else: - filtered_window = window_rows - - window_min_id = window_rows[0]["id"] - window_max_id = window_rows[-1]["id"] - - # Fetch bookends only when there's room outside the window. SQL filters - # by id range, role, and non-empty content — tool-call-only assistant - # turns (content='' with tool_calls populated) are excluded so they - # don't crowd out actual prose openings/closings. - bookend_start_rows: List[Any] = [] - bookend_end_rows: List[Any] = [] - if bookend > 0: - with self._read_ctx() as conn: - role_clause = "" - role_params: list = [] - if keep_roles is not None: - role_placeholders = ",".join("?" for _ in keep_roles) - role_clause = f" AND role IN ({role_placeholders})" - role_params = list(keep_roles) - - bookend_start_rows = conn.execute( - f"SELECT * FROM messages " - f"WHERE session_id = ? AND id < ?{role_clause} " - f"AND length(content) > 0 " - f"ORDER BY id ASC LIMIT ?", - (session_id, window_min_id, *role_params, bookend), - ).fetchall() - - bookend_end_rows = conn.execute( - f"SELECT * FROM messages " - f"WHERE session_id = ? AND id > ?{role_clause} " - f"AND length(content) > 0 " - f"ORDER BY id DESC LIMIT ?", - (session_id, window_max_id, *role_params, bookend), - ).fetchall() - # End rows came back DESC for the LIMIT cap; flip to ASC. - bookend_end_rows = list(reversed(bookend_end_rows)) - - def _hydrate(row) -> Dict[str, Any]: - msg = dict(row) - if "content" in msg: - msg["content"] = self._decode_content(msg["content"]) - if msg.get("tool_calls"): - try: - msg["tool_calls"] = json.loads(msg["tool_calls"]) - except (json.JSONDecodeError, TypeError): - logger.warning( - "Failed to deserialize tool_calls in get_anchored_view, falling back to []" - ) - msg["tool_calls"] = [] - if msg.get("display_metadata") is not None: - msg["display_metadata"] = self._decode_display_metadata(msg["display_metadata"]) - return msg - - return { - "window": filtered_window, - "messages_before": primitive["messages_before"], - "messages_after": primitive["messages_after"], - "bookend_start": [_hydrate(r) for r in bookend_start_rows], - "bookend_end": [_hydrate(r) for r in bookend_end_rows], - } - def resolve_resume_session_id(self, session_id: str) -> str: """Redirect a resume target to the descendant session that holds the messages. @@ -8256,1053 +6186,10 @@ class SessionDB: return self._execute_write(_do) - def list_recent_user_messages( - self, - session_id: str, - limit: int = 20, - include_inactive: bool = False, - ) -> List[Dict[str, Any]]: - """Return the *limit* most-recent user messages, newest first. - - Each entry is a dict with keys ``id``, ``timestamp``, ``preview``. - ``preview`` is the first 80 characters of the message content - (with line breaks collapsed to spaces). Used by the /rewind - slash command picker, CLI/TUI/gateway ``/undo [N]``, and any other - caller that needs real user-turn targets. - - Bookkeeping timeline rows (``display_kind`` set — e.g. model_switch, - async_delegation_complete, auto_continue, hidden) are excluded. They - are durable ``role='user'`` rows for the API transcript, but no client - counts them as user turns (desktop demotes them to system / drops them; - the CLI already uses ``not m.get("display_kind")``). Including them here - made ``/undo`` soft-delete from a marker instead of the last real turn — - same class of index skew as the prompt.submit ordinal bug. - - By default only active messages are returned. - """ - active_clause = "" if include_inactive else " AND active = 1" - # Match CLI/desktop: only real user turns, not timeline bookkeeping. - display_clause = " AND (display_kind IS NULL OR display_kind = '')" - with self._lock: - cursor = self._conn.execute( - "SELECT id, timestamp, content FROM messages " - "WHERE session_id = ? AND role = 'user'" - f"{active_clause}{display_clause} " - "ORDER BY id DESC LIMIT ?", - (session_id, int(limit)), - ) - rows = cursor.fetchall() - - result: List[Dict[str, Any]] = [] - for row in rows: - decoded = self._decode_content(row["content"]) - if isinstance(decoded, list): - # Multimodal — flatten text parts. - text_parts = [ - p.get("text", "") for p in decoded - if isinstance(p, dict) and p.get("type") == "text" - ] - preview = " ".join(t for t in text_parts if t).strip() - if not preview: - preview = "[multimodal content]" - elif isinstance(decoded, str): - # A /skill turn embeds the whole skill body; show what the user - # typed instead of the skill's opening prose. - preview = describe_skill_invocation(decoded) or decoded - else: - preview = "" - preview = " ".join(preview.split()) # collapse whitespace - if len(preview) > 80: - preview = preview[:77] + "..." - result.append( - { - "id": row["id"], - "timestamp": row["timestamp"], - "preview": preview, - } - ) - return result - # ========================================================================= # Search # ========================================================================= - @staticmethod - def _sanitize_fts5_query(query: str) -> str: - """Sanitize user input for safe use in FTS5 MATCH queries. - - FTS5 has its own query syntax where characters like ``"``, ``(``, ``)``, - ``+``, ``*``, ``{``, ``}``, the column-filter operator ``:`` and bare - boolean operators (``AND``, ``OR``, ``NOT``) have special meaning. - Passing raw user input directly to MATCH can cause - ``sqlite3.OperationalError``. - - Strategy: - - Preserve properly paired quoted phrases (``"exact phrase"``) - - Strip unmatched FTS5-special characters that would cause errors - - Wrap unquoted hyphenated and dotted terms in quotes so FTS5 - matches them as exact phrases instead of splitting on the - hyphen/dot (e.g. ``chat-send``, ``P2.2``, ``my-app.config.ts``) - """ - # Cap user-controlled FTS input before any regex processing. Search - # queries do not need to be arbitrarily large, and bounding them keeps - # sanitizer/runtime behavior predictable under adversarial input. - query = query[:MAX_FTS5_QUERY_CHARS] - - # Step 1: Extract balanced double-quoted phrases and protect them - # from further processing via numbered placeholders. Do this with a - # single linear scan rather than a regex so pathological quote runs - # cannot induce backtracking. - _quoted_parts: list = [] - pieces: list[str] = [] - i = 0 - while i < len(query): - ch = query[i] - if ch != '"': - pieces.append(ch) - i += 1 - continue - end = query.find('"', i + 1) - if end == -1: - # Unmatched quote: replace with whitespace like the old - # sanitizer's special-char stripping step. - pieces.append(" ") - i += 1 - continue - _quoted_parts.append(query[i:end + 1]) - pieces.append(f"\x00Q{len(_quoted_parts) - 1}\x00") - i = end + 1 - - sanitized = "".join(pieces) - - # Step 2: Strip remaining (unmatched) FTS5-special characters. ``:`` is - # FTS5's column-filter operator (``col:term``); since the FTS table has a - # single ``content`` column, an unquoted colon query like ``TODO: fix`` - # parses as ``column:term`` and raises "no such column" — swallowed at - # the execute site into zero results. Strip it like the others. - sanitized = re.sub(r'[+{}():\"^]', " ", sanitized) - - # Step 3: Collapse repeated * (e.g. "***") into a single one, - # and remove leading * (prefix-only needs at least one char before *) - sanitized = re.sub(r"\*+", "*", sanitized) - sanitized = re.sub(r"(^|\s)\*", r"\1", sanitized) - - # Step 4: Remove dangling boolean operators at start/end that would - # cause syntax errors (e.g. "hello AND" or "OR world") - sanitized = re.sub(r"(?i)^(AND|OR|NOT)\b\s*", "", sanitized.strip()) - sanitized = re.sub(r"(?i)\s+(AND|OR|NOT)\s*$", "", sanitized.strip()) - - # Step 5: Wrap unquoted dotted and/or hyphenated terms in double - # quotes. FTS5's tokenizer splits on dots and hyphens, turning - # ``chat-send`` into ``chat AND send`` and ``P2.2`` into ``p2 AND 2``. - # Quoting preserves phrase semantics. A single pass avoids the - # double-quoting bug that would occur if dotted, hyphenated and underscored - # patterns were applied sequentially (e.g. ``my-app.config``). - sanitized = re.sub(r"\b(\w+(?:[._-]\w+)+)\b", r'"\1"', sanitized) - - # Step 6: Restore preserved quoted phrases - for i, quoted in enumerate(_quoted_parts): - sanitized = sanitized.replace(f"\x00Q{i}\x00", quoted) - - return sanitized.strip() - - - @staticmethod - def _is_cjk_codepoint(cp: int) -> bool: - return (0x4E00 <= cp <= 0x9FFF or # CJK Unified Ideographs - 0x3400 <= cp <= 0x4DBF or # CJK Extension A - 0x20000 <= cp <= 0x2A6DF or # CJK Extension B - 0x3000 <= cp <= 0x303F or # CJK Symbols - 0x3040 <= cp <= 0x309F or # Hiragana - 0x30A0 <= cp <= 0x30FF or # Katakana - 0xAC00 <= cp <= 0xD7AF) # Hangul Syllables - - @staticmethod - def _contains_cjk(text: str) -> bool: - """Check if text contains CJK (Chinese, Japanese, Korean) characters.""" - for ch in text: - cp = ord(ch) - if (0x4E00 <= cp <= 0x9FFF or # CJK Unified Ideographs - 0x3400 <= cp <= 0x4DBF or # CJK Extension A - 0x20000 <= cp <= 0x2A6DF or # CJK Extension B - 0x3000 <= cp <= 0x303F or # CJK Symbols - 0x3040 <= cp <= 0x309F or # Hiragana - 0x30A0 <= cp <= 0x30FF or # Katakana - 0xAC00 <= cp <= 0xD7AF): # Hangul Syllables - return True - return False - - @classmethod - def _count_cjk(cls, text: str) -> int: - """Count CJK characters in text.""" - return sum(1 for ch in text if cls._is_cjk_codepoint(ord(ch))) - - @classmethod - def _has_lone_cjk_run(cls, query: str) -> bool: - """True when any maximal CJK run in the query is a single char. - - The cjk-bigram index stores bigrams for runs >=2 chars and unigrams - only for isolated chars, so a 1-char CJK term can't match inside - longer runs there — those queries keep the LIKE substring route. - """ - run = 0 - for ch in query: - if cls._is_cjk_codepoint(ord(ch)): - run += 1 - else: - if run == 1: - return True - run = 0 - return run == 1 - - @staticmethod - def _trigram_eligible_tokens(query: str) -> bool: - """True when every non-operator token is long enough for the trigram - tokenizer to match (>=3 chars). - - The trigram tokenizer indexes overlapping 3-character sequences, so a - token shorter than 3 chars produces no trigrams and can never match. - With FTS5's implicit-AND between tokens, a single short token makes the - whole MATCH return nothing, so the trigram path is only worth taking - when every searchable token qualifies. - """ - tokens = [ - t for t in query.strip('"').strip().split() - if t.upper() not in {"AND", "OR", "NOT"} - ] - return bool(tokens) and all(len(t) >= 3 for t in tokens) - - def _run_trigram_search( - self, - raw_query: str, - *, - table: str = "messages_fts_trigram", - order_by_sql: str, - include_inactive: bool, - source_filter: List[str] = None, - exclude_sources: List[str] = None, - role_filter: List[str] = None, - limit: int = 20, - offset: int = 0, - ) -> Optional[List[Dict[str, Any]]]: - """Run a search against a substring-capable FTS index. - - ``table`` is ``messages_fts_trigram`` (default) or - ``messages_fts_cjk``. The trigram tokenizer indexes overlapping - 3-byte sequences, so it matches substrings regardless of word - boundaries — both CJK phrases the unicode61 tokenizer splits into - single characters and Latin runs the unicode61 tokenizer fuses onto - adjacent CJK (e.g. ``修改youer服务端``). The cjk-bigram tokenizer - splits Latin runs off adjacent CJK, giving the same recovery as an - exact ranked token match. Each non-operator token is quoted to - neutralise FTS5 special characters while boolean operators - (AND/OR/NOT) are preserved. - - Returns the matching rows, or ``None`` when the query cannot be - executed (e.g. the tokenizer is unavailable at runtime) so the - caller can fall back to another strategy. - """ - tokens = raw_query.split() - parts = [] - for tok in tokens: - if tok.upper() in {"AND", "OR", "NOT"}: - parts.append(tok) - else: - parts.append('"' + tok.replace('"', '""') + '"') - trigram_query = " ".join(parts) - tri_where = [f"{table} MATCH ?"] - tri_params: list = [trigram_query] - if not include_inactive: - tri_where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - tri_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - tri_params.extend(source_filter) - if exclude_sources is not None: - tri_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") - tri_params.extend(exclude_sources) - if role_filter: - tri_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - tri_params.extend(role_filter) - tri_sql = f""" - SELECT - m.id, - m.session_id, - m.role, - snippet({table}, -1, '>>>', '<<<', '...', 40) AS snippet, - m.content, - m.timestamp, - m.tool_name, - s.source, - s.model, - s.started_at AS session_started - FROM {table} - JOIN messages m ON m.id = {table}.rowid - JOIN sessions s ON s.id = m.session_id - WHERE {' AND '.join(tri_where)} - {order_by_sql} - LIMIT ? OFFSET ? - """ - tri_params.extend([limit, offset]) - with self._read_ctx() as conn: - try: - tri_cursor = conn.execute(tri_sql, tri_params) - except sqlite3.OperationalError: - # Query failed at runtime — let the caller fall back. - return None - return [dict(row) for row in tri_cursor.fetchall()] - - def search_messages( - self, - query: str, - source_filter: List[str] = None, - exclude_sources: List[str] = None, - role_filter: List[str] = None, - limit: int = 20, - offset: int = 0, - sort: str = None, - include_inactive: bool = False, - ) -> List[Dict[str, Any]]: - """Instrumented wrapper around :meth:`_search_messages_impl`. - - Logs one line per slow search with the routing path taken, so - production latency stays attributable per query shape (the 2026-07 - session_search investigation needed trace archaeology to discover - the LIKE full scans; this makes the next regression a grep). - Threshold: HERMES_SEARCH_SLOW_MS (default 1000; 0 logs every call). - """ - started = time.time() - rows = None - try: - rows = self._search_messages_impl( - query, - source_filter=source_filter, - exclude_sources=exclude_sources, - role_filter=role_filter, - limit=limit, - offset=offset, - sort=sort, - include_inactive=include_inactive, - ) - return rows - finally: - try: - threshold = float(os.getenv("HERMES_SEARCH_SLOW_MS", "1000")) - except (TypeError, ValueError): - threshold = 1000.0 - elapsed_ms = (time.time() - started) * 1000.0 - if elapsed_ms >= threshold: - logger.info( - "slow session search: path=%s elapsed=%.0fms rows=%s query=%r", - self._describe_search_path(query), - elapsed_ms, - len(rows) if rows is not None else "err", - query[:200], - ) - - def _describe_search_path(self, query: str) -> str: - """Best-effort name of the routing path a query takes (log-only).""" - try: - sanitized = self._sanitize_fts5_query(query or "") - if not sanitized: - return "empty" - if not self._contains_cjk(sanitized): - return "fts5" - raw = sanitized.strip('"').strip() - if self._fts_cjk_available and not self._has_lone_cjk_run(raw): - return "fts_cjk" - tokens = [ - t for t in raw.split() - if t.upper() not in {"AND", "OR", "NOT"} and self._contains_cjk(t) - ] - short = any(self._count_cjk(t) < 3 for t in tokens) - if self._count_cjk(raw) >= 3 and not short and self._trigram_available: - return "trigram" - return "like_scan" - except Exception: - return "unknown" - - def _search_messages_impl( - self, - query: str, - source_filter: List[str] = None, - exclude_sources: List[str] = None, - role_filter: List[str] = None, - limit: int = 20, - offset: int = 0, - sort: str = None, - include_inactive: bool = False, - ) -> List[Dict[str, Any]]: - """ - Full-text search across session messages using FTS5. - - Supports FTS5 query syntax: - - Simple keywords: "docker deployment" - - Phrases: '"exact phrase"' - - Boolean: "docker OR kubernetes", "python NOT java" - - Prefix: "deploy*" - - Returns matching messages with session metadata, content snippet, - and surrounding context (1 message before and after the match). - - ``sort`` controls temporal ordering: - - ``None`` (default): FTS5 BM25 relevance only. Time-neutral. - - ``"newest"``: order by message timestamp DESC, then by rank. - - ``"oldest"``: order by message timestamp ASC, then by rank. - - The short-CJK LIKE fallback already orders by timestamp DESC and - ignores ``sort``. The trigram CJK path honours ``sort`` like the main - FTS5 path. - - Rewound (``active=0``, ``compacted=0``) rows are excluded by default — - the user took those back. Compaction-archived rows (``active=0``, - ``compacted=1``) ARE included by default: they were summarized away from - the live context but remain part of the conversation's record, so the - pre-compaction transcript stays discoverable after in-place compaction - (#38763). Pass ``include_inactive=True`` to search every row regardless. - """ - if not self._fts_enabled: - return [] - - if not query or not query.strip(): - return [] - - query = self._sanitize_fts5_query(query) - if not query: - return [] - - # Normalise sort. Anything not in the allowed set falls back to None - # (FTS5 rank-only) so callers can pass through user input without - # validation. - if isinstance(sort, str): - sort_norm = sort.strip().lower() - if sort_norm not in ("newest", "oldest"): - sort_norm = None - else: - sort_norm = None - - # ORDER BY shared across the main FTS5 path and trigram CJK path. - # With sort set, timestamp is primary and rank is the tiebreaker. - if sort_norm == "newest": - order_by_sql = "ORDER BY m.timestamp DESC, rank" - elif sort_norm == "oldest": - order_by_sql = "ORDER BY m.timestamp ASC, rank" - else: - order_by_sql = "ORDER BY rank" - - # Build WHERE clauses dynamically - where_clauses = ["messages_fts MATCH ?"] - params: list = [query] - if not include_inactive: - # Live rows (active=1) AND compaction-archived rows (compacted=1) - # are discoverable; only rewind/undo rows (active=0, compacted=0) - # are hidden. See archive_and_compact() / #38763. - where_clauses.append("(m.active = 1 OR m.compacted = 1)") - - if source_filter is not None: - source_placeholders = ",".join("?" for _ in source_filter) - where_clauses.append(f"s.source IN ({source_placeholders})") - params.extend(source_filter) - - if exclude_sources is not None: - exclude_placeholders = ",".join("?" for _ in exclude_sources) - where_clauses.append(f"s.source NOT IN ({exclude_placeholders})") - params.extend(exclude_sources) - - if role_filter: - role_placeholders = ",".join("?" for _ in role_filter) - where_clauses.append(f"m.role IN ({role_placeholders})") - params.extend(role_filter) - - where_sql = " AND ".join(where_clauses) - params.extend([limit, offset]) - - sql = f""" - SELECT - m.id, - m.session_id, - m.role, - snippet(messages_fts, -1, '>>>', '<<<', '...', 40) AS snippet, - m.content, - m.timestamp, - m.tool_name, - s.source, - s.model, - s.started_at AS session_started - FROM messages_fts - JOIN messages m ON m.id = messages_fts.rowid - JOIN sessions s ON s.id = m.session_id - WHERE {where_sql} - {order_by_sql} - LIMIT ? OFFSET ? - """ - - # CJK queries bypass the unicode61 FTS5 table. The default tokenizer - # splits CJK characters into individual tokens, so "大别山项目" becomes - # "大 AND 别 AND 山 AND 项 AND 目" — producing false positives and - # missing exact phrase matches. - # - # For queries with 3+ CJK characters, we use the trigram FTS5 table - # (indexed substring matching with ranking and snippets). For shorter - # CJK queries (1-2 chars), trigram can't match (it needs ≥9 UTF-8 - # bytes = 3 CJK chars), so we fall back to LIKE. - is_cjk = self._contains_cjk(query) - if is_cjk: - raw_query = query.strip('"').strip() - cjk_count = self._count_cjk(raw_query) - - # Per-token CJK length check (#20494): trigram needs >=3 CJK chars - # per token. A query like "广西 OR 桂林 OR 漓江" has cjk_count=6 - # (>=3) but each individual token is only 2 chars — trigram returns 0. - # Route to LIKE when any non-operator CJK token is <3 CJK chars. - _tokens_for_check = [ - t for t in raw_query.split() - if t.upper() not in {"AND", "OR", "NOT"} and self._contains_cjk(t) - ] - _any_short_cjk = any( - self._count_cjk(t) < 3 for t in _tokens_for_check - ) - - _trigram_succeeded = False - # Tool rows are excluded from the trigram index (they're ~90% of - # message bytes and machine noise — see FTS_TRIGRAM_SQL). A CJK - # query explicitly filtering on role='tool' must therefore use - # the LIKE fallback, which scans the base table directly. - _wants_tool_rows = bool(role_filter) and "tool" in role_filter - - # ── CJK-bigram route (messages_fts_cjk, cjk_unicode61) ────── - # When the bigram index is available it serves EVERY CJK query - # shape the legacy code split between trigram (>=3 chars/token) - # and LIKE full scans (1-2 char tokens) — the whole point of the - # index (PR #65544). Exceptions stay on the legacy routes: - # - role_filter=['tool'] queries (tool rows aren't in the cjk - # index, same exclusion as trigram), - # - queries containing a LONE 1-char CJK run: the index stores - # bigrams for runs >=2, so a single-char term can only match - # isolated chars — LIKE substring semantics are broader. - if ( - self._fts_cjk_available - and not _wants_tool_rows - and not self._has_lone_cjk_run(raw_query) - ): - tokens = raw_query.split() - parts = [] - for tok in tokens: - if tok.upper() in {"AND", "OR", "NOT"}: - parts.append(tok) - else: - parts.append('"' + tok.replace('"', '""') + '"') - cjk_query = " ".join(parts) - cjk_where = ["messages_fts_cjk MATCH ?"] - cjk_params: list = [cjk_query] - if not include_inactive: - cjk_where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - cjk_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - cjk_params.extend(source_filter) - if exclude_sources is not None: - cjk_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") - cjk_params.extend(exclude_sources) - if role_filter: - cjk_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - cjk_params.extend(role_filter) - cjk_sql = f""" - SELECT - m.id, - m.session_id, - m.role, - snippet(messages_fts_cjk, -1, '>>>', '<<<', '...', 40) AS snippet, - m.content, - m.timestamp, - m.tool_name, - s.source, - s.model, - s.started_at AS session_started - FROM messages_fts_cjk - JOIN messages m ON m.id = messages_fts_cjk.rowid - JOIN sessions s ON s.id = m.session_id - WHERE {' AND '.join(cjk_where)} - {order_by_sql} - LIMIT ? OFFSET ? - """ - cjk_params.extend([limit, offset]) - try: - with self._read_ctx() as conn: - cjk_cursor = conn.execute(cjk_sql, cjk_params) - matches = [dict(row) for row in cjk_cursor.fetchall()] - _trigram_succeeded = True - except sqlite3.OperationalError: - # Tokenizer missing on this connection / query syntax — - # the trigram + LIKE routes below still answer. - logger.debug( - "messages_fts_cjk query failed; falling back to " - "trigram/LIKE", exc_info=True, - ) - except sqlite3.DatabaseError as exc: - # Same corruption class as the other FTS reads: rebuild - # in place once and retry; on refusal/failure fall back. - if self._try_runtime_fts_rebuild(exc): - try: - with self._read_ctx() as conn: - cjk_cursor = conn.execute( - cjk_sql, cjk_params - ) - matches = [ - dict(row) for row in cjk_cursor.fetchall() - ] - _trigram_succeeded = True - except sqlite3.DatabaseError: - logger.warning( - "CJK-bigram FTS search still failing after " - "in-place rebuild; falling back to " - "trigram/LIKE." - ) - else: - logger.warning( - "CJK-bigram FTS search hit a corruption error " - "(%s) and no in-place rebuild was possible; " - "falling back to trigram/LIKE.", exc, - ) - - if ( - not _trigram_succeeded - and cjk_count >= 3 - and not _any_short_cjk - and self._trigram_available - and not _wants_tool_rows - ): - # Trigram FTS5 path — quote each non-operator token to handle - # FTS5 special chars (%, *, etc.) while preserving boolean - # operators (AND, OR, NOT) for multi-term queries. - tokens = raw_query.split() - parts = [] - for tok in tokens: - if tok.upper() in {"AND", "OR", "NOT"}: - parts.append(tok) - else: - parts.append('"' + tok.replace('"', '""') + '"') - trigram_query = " ".join(parts) - tri_where = ["messages_fts_trigram MATCH ?"] - tri_params: list = [trigram_query] - if not include_inactive: - tri_where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - tri_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - tri_params.extend(source_filter) - if exclude_sources is not None: - tri_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") - tri_params.extend(exclude_sources) - if role_filter: - tri_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - tri_params.extend(role_filter) - tri_sql = f""" - SELECT - m.id, - m.session_id, - m.role, - snippet(messages_fts_trigram, -1, '>>>', '<<<', '...', 40) AS snippet, - m.content, - m.timestamp, - m.tool_name, - s.source, - s.model, - s.started_at AS session_started - FROM messages_fts_trigram - JOIN messages m ON m.id = messages_fts_trigram.rowid - JOIN sessions s ON s.id = m.session_id - WHERE {' AND '.join(tri_where)} - {order_by_sql} - LIMIT ? OFFSET ? - """ - tri_params.extend([limit, offset]) - try: - with self._read_ctx() as conn: - tri_cursor = conn.execute(tri_sql, tri_params) - matches = [dict(row) for row in tri_cursor.fetchall()] - _trigram_succeeded = True - except sqlite3.OperationalError: - # Trigram query failed at runtime — fall through to LIKE. - pass - except sqlite3.DatabaseError as exc: - # Same corruption class the main FTS5 MATCH branch - # self-heals above: a corrupt trigram shadow table raises - # malformed / "fts5: corrupt structure record", which is a - # DatabaseError (parent of the OperationalError syntax arm - # caught first). Rebuild once outside the lock — the lock - # is released here so rebuild_fts() can re-acquire it — - # and retry the trigram query. If the rebuild is refused - # (already attempted / FTS disabled / different error - # class) or the retry fails again, fall through to the - # LIKE substring path, which reads only the canonical - # messages table, so CJK search stays available. - if self._try_runtime_fts_rebuild(exc): - try: - with self._read_ctx() as conn: - tri_cursor = conn.execute( - tri_sql, tri_params - ) - matches = [ - dict(row) for row in tri_cursor.fetchall() - ] - _trigram_succeeded = True - except sqlite3.DatabaseError: - logger.warning( - "Trigram FTS search still failing after " - "in-place rebuild; falling back to LIKE." - ) - else: - logger.warning( - "Trigram FTS search hit a corruption error (%s) " - "and no in-place rebuild was possible; falling " - "back to LIKE.", exc, - ) - if not _trigram_succeeded: - # Short / mixed CJK query, trigram unavailable, or trigram - # <3 CJK chars. Fall back to LIKE substring search. - # For multi-token OR queries (e.g. "广西 OR 桂林 OR 漓江"), - # build one LIKE condition per non-operator token so each term - # is matched independently (#20494). - non_op_tokens = [ - t for t in raw_query.split() - if t.upper() not in {"AND", "OR", "NOT"} - ] or [raw_query] - token_clauses = [] - like_params: list = [] - for tok in non_op_tokens: - esc = tok.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - token_clauses.append( - "(m.content LIKE ? ESCAPE '\\' OR m.tool_name LIKE ? ESCAPE '\\' OR m.tool_calls LIKE ? ESCAPE '\\')" - ) - like_params += [f"%{esc}%", f"%{esc}%", f"%{esc}%"] - like_where = [f"({' OR '.join(token_clauses)})"] - if not include_inactive: - # Same visibility rule as the FTS5 paths: live rows and - # compaction-archived rows are discoverable; rewind/undo - # rows (active=0, compacted=0) are hidden (#38763). - like_where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - like_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - like_params.extend(source_filter) - if exclude_sources is not None: - like_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") - like_params.extend(exclude_sources) - if role_filter: - like_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - like_params.extend(role_filter) - like_sql = f""" - SELECT m.id, m.session_id, m.role, - substr(m.content, - max(1, instr(m.content, ?) - 40), - 120) AS snippet, - m.content, m.timestamp, m.tool_name, - s.source, s.model, s.started_at AS session_started - FROM messages m - JOIN sessions s ON s.id = m.session_id - WHERE {' AND '.join(like_where)} - ORDER BY m.timestamp DESC - LIMIT ? OFFSET ? - """ - like_params.extend([limit, offset]) - # instr() for snippet uses first search token - like_params = [non_op_tokens[0]] + like_params - with self._read_ctx() as conn: - like_cursor = conn.execute(like_sql, like_params) - matches = [dict(row) for row in like_cursor.fetchall()] - else: - try: - with self._read_ctx() as conn: - cursor = conn.execute(sql, params) - matches = [dict(row) for row in cursor.fetchall()] - except sqlite3.OperationalError: - # FTS5 query syntax error despite sanitization — return empty - return [] - except sqlite3.DatabaseError as exc: - # A corrupt FTS index raises the malformed / "fts5: corrupt - # structure record" class on the MATCH read, the same class the - # write path self-heals (#66296). OperationalError (query - # syntax) is a subclass caught above; this arm is the corruption - # parent. Rebuild the index in place once — the read context - # holds no writer lock, so rebuild_fts() can acquire it — and - # retry, so search self-heals for read-only sessions (cron/CLI - # history search) that never trigger a write to repair it first. - if not self._try_runtime_fts_rebuild(exc): - raise - with self._read_ctx() as conn: - cursor = conn.execute(sql, params) - matches = [dict(row) for row in cursor.fetchall()] - - # Deferred-rebuild supplement (schema v23): while the background - # backfill is pending, the FTS indexes only cover rows outside the - # (progress, high_water] gap. Top the results up with a bounded LIKE - # scan over just that id range so search never silently loses old - # messages mid-rebuild. The range shrinks as the backfill advances, - # so this cost decays to zero. The CJK LIKE-fallback path above - # already scans the whole base table and needs no supplement. - rebuild_status = self.fts_rebuild_status() - if rebuild_status is not None and len(matches) < limit: - try: - gap_matches = self._search_unindexed_gap( - query, - limit - len(matches), - include_inactive=include_inactive, - source_filter=source_filter, - exclude_sources=exclude_sources, - role_filter=role_filter, - ) - seen_ids = {m["id"] for m in matches} - matches.extend(m for m in gap_matches if m["id"] not in seen_ids) - except sqlite3.OperationalError as exc: - logger.debug("Unindexed-gap supplement skipped: %s", exc) - - # Pure-Latin queries run against the unicode61 ``messages_fts`` table, - # whose tokenizer does not insert a boundary between Latin letters and - # adjacent CJK characters: "修改youer服务端" is indexed as one token, - # so MATCH "youer" finds nothing even though the substring is present - # (#54242). When the exact-token search returns nothing, retry on the - # substring-capable indexes. Preference order: - # 1. messages_fts_cjk (when built): its tokenizer splits Latin runs - # off adjacent CJK, so "youer" is an exact ranked token match. - # 2. messages_fts_trigram: substring matching, needs >=3-char - # tokens (shorter tokens produce no trigrams). - # Gated on a zero-result miss so successful Latin searches keep their - # unicode61 ranking — strictly additive, never reorders existing - # hits. Trade-off on the trigram leg: any zero-result Latin query - # gains substring semantics (e.g. "cat" can then match - # "concatenate"). Genuinely absent terms still return []. Skipped for - # role_filter=['tool'] queries — both fallback indexes exclude tool - # rows (v23), so a retry could never add hits. - if ( - not matches - and not is_cjk - and not (bool(role_filter) and "tool" in role_filter) - ): - _fb_query = query.strip('"').strip() - if self._fts_cjk_available: - cjk_fb = self._run_trigram_search( - _fb_query, - table="messages_fts_cjk", - order_by_sql=order_by_sql, - include_inactive=include_inactive, - source_filter=source_filter, - exclude_sources=exclude_sources, - role_filter=role_filter, - limit=limit, - offset=offset, - ) - if cjk_fb: - matches = cjk_fb - if ( - not matches - and self._trigram_available - and self._trigram_eligible_tokens(query) - ): - tri_matches = self._run_trigram_search( - _fb_query, - order_by_sql=order_by_sql, - include_inactive=include_inactive, - source_filter=source_filter, - exclude_sources=exclude_sources, - role_filter=role_filter, - limit=limit, - offset=offset, - ) - if tri_matches: - matches = tri_matches - - # Add surrounding context (1 message before + after each match). - # Each query takes its own fresh read transaction via _read_ctx, so - # we never hold a lock across N sequential queries. - for match in matches: - try: - with self._read_ctx() as conn: - ctx_cursor = conn.execute( - """WITH target AS ( - SELECT session_id, timestamp, id - FROM messages - WHERE id = ? - ) - SELECT role, content - FROM ( - SELECT m.id, m.timestamp, m.role, m.content - FROM messages m - JOIN target t ON t.session_id = m.session_id - WHERE (m.timestamp < t.timestamp) - OR (m.timestamp = t.timestamp AND m.id < t.id) - ORDER BY m.timestamp DESC, m.id DESC - LIMIT 1 - ) - UNION ALL - SELECT role, content - FROM messages - WHERE id = ? - UNION ALL - SELECT role, content - FROM ( - SELECT m.id, m.timestamp, m.role, m.content - FROM messages m - JOIN target t ON t.session_id = m.session_id - WHERE (m.timestamp > t.timestamp) - OR (m.timestamp = t.timestamp AND m.id > t.id) - ORDER BY m.timestamp ASC, m.id ASC - LIMIT 1 - )""", - (match["id"], match["id"]), - ) - context_msgs = [] - for r in ctx_cursor.fetchall(): - raw = r["content"] - decoded = self._decode_content(raw) - # Multimodal context: render a compact text-only - # summary for search previews. - if isinstance(decoded, list): - text_parts = [ - p.get("text", "") for p in decoded - if isinstance(p, dict) and p.get("type") == "text" - ] - text = " ".join(t for t in text_parts if t).strip() - preview = text or "[multimodal content]" - elif isinstance(decoded, str): - preview = decoded - else: - preview = "" - context_msgs.append( - {"role": r["role"], "content": preview[:200]} - ) - match["context"] = context_msgs - except Exception: - match["context"] = [] - - # Remove full content from result (snippet is enough, saves tokens) - for match in matches: - match.pop("content", None) - - return matches - - def _search_unindexed_gap( - self, - fts_query: str, - limit: int, - *, - include_inactive: bool = False, - source_filter: Optional[List[str]] = None, - exclude_sources: Optional[List[str]] = None, - role_filter: Optional[List[str]] = None, - ) -> List[Dict[str, Any]]: - """LIKE-scan the rows the deferred rebuild hasn't indexed yet. - - Only touches ids in (fts_rebuild_progress, fts_rebuild_high_water] — - a range that shrinks to nothing as the backfill advances. The FTS - query is degraded to per-token substring terms (AND-joined; quoted - phrases kept whole), which is deliberately recall-over-precision: - temporary results beat silently missing ones mid-rebuild. - """ - status = self.fts_rebuild_status() - if status is None or limit <= 0: - return [] - progress, high_water = status["indexed"], status["total"] - - # Degrade the FTS query to LIKE terms: strip operators/wildcards, - # keep quoted phrases intact, AND the rest. - terms: List[str] = [] - for raw_tok in re.findall(r'"[^"]+"|\S+', fts_query): - tok = raw_tok.strip('"').strip("*").strip() - if not tok or tok.upper() in {"AND", "OR", "NOT", "NEAR"}: - continue - terms.append(tok) - if not terms: - return [] - - where = ["m.id > ? AND m.id <= ?"] - params: list = [progress, high_water] - for term in terms: - esc = term.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - where.append( - "(m.content LIKE ? ESCAPE '\\' OR m.tool_name LIKE ? ESCAPE '\\' " - "OR m.tool_calls LIKE ? ESCAPE '\\')" - ) - params += [f"%{esc}%"] * 3 - if not include_inactive: - where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - params.extend(source_filter) - if exclude_sources is not None: - where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") - params.extend(exclude_sources) - if role_filter: - where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - params.extend(role_filter) - - sql = f""" - SELECT m.id, m.session_id, m.role, - substr(m.content, - max(1, instr(m.content, ?) - 40), - 120) AS snippet, - m.content, m.timestamp, m.tool_name, - s.source, s.model, s.started_at AS session_started - FROM messages m - JOIN sessions s ON s.id = m.session_id - WHERE {' AND '.join(where)} - ORDER BY m.timestamp DESC - LIMIT ? - """ - params = [terms[0]] + params + [limit] - with self._read_ctx() as conn: - rows = conn.execute(sql, params).fetchall() - return [dict(r) for r in rows] - - def search_sessions_by_id( - self, - query: str, - limit: int = 20, - include_archived: bool = True, - source: str = None, - sources: List[str] = None, - exclude_sources: List[str] = None, - ) -> List[Dict[str, Any]]: - """Search surfaced sessions by exact/prefix/substring session id. - - Desktop search uses this alongside FTS message search so users can paste - a session id from logs, CLI output, or another Hermes surface and jump - straight to that conversation. Matching also checks ``_lineage_root_id`` - for projected compression-chain tips, so an old root id still resolves to - the live continuation row. - """ - needle = (query or "").strip().lower() - if not needle or limit <= 0: - return [] - - # SQL-bounded: list_sessions_rich pushes the id LIKE filter into the - # query (matching the row's own id AND any id in its forward - # compression chain), so we only materialize matching rows instead of - # scanning every session. Fetch a small multiple of `limit` so the - # in-Python exact/prefix/substring ranking below has enough candidates - # to order, then truncate. - candidates = self.list_sessions_rich( - source=source, - sources=sources, - exclude_sources=exclude_sources, - limit=max(limit * 4, limit), - offset=0, - include_archived=include_archived, - order_by_last_active=True, - id_query=needle, - ) - - def score(row: Dict[str, Any]) -> int: - ids = [str(row.get("id") or ""), str(row.get("_lineage_root_id") or "")] - normalized = [value.lower() for value in ids if value] - if any(value == needle for value in normalized): - return 0 - if any(value.startswith(needle) for value in normalized): - return 1 - return 2 - - ranked = sorted( - enumerate(candidates), - key=lambda item: (score(item[1]), item[0]), - ) - return [row for _, row in ranked[:limit]] - def search_sessions( self, source: str = None, @@ -9554,443 +6441,6 @@ class SessionDB: continue return lineage if session_id in lineage else [session_id] - def export_session(self, session_id: str) -> Optional[Dict[str, Any]]: - """Export a single session with all its messages as a dict.""" - session = self.get_session(session_id) - if not session: - return None - messages = self.get_messages(session_id) - return {**session, "messages": messages} - - def export_session_lineage(self, session_id: str) -> Optional[Dict[str, Any]]: - """Export a compression lineage as one logical session dict.""" - lineage_ids = self.get_compression_lineage(session_id) - if not lineage_ids: - return None - segments = [] - for sid in lineage_ids: - segment = self.export_session(sid) - if segment: - segments.append(segment) - if not segments: - return None - base = dict(segments[-1]) - total_messages = sum(len(seg.get("messages") or []) for seg in segments) - base["segments"] = segments - base["lineage_session_ids"] = [seg["id"] for seg in segments] - base["message_count"] = total_messages - base["messages"] = [msg for seg in segments for msg in (seg.get("messages") or [])] - return base - - def export_all(self, source: str = None) -> List[Dict[str, Any]]: - """ - Export all sessions (with messages) as a list of dicts. - Suitable for writing to a JSONL file for backup/analysis. - """ - sessions = self.search_sessions(source=source, limit=100000) - results = [] - for session in sessions: - messages = self.get_messages(session["id"]) - results.append({**session, "messages": messages}) - return results - - @staticmethod - def _import_text_or_none(value: Any, field: str) -> Optional[str]: - if value is None: - return None - if isinstance(value, str): - return value - raise ValueError(f"{field} must be a string") - - @staticmethod - def _import_json_object_or_none(value: Any, field: str) -> Optional[str]: - if value is None: - return None - if isinstance(value, str): - try: - parsed = json.loads(value) - except json.JSONDecodeError as exc: - raise ValueError(f"{field} must be valid JSON") from exc - if not isinstance(parsed, dict): - raise ValueError(f"{field} must be a JSON object") - return value - if not isinstance(value, dict): - raise ValueError(f"{field} must be a JSON object") - try: - return json.dumps(value) - except (TypeError, ValueError) as exc: - raise ValueError(f"{field} must be JSON serializable") from exc - - @staticmethod - def _float_or_none(value: Any) -> Optional[float]: - if value is None: - return None - try: - return float(value) - except (TypeError, ValueError): - return None - - @staticmethod - def _import_int_or_none(value: Any, field: str) -> Optional[int]: - if value is None: - return None - try: - return int(value) - except (TypeError, ValueError) as exc: - raise ValueError(f"{field} must be an integer") from exc - - @staticmethod - def _int_or_default(value: Any, default: int = 0) -> int: - if value is None: - return default - try: - return int(value) - except (TypeError, ValueError): - return default - - @staticmethod - def _reasoning_json_value(value: Any) -> Any: - if not isinstance(value, str): - return value - try: - return json.loads(value) - except (json.JSONDecodeError, TypeError): - return value - - @staticmethod - def _import_error(index: int, session_id: str, error: str) -> Dict[str, Any]: - item: Dict[str, Any] = {"index": index, "error": error} - if session_id: - item["session_id"] = session_id - return item - - def import_sessions(self, sessions: List[Dict[str, Any]]) -> Dict[str, Any]: - """Import sessions exported by :meth:`export_session` or ``export_all``. - - Existing session IDs are skipped. Imported child sessions keep their - parent only when that parent already exists or is included in the same - import payload; otherwise the child is detached so partial imports don't - fail foreign-key validation. Gateway routing, handoff, rewind, and other - live runtime state are intentionally reset: this restores conversation - history, not ownership of a live channel or process. - """ - if not isinstance(sessions, list): - raise ValueError("sessions must be a list") - if len(sessions) > self._IMPORT_MAX_SESSIONS: - raise ValueError( - f"sessions must contain at most {self._IMPORT_MAX_SESSIONS} entries" - ) - - normalized: List[Dict[str, Any]] = [] - errors: List[Dict[str, Any]] = [] - seen_ids: set[str] = set() - total_messages = 0 - total_bytes = 0 - session_text_fields = ( - "source", - "user_id", - "model", - "system_prompt", - "end_reason", - "cwd", - "git_branch", - "git_repo_root", - "billing_provider", - "billing_base_url", - "billing_mode", - "cost_status", - "cost_source", - "pricing_version", - "title", - ) - message_text_fields = ( - "role", - "tool_call_id", - "tool_name", - "effect_disposition", - "finish_reason", - "reasoning", - "reasoning_content", - "platform_message_id", - "message_id", - ) - - for index, raw in enumerate(sessions): - if not isinstance(raw, dict): - errors.append(self._import_error(index, "", "session must be an object")) - continue - session_id = str(raw.get("id") or "").strip() - if not session_id: - errors.append(self._import_error(index, "", "session id is required")) - continue - if session_id in seen_ids: - errors.append(self._import_error(index, session_id, "duplicate session id")) - continue - messages = raw.get("messages") or [] - if not isinstance(messages, list): - errors.append(self._import_error(index, session_id, "messages must be a list")) - continue - if len(messages) > self._IMPORT_MAX_MESSAGES_PER_SESSION: - errors.append( - self._import_error( - index, - session_id, - "messages exceeds the per-session import limit", - ) - ) - continue - if any(not isinstance(msg, dict) for msg in messages): - errors.append( - self._import_error( - index, - session_id, - "messages must contain only objects", - ) - ) - continue - - try: - session_bytes = len( - json.dumps(raw, ensure_ascii=False, separators=(",", ":")).encode("utf-8") - ) - except (TypeError, ValueError): - errors.append( - self._import_error(index, session_id, "session must be JSON serializable") - ) - continue - if session_bytes > self._IMPORT_MAX_SESSION_BYTES: - errors.append( - self._import_error(index, session_id, "session exceeds the import size limit") - ) - continue - total_bytes += session_bytes - if total_bytes > self._IMPORT_MAX_TOTAL_BYTES: - errors.append( - self._import_error(index, session_id, "import exceeds the total size limit") - ) - continue - - try: - clean_session = dict(raw) - clean_session["id"] = session_id - clean_session["model_config"] = self._import_json_object_or_none( - clean_session.get("model_config"), "model_config" - ) - clean_session["parent_session_id"] = self._import_text_or_none( - clean_session.get("parent_session_id"), "parent_session_id" - ) - for field in session_text_fields: - clean_session[field] = self._import_text_or_none( - clean_session.get(field), field - ) - - clean_messages: List[Dict[str, Any]] = [] - for message_index, message in enumerate(messages): - clean_message = dict(message) - role = clean_message.get("role") - if not isinstance(role, str) or not role: - raise ValueError(f"messages[{message_index}].role must be a non-empty string") - for field in message_text_fields: - if field == "role": - continue - clean_message[field] = self._import_text_or_none( - clean_message.get(field), field - ) - clean_message["token_count"] = self._import_int_or_none( - clean_message.get("token_count"), "token_count" - ) - clean_messages.append(clean_message) - except ValueError as exc: - errors.append(self._import_error(index, session_id, str(exc))) - continue - - total_messages += len(clean_messages) - if total_messages > self._IMPORT_MAX_TOTAL_MESSAGES: - errors.append( - self._import_error( - index, - session_id, - "messages exceeds the total import limit", - ) - ) - continue - seen_ids.add(session_id) - normalized.append( - {"index": index, "session": clean_session, "messages": clean_messages} - ) - - if errors: - return { - "ok": False, - "imported": 0, - "skipped": 0, - "detached": 0, - "errors": errors, - } - - def _do(conn): - imported_ids: List[str] = [] - skipped_ids: List[str] = [] - parent_updates: List[tuple[str, str]] = [] - detached = 0 - - for item in normalized: - raw = item["session"] - messages = item["messages"] - session_id = str(raw.get("id") or "").strip() - exists = conn.execute( - "SELECT 1 FROM sessions WHERE id = ? LIMIT 1", - (session_id,), - ).fetchone() - if exists: - skipped_ids.append(session_id) - continue - - started_at = self._float_or_none(raw.get("started_at")) - if started_at is None: - started_at = time.time() - archived = 1 if raw.get("archived") else 0 - - conn.execute( - """INSERT INTO sessions ( - id, source, user_id, model, model_config, system_prompt, - parent_session_id, started_at, ended_at, end_reason, - message_count, tool_call_count, input_tokens, output_tokens, - cache_read_tokens, cache_write_tokens, reasoning_tokens, - cwd, git_branch, git_repo_root, - billing_provider, billing_base_url, billing_mode, - estimated_cost_usd, actual_cost_usd, cost_status, cost_source, - pricing_version, title, api_call_count, archived - ) - VALUES ( - :id, :source, :user_id, :model, :model_config, - :system_prompt, NULL, :started_at, :ended_at, - :end_reason, 0, 0, :input_tokens, :output_tokens, - :cache_read_tokens, :cache_write_tokens, - :reasoning_tokens, :cwd, :git_branch, :git_repo_root, - :billing_provider, :billing_base_url, :billing_mode, - :estimated_cost_usd, :actual_cost_usd, :cost_status, - :cost_source, :pricing_version, :title, - :api_call_count, :archived - )""", - { - "id": session_id, - "source": str(raw.get("source") or "import"), - "user_id": raw.get("user_id"), - "model": raw.get("model"), - "model_config": raw.get("model_config"), - "system_prompt": raw.get("system_prompt"), - "started_at": started_at, - "ended_at": self._float_or_none(raw.get("ended_at")), - "end_reason": raw.get("end_reason"), - "input_tokens": self._int_or_default(raw.get("input_tokens")), - "output_tokens": self._int_or_default(raw.get("output_tokens")), - "cache_read_tokens": self._int_or_default( - raw.get("cache_read_tokens") - ), - "cache_write_tokens": self._int_or_default( - raw.get("cache_write_tokens") - ), - "reasoning_tokens": self._int_or_default( - raw.get("reasoning_tokens") - ), - "cwd": raw.get("cwd"), - "git_branch": raw.get("git_branch"), - "git_repo_root": raw.get("git_repo_root"), - "billing_provider": raw.get("billing_provider"), - "billing_base_url": raw.get("billing_base_url"), - "billing_mode": raw.get("billing_mode"), - "estimated_cost_usd": self._float_or_none( - raw.get("estimated_cost_usd") - ), - "actual_cost_usd": self._float_or_none( - raw.get("actual_cost_usd") - ), - "cost_status": raw.get("cost_status"), - "cost_source": raw.get("cost_source"), - "pricing_version": raw.get("pricing_version"), - "title": raw.get("title"), - "api_call_count": self._int_or_default(raw.get("api_call_count")), - "archived": archived, - }, - ) - - sanitized_messages: List[Dict[str, Any]] = [] - for msg in messages: - clean = dict(msg) - for key in ( - "reasoning_details", - "codex_reasoning_items", - "codex_message_items", - ): - clean[key] = self._reasoning_json_value(clean.get(key)) - sanitized_messages.append(clean) - - total_messages, total_tool_calls = self._insert_message_rows( - conn, - session_id, - sanitized_messages, - ) - conn.execute( - "UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?", - (total_messages, total_tool_calls, session_id), - ) - - parent_id = str(raw.get("parent_session_id") or "").strip() - if parent_id: - parent_updates.append((session_id, parent_id)) - imported_ids.append(session_id) - - parent_by_child = dict(parent_updates) - - def _would_create_cycle(session_id: str, parent_id: str) -> bool: - seen = {session_id} - current = parent_id - while current: - if current in seen: - return True - seen.add(current) - if current in parent_by_child: - current = parent_by_child[current] - continue - row = conn.execute( - "SELECT parent_session_id FROM sessions WHERE id = ? LIMIT 1", - (current,), - ).fetchone() - if row is None: - return False - current = row["parent_session_id"] - return False - - for session_id, parent_id in parent_updates: - parent_exists = conn.execute( - "SELECT 1 FROM sessions WHERE id = ? LIMIT 1", - (parent_id,), - ).fetchone() - if parent_exists and not _would_create_cycle(session_id, parent_id): - conn.execute( - "UPDATE sessions SET parent_session_id = ? WHERE id = ?", - (parent_id, session_id), - ) - else: - # Drop only the closing edge. Later entries can still attach - # to this now-root session, preserving the acyclic portion - # of a malformed imported lineage. - parent_by_child.pop(session_id, None) - detached += 1 - - return { - "ok": True, - "imported": len(imported_ids), - "skipped": len(skipped_ids), - "detached": detached, - "imported_ids": imported_ids, - "skipped_ids": skipped_ids, - "errors": [], - } - - return self._execute_write(_do) - def clear_messages(self, session_id: str) -> None: """Delete all messages for a session and reset its counters.""" def _do(conn): @@ -11217,88 +7667,6 @@ class SessionDB: # is present — so we probe each before touching it (see optimize_fts). _FTS_TABLES = ("messages_fts", "messages_fts_trigram", "messages_fts_cjk") - def _fts_table_exists(self, name: str) -> bool: - """True if an FTS5 virtual table is queryable in this DB.""" - try: - self._conn.execute(f"SELECT 1 FROM {name} LIMIT 0") - return True - except sqlite3.DatabaseError: - # OperationalError ("no such table") or the broader - # DatabaseError class ("vtable constructor failed", raised when - # e.g. a required tokenizer is missing or the table is mid- - # teardown) — in every case the table is not queryable. - return False - - def optimize_fts(self) -> int: - """Merge fragmented FTS5 b-tree segments into one per index. - - FTS5 indexes grow as a series of incremental segments — one per - ``INSERT`` batch driven by the message triggers. Over tens of - thousands of messages these segments accumulate, which both bloats - the ``*_data`` shadow tables and slows ``MATCH`` queries that must - scan every segment. The special ``'optimize'`` command rewrites each - index as a single merged segment. - - This is purely a maintenance operation — it changes neither search - results nor ``snippet()`` output, only on-disk layout and query - speed. It is complementary to VACUUM: ``optimize`` compacts the FTS - index internally, then VACUUM returns the freed pages to the OS. - - Skips any FTS table that does not exist (e.g. the trigram index when - disabled via ``HERMES_DISABLE_FTS_TRIGRAM`` or not yet created), so - it is safe to call unconditionally. - - Returns the number of FTS indexes that were optimized. - """ - optimized = 0 - with self._lock: - for tbl in self._FTS_TABLES: - if not self._fts_table_exists(tbl): - continue - try: - # The column name in the INSERT must match the table name - # for FTS5 special commands. - self._conn.execute( - f"INSERT INTO {tbl}({tbl}) VALUES('optimize')" - ) - optimized += 1 - except sqlite3.OperationalError as exc: - logger.warning( - "FTS optimize failed for %s: %s", tbl, exc - ) - return optimized - - def rebuild_fts(self) -> int: - """Rebuild FTS5 indexes from the canonical ``messages`` table. - - Uses the FTS5 ``'rebuild'`` command, which rewrites the internal - b-tree segments from the content rows. This is the documented - recovery for a corrupt FTS index that rejects message writes while - reads still succeed (issue #50502). Unlike ``optimize_fts`` (which - merges existing segments), ``rebuild`` discards and recreates the - index data entirely. - - Safe to call when FTS tables don't exist (skips them). - Returns the number of FTS indexes that were rebuilt. - """ - rebuilt = 0 - with self._lock: - for tbl in self._FTS_TABLES: - if not self._fts_table_exists(tbl): - continue - try: - self._conn.execute( - f"INSERT INTO {tbl}({tbl}) VALUES('rebuild')" - ) - self._conn.commit() - rebuilt += 1 - except sqlite3.OperationalError as exc: - self._conn.rollback() - logger.warning( - "FTS rebuild failed for %s: %s", tbl, exc - ) - return rebuilt - def logical_size_bytes(self) -> Optional[int]: """Database size in bytes as SQLite itself accounts for it. @@ -11327,77 +7695,6 @@ class SessionDB: logger.debug("Could not read logical DB size: %s", exc) return None - def _merge_fts_incrementally( - self, *, max_pages: int, max_commands: Optional[int] = None - ) -> int: - """Run bounded FTS5 ``'merge'`` commands against each present index. - - A positive merge rank tells SQLite to stop after approximately that - many output pages, so each command holds the write lock for - milliseconds regardless of index size — unlike ``'optimize'``, which - rewrites the whole index in one transaction (measured 9-18 s per - index on a 10 GB production DB, long enough to exhaust a competing - writer's entire lock-retry patience). - - Protocol (SQLite FTS5 §6.8-6.9): - - - ``usermerge`` is lowered to its minimum of 2 (persisted in the - ``%_config`` shadow table, applied once per instance) so a - positive merge acts on ANY level holding >= 2 segments. With the - default of 4, levels below that threshold are never merged by a - positive-rank command and a fragmented index cannot converge. - - Up to *max_commands* merge commands run per index, stopping early - on the documented no-progress signal: the delta in - ``total_changes`` is < 2 (the command's own INSERT accounts - for 1 change; >= 2 means real merge work happened). - - Each command is its own implicit transaction (the connection runs - with ``isolation_level=None``), so the SQLite write lock is released - between commands and competing processes can interleave writes - mid-pass. Missing tables are valid schema variants (FTS variants are - optional, and ``optimize_fts_storage`` legitimately drops + backfills - these tables while writers keep running) and are skipped, mirroring - ``optimize_fts``. Other SQLite errors propagate to the caller. - - Returns the number of merge commands executed. - """ - if isinstance(max_pages, bool) or not isinstance(max_pages, int): - raise TypeError("max_pages must be an integer") - if max_pages <= 0: - raise ValueError("max_pages must be greater than zero") - if max_commands is None: - max_commands = self._FTS_MERGE_COMMANDS_PER_PASS - if isinstance(max_commands, bool) or not isinstance(max_commands, int): - raise TypeError("max_commands must be an integer") - if max_commands <= 0: - raise ValueError("max_commands must be greater than zero") - - executed = 0 - with self._lock: - for tbl in self._FTS_TABLES: - if not self._fts_table_exists(tbl): - continue - # One-time (per instance) usermerge floor; the value is - # persisted in the index's config shadow table so future - # connections inherit it. Setting config is a metadata-only - # write — it never touches segment data. - if not getattr(self, "_fts_usermerge_floor_applied", False): - self._conn.execute( - f"INSERT INTO {tbl}({tbl}, rank) " - "VALUES('usermerge', 2)" - ) - for _ in range(max_commands): - before = self._conn.total_changes - self._conn.execute( - f"INSERT INTO {tbl}({tbl}, rank) VALUES('merge', ?)", - (max_pages,), - ) - executed += 1 - if self._conn.total_changes - before < 2: - break - self._fts_usermerge_floor_applied = True - return executed - def vacuum(self) -> int: """Run VACUUM to reclaim disk space after large deletes. diff --git a/hermes_state_common.py b/hermes_state_common.py new file mode 100644 index 00000000000..19dbaa39d78 --- /dev/null +++ b/hermes_state_common.py @@ -0,0 +1,523 @@ +"""Shared module-level constants for the SessionDB family of modules. + +Extracted verbatim from hermes_state.py so the SessionDB mixin modules +(hermes_state_search / hermes_state_schema / hermes_state_portability) can +reference them without importing hermes_state (which would be a cycle). +hermes_state re-imports every name here for backward compatibility. +""" + +from typing import Any + +from agent.skill_commands import ( + SKILL_EXCERPT_JOINT, + SKILL_SCAFFOLD_SQL_LIKE, + describe_skill_invocation, +) + + +# Session preview = the head of the first user message, shown wherever a +# session has no title (sidebar rows, pickers, exports, the desktop's +# `sessionTitle` fallback). +# +# A /skill invocation expands into a message that embeds the whole skill body, +# so the plain head of it previews the SKILL's opening prose as if the user had +# written it. Scaffolded rows therefore carry a wider excerpt so +# ``_shape_preview`` can hand it to ``describe_skill_invocation`` and recover +# ``/work — fix the title leak``: the whole message while it stays under the +# budget, and head + tail (where the typed instruction lands) once it doesn't. +_PREVIEW_HEAD_CHARS = 63 + + +_PREVIEW_SCAFFOLD_WINDOW = 400 + + +_PREVIEW_MAX_CHARS = 60 + + +_PREVIEW_CONTENT_SQL = "REPLACE(REPLACE(m.content, X'0A', ' '), X'0D', ' ')" + + +_PREVIEW_SCAFFOLDED_SQL = f"m.content LIKE '{SKILL_SCAFFOLD_SQL_LIKE}'" + + +# The shared ``_preview_raw`` SELECT expression, interpolated by every listing +# query. A scaffolded row gets a wider excerpt: the whole message while it fits +# the budget, else head + tail (where the typed instruction lands) spliced +# around SKILL_EXCERPT_JOINT. +_PREVIEW_RAW_SELECT = ( + f"CASE WHEN {_PREVIEW_SCAFFOLDED_SQL}" + f" AND LENGTH(m.content) > {_PREVIEW_SCAFFOLD_WINDOW * 2}" + f" THEN SUBSTR({_PREVIEW_CONTENT_SQL}, 1, {_PREVIEW_SCAFFOLD_WINDOW})" + f" || '{SKILL_EXCERPT_JOINT}'" + f" || SUBSTR({_PREVIEW_CONTENT_SQL}, -{_PREVIEW_SCAFFOLD_WINDOW})" + f" WHEN {_PREVIEW_SCAFFOLDED_SQL}" + f" THEN SUBSTR({_PREVIEW_CONTENT_SQL}, 1, {_PREVIEW_SCAFFOLD_WINDOW * 2})" + f" ELSE SUBSTR({_PREVIEW_CONTENT_SQL}, 1, {_PREVIEW_HEAD_CHARS}) END" +) + + +def _shape_preview(raw: Any) -> str: + """Turn a ``_preview_raw`` column into the short preview callers show.""" + text = str(raw or "").strip() + if not text: + return "" + described = describe_skill_invocation(text) + text = described if described is not None else text.split(SKILL_EXCERPT_JOINT)[0] + if len(text) > _PREVIEW_MAX_CHARS: + return text[:_PREVIEW_MAX_CHARS] + "..." + return text + + +# A child session counts as a /branch (kept visible, never cascade-deleted) if +# it carries the stable marker OR the legacy end_reason heuristic holds. +_BRANCH_CHILD_SQL = ( + "json_extract(COALESCE({a}.model_config, '{{}}'), '$._branched_from') IS NOT NULL" + " OR EXISTS (SELECT 1 FROM sessions p" + " WHERE p.id = {a}.parent_session_id" + " AND p.end_reason = 'branched'" + " AND {a}.started_at >= p.ended_at)" +) + + +_COMPRESSION_CHILD_SQL = ( + "EXISTS (SELECT 1 FROM sessions p" + " WHERE p.id = {a}.parent_session_id" + " AND p.end_reason = 'compression')" +) + + +# Rows that surface in pickers: roots + branch children (subagent runs and +# compression continuations stay hidden). +_LISTABLE_CHILD_SQL = f"(s.parent_session_id IS NULL OR {_BRANCH_CHILD_SQL.format(a='s')})" + + +def _ephemeral_child_sql(alias: str = "s") -> str: + """Subagent runs (cascade-delete targets), not branches or compression tips.""" + branch = _BRANCH_CHILD_SQL.format(a=alias) + compression = _COMPRESSION_CHILD_SQL.format(a=alias) + return ( + f"({alias}.parent_session_id IS NOT NULL" + f" AND NOT ({branch})" + f" AND NOT ({compression}))" + ) + + +SCHEMA_VERSION = 23 + + +# FTS storage-layout version, tracked INDEPENDENTLY of SCHEMA_VERSION in the +# state_meta key ``fts_storage_version``. The main schema version advances +# freely on open (so future migrations always land); the FTS *layout* only +# reaches the current version when a DB is either born fresh or explicitly +# optimized via ``hermes sessions optimize-storage``. A legacy DB sits at +# layout 0 (marker absent) with a working inline index until the user opts in. +# 1 = v23 external-content layout (content/tool_name/tool_calls, +# tool-row-excluded trigram) +FTS_STORAGE_VERSION = 1 + + +# Cap on user-controlled FTS5 query input before regex/sanitizer processing. +# Search queries do not need to be arbitrarily large, and bounding them keeps +# sanitizer/runtime behavior predictable under adversarial input. +MAX_FTS5_QUERY_CHARS = 2_048 + + +_FTS_TRIGGERS = ( + "messages_fts_insert", + "messages_fts_delete", + "messages_fts_update", + "messages_fts_trigram_insert", + "messages_fts_trigram_delete", + "messages_fts_trigram_update", +) + + +SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS schema_version ( + version INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS sessions ( + id TEXT PRIMARY KEY, + source TEXT NOT NULL, + user_id TEXT, + session_key TEXT, + chat_id TEXT, + chat_type TEXT, + thread_id TEXT, + display_name TEXT, + origin_json TEXT, + expiry_finalized INTEGER DEFAULT 0, + model TEXT, + model_config TEXT, + system_prompt TEXT, + parent_session_id TEXT, + started_at REAL NOT NULL, + ended_at REAL, + end_reason TEXT, + message_count INTEGER DEFAULT 0, + tool_call_count INTEGER DEFAULT 0, + input_tokens INTEGER DEFAULT 0, + output_tokens INTEGER DEFAULT 0, + cache_read_tokens INTEGER DEFAULT 0, + cache_write_tokens INTEGER DEFAULT 0, + reasoning_tokens INTEGER DEFAULT 0, + cwd TEXT, + git_branch TEXT, + git_repo_root TEXT, + billing_provider TEXT, + billing_base_url TEXT, + billing_mode TEXT, + estimated_cost_usd REAL, + actual_cost_usd REAL, + cost_status TEXT, + cost_source TEXT, + pricing_version TEXT, + title TEXT, + api_call_count INTEGER DEFAULT 0, + handoff_state TEXT, + handoff_platform TEXT, + handoff_error TEXT, + compression_failure_cooldown_until REAL, + compression_failure_error TEXT, + compression_fallback_streak INTEGER NOT NULL DEFAULT 0, + compression_ineffective_count INTEGER NOT NULL DEFAULT 0, + profile_name TEXT, + rewind_count INTEGER NOT NULL DEFAULT 0, + archived INTEGER NOT NULL DEFAULT 0, + pinned INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY (parent_session_id) REFERENCES sessions(id) +); + +CREATE TABLE IF NOT EXISTS messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL REFERENCES sessions(id), + role TEXT NOT NULL, + content TEXT, + tool_call_id TEXT, + tool_calls TEXT, + tool_name TEXT, + effect_disposition TEXT, + timestamp REAL NOT NULL, + token_count INTEGER, + finish_reason TEXT, + reasoning TEXT, + reasoning_content TEXT, + reasoning_details TEXT, + codex_reasoning_items TEXT, + codex_message_items TEXT, + platform_message_id TEXT, + observed INTEGER DEFAULT 0, + active INTEGER NOT NULL DEFAULT 1, + compacted INTEGER NOT NULL DEFAULT 0, + api_content TEXT, + display_kind TEXT, + display_metadata TEXT +); + +CREATE TABLE IF NOT EXISTS session_model_usage ( + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + model TEXT NOT NULL, + billing_provider TEXT NOT NULL DEFAULT '', + billing_base_url TEXT NOT NULL DEFAULT '', + billing_mode TEXT NOT NULL DEFAULT '', + task TEXT NOT NULL DEFAULT '', + api_call_count INTEGER NOT NULL DEFAULT 0, + input_tokens INTEGER NOT NULL DEFAULT 0, + output_tokens INTEGER NOT NULL DEFAULT 0, + cache_read_tokens INTEGER NOT NULL DEFAULT 0, + cache_write_tokens INTEGER NOT NULL DEFAULT 0, + reasoning_tokens INTEGER NOT NULL DEFAULT 0, + estimated_cost_usd REAL NOT NULL DEFAULT 0, + actual_cost_usd REAL NOT NULL DEFAULT 0, + cost_status TEXT, + cost_source TEXT, + first_seen REAL, + last_seen REAL, + PRIMARY KEY (session_id, model, billing_provider, billing_base_url, billing_mode, task) +); + +CREATE TABLE IF NOT EXISTS state_meta ( + key TEXT PRIMARY KEY, + value TEXT +); + +CREATE TABLE IF NOT EXISTS gateway_routing ( + scope TEXT NOT NULL DEFAULT '', + session_key TEXT NOT NULL, + entry_json TEXT NOT NULL, + updated_at REAL NOT NULL, + PRIMARY KEY (scope, session_key) +); + +CREATE TABLE IF NOT EXISTS compression_locks ( + session_id TEXT PRIMARY KEY, + holder TEXT NOT NULL, + acquired_at REAL NOT NULL, + expires_at REAL NOT NULL +); + +CREATE TABLE IF NOT EXISTS async_delegations ( + delegation_id TEXT PRIMARY KEY, + origin_session TEXT NOT NULL, + origin_ui_session_id TEXT NOT NULL DEFAULT '', + parent_session_id TEXT, + state TEXT NOT NULL, + dispatched_at REAL NOT NULL, + completed_at REAL, + updated_at REAL NOT NULL, + event_json TEXT, + result_json TEXT, + delivery_state TEXT NOT NULL DEFAULT 'pending', + delivery_attempts INTEGER NOT NULL DEFAULT 0, + delivered_at REAL, + owner_pid INTEGER, + owner_started_at INTEGER, + task_json TEXT, + delivery_claim TEXT, + delivery_claimed_at REAL +); + +CREATE INDEX IF NOT EXISTS idx_sessions_source ON sessions(source); +CREATE INDEX IF NOT EXISTS idx_sessions_source_id ON sessions(source, id); +CREATE INDEX IF NOT EXISTS idx_sessions_parent ON sessions(parent_session_id); +CREATE INDEX IF NOT EXISTS idx_sessions_started ON sessions(started_at DESC); +CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_id, timestamp); +CREATE INDEX IF NOT EXISTS idx_compression_locks_expires ON compression_locks(expires_at); +CREATE INDEX IF NOT EXISTS idx_session_model_usage_session ON session_model_usage(session_id); +CREATE INDEX IF NOT EXISTS idx_session_model_usage_model ON session_model_usage(model); +CREATE INDEX IF NOT EXISTS idx_async_delegations_delivery + ON async_delegations(delivery_state, completed_at); +""" + + +# Indexes that reference columns added in later schema versions must be +# created AFTER _reconcile_columns() has had a chance to ADD them on +# existing databases. SCHEMA_SQL above is run by sqlite executescript +# which would otherwise fail on legacy DBs ("no such column: active"). +DEFERRED_INDEX_SQL = """ +CREATE INDEX IF NOT EXISTS idx_messages_session_active + ON messages(session_id, active, timestamp); +CREATE INDEX IF NOT EXISTS idx_messages_active_null + ON messages(active) WHERE active IS NULL; +CREATE INDEX IF NOT EXISTS idx_sessions_session_key + ON sessions(session_key, started_at DESC); +CREATE INDEX IF NOT EXISTS idx_sessions_gateway_peer + ON sessions(source, user_id, chat_id, chat_type, thread_id, started_at DESC); +CREATE INDEX IF NOT EXISTS idx_sessions_handoff_state + ON sessions(handoff_state, started_at); +""" + + +# ── Deferred FTS rebuild bookkeeping (schema v23) ── +# While a background index rebuild is pending, two state_meta keys define +# which message rows are currently IN the FTS indexes: +# +# fts_rebuild_high_water H — MAX(messages.id) at the moment the old +# indexes were dropped +# fts_rebuild_progress P — highest id the chunked backfill has indexed +# +# A row is indexed iff id <= P (backfilled) OR id > H (inserted after +# the drop; ids are AUTOINCREMENT so new rows are always > H and the insert +# triggers index them live). Rows in (P, H] are not yet indexed. +# +# Every trigger below gates on that same predicate: firing an FTS5 +# external-content 'delete' for a row that is NOT in the index corrupts the +# index, and skipping it for a row that IS indexed leaves a stale entry. +# When no rebuild is pending both keys are absent and COALESCE turns the +# predicate into a tautology (id > -1 OR id <= -1), i.e. normal operation. +# The two state_meta PK probes per write are negligible next to the FTS +# insert itself. +FTS_SQL = """ +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + content, + tool_name, + tool_calls, + content='messages', + content_rowid='id' +); + +CREATE TRIGGER IF NOT EXISTS messages_fts_insert AFTER INSERT ON messages +WHEN (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_high_water'), -1) + OR new.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_progress'), -1)) +BEGIN + INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) + VALUES (new.id, new.content, new.tool_name, new.tool_calls); +END; + +CREATE TRIGGER IF NOT EXISTS messages_fts_delete AFTER DELETE ON messages +WHEN (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_high_water'), -1) + OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_progress'), -1)) +BEGIN + INSERT INTO messages_fts(messages_fts, rowid, content, tool_name, tool_calls) + VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls); +END; + +CREATE TRIGGER IF NOT EXISTS messages_fts_update AFTER UPDATE ON messages +WHEN (old.content IS NOT new.content + OR old.tool_name IS NOT new.tool_name + OR old.tool_calls IS NOT new.tool_calls) + AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_high_water'), -1) + OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_progress'), -1)) +BEGIN + INSERT INTO messages_fts(messages_fts, rowid, content, tool_name, tool_calls) + VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls); + INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) + VALUES (new.id, new.content, new.tool_name, new.tool_calls); +END; +""" + + +# Trigram FTS5 table for CJK substring search. The default unicode61 +# tokenizer splits CJK characters into individual tokens, breaking phrase +# matching. The trigram tokenizer creates overlapping 3-byte sequences so +# substring queries work natively for any script (CJK, Thai, etc.). +# +# The trigram index is the most expensive index in state.db (~2.6x the size +# of the text it covers), and ``role='tool'`` rows are ~90% of message bytes +# while being almost entirely machine noise (base64 payloads, file dumps, +# delegation transcripts). The index therefore reads through +# ``messages_fts_trigram_src``, a view that excludes tool rows — they stay +# fully stored in ``messages`` and fully searchable via the standard +# ``messages_fts`` index; they just don't get trigram (CJK substring) +# treatment. ``search_messages`` routes CJK queries that filter on +# ``role='tool'`` to the LIKE fallback for the same reason. +FTS_TRIGRAM_SQL = """ +CREATE VIEW IF NOT EXISTS messages_fts_trigram_src AS + SELECT id, role, content, tool_name, tool_calls + FROM messages + WHERE role <> 'tool'; + +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5( + content, + tool_name, + tool_calls, + content='messages_fts_trigram_src', + content_rowid='id', + tokenize='trigram' +); + +CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_insert AFTER INSERT ON messages +WHEN new.role <> 'tool' + AND (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_high_water'), -1) + OR new.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_progress'), -1)) +BEGIN + INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) + VALUES (new.id, new.content, new.tool_name, new.tool_calls); +END; + +CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_delete AFTER DELETE ON messages +WHEN old.role <> 'tool' + AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_high_water'), -1) + OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_progress'), -1)) +BEGIN + INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content, tool_name, tool_calls) + VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls); +END; + +CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_update AFTER UPDATE ON messages +WHEN (old.content IS NOT new.content + OR old.tool_name IS NOT new.tool_name + OR old.tool_calls IS NOT new.tool_calls + OR old.role IS NOT new.role) + AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_high_water'), -1) + OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta + WHERE key = 'fts_rebuild_progress'), -1)) +BEGIN + INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content, tool_name, tool_calls) + SELECT 'delete', old.id, old.content, old.tool_name, old.tool_calls + WHERE old.role <> 'tool'; + INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) + SELECT new.id, new.content, new.tool_name, new.tool_calls + WHERE new.role <> 'tool'; +END; +""" + + +_FTS_CJK_TRIGGERS = ( + "messages_fts_cjk_insert", + "messages_fts_cjk_delete", + "messages_fts_cjk_update", +) + + +# state_meta breadcrumb set when a tokenizer-less process had to drop the +# cjk triggers to keep message writes alive: rows written from that moment +# on are missing from the cjk index, so it must not serve reads until +# `hermes sessions optimize-storage` rebuilds it on a capable host. +FTS_CJK_STALE_KEY = "fts_cjk_stale" + + +# ── Legacy (v22 / inline-content) FTS DDL ────────────────────────────── +# Used ONLY to keep an existing pre-v23 install's search working and its +# triggers repairable UNTIL the user opts into `hermes db optimize`. This is +# the exact inline shape v11..v22 shipped: each virtual table stores its own +# copy of ``content || tool_name || tool_calls`` and the trigram table indexes +# every row (including role='tool'). We never CREATE these on a fresh install — +# fresh installs are born on the v23 external-content schema above. These +# constants exist so a legacy DB is never accidentally handed the v23 DDL +# (which would create the external-content trigram source VIEW and leave the +# DB in a mixed, broken state). `optimize_fts_storage()` is what migrates a +# legacy DB to the v23 shape. +LEGACY_FTS_SQL = """ +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + content +); + +CREATE TRIGGER IF NOT EXISTS messages_fts_insert AFTER INSERT ON messages BEGIN + INSERT INTO messages_fts(rowid, content) VALUES ( + new.id, + COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') + ); +END; + +CREATE TRIGGER IF NOT EXISTS messages_fts_delete AFTER DELETE ON messages BEGIN + DELETE FROM messages_fts WHERE rowid = old.id; +END; + +CREATE TRIGGER IF NOT EXISTS messages_fts_update AFTER UPDATE ON messages BEGIN + DELETE FROM messages_fts WHERE rowid = old.id; + INSERT INTO messages_fts(rowid, content) VALUES ( + new.id, + COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') + ); +END; +""" + + +LEGACY_FTS_TRIGRAM_SQL = """ +CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5( + content, + tokenize='trigram' +); + +CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_insert AFTER INSERT ON messages BEGIN + INSERT INTO messages_fts_trigram(rowid, content) VALUES ( + new.id, + COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') + ); +END; + +CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_delete AFTER DELETE ON messages BEGIN + DELETE FROM messages_fts_trigram WHERE rowid = old.id; +END; + +CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_update AFTER UPDATE ON messages BEGIN + DELETE FROM messages_fts_trigram WHERE rowid = old.id; + INSERT INTO messages_fts_trigram(rowid, content) VALUES ( + new.id, + COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '') + ); +END; +""" diff --git a/hermes_state_portability.py b/hermes_state_portability.py new file mode 100644 index 00000000000..f6adcf71192 --- /dev/null +++ b/hermes_state_portability.py @@ -0,0 +1,656 @@ +"""Session listing/rich rows, export, and import (portability) for SessionDB. + +Mixin contract: this is a plain mixin class consumed by +``hermes_state.SessionDB``. It defines no ``__init__`` and no state of its +own; methods access the host's attributes (``self._conn``, ``self.db_path``, +``self._execute_write`` and other SessionDB methods) established by +``SessionDB.__init__``. It must never import hermes_state (cycle) — shared +module-level constants live in hermes_state_common. +""" + +import logging +import json +import time +from typing import Any, Dict, List, Optional + +from agent.skill_commands import SKILL_SCAFFOLD_SQL_LIKE +from hermes_state_common import ( + SCHEMA_SQL, + _PREVIEW_RAW_SELECT, + _shape_preview, +) + +# Moved methods logged under the "hermes_state" logger before the split; +# keep that logger identity so log filtering/capture behavior is unchanged. +logger = logging.getLogger("hermes_state") + + +class SessionPortabilityMixin: + """See module docstring — mixin for SessionDB (Port cluster).""" + + @classmethod + def _compact_session_cols(cls) -> str: + """SELECT list for compact_rows: every ``sessions`` column declared in + SCHEMA_SQL except the ``system_prompt`` blob, aliased with the ``s`` + prefix used by list_sessions_rich/_get_session_rich_row queries.""" + if cls._session_compact_cols_sql is None: + declared = cls._parse_schema_columns(SCHEMA_SQL)["sessions"] + cls._session_compact_cols_sql = ", ".join( + f"s.{name}" for name in declared + if name not in cls._SESSION_COMPACT_EXCLUDED + ) + return cls._session_compact_cols_sql + + def distinct_session_cwds(self, include_archived: bool = False) -> List[Dict[str, Any]]: + """Distinct non-empty session cwds with usage stats, for repo discovery. + + Aggregates across ALL session history (not a single page), so the desktop + can surface every git repo the user has worked in — not just the repos + that happen to be in the currently-loaded recents. Children/branches + count: a worktree session is still a real workspace signal. + """ + where = "cwd IS NOT NULL AND TRIM(cwd) != ''" + if not include_archived: + where += " AND archived = 0" + with self._lock: + rows = self._conn.execute( + "SELECT cwd AS cwd, COUNT(*) AS sessions, " + "MAX(COALESCE(ended_at, started_at, 0)) AS last_active " + f"FROM sessions WHERE {where} GROUP BY cwd" + ).fetchall() + return [ + { + "cwd": r["cwd"], + "sessions": int(r["sessions"] or 0), + "last_active": float(r["last_active"] or 0), + } + for r in rows + ] + + def list_cron_job_runs( + self, + job_id: str, + limit: int = 20, + offset: int = 0, + ) -> List[Dict[str, Any]]: + """List the run sessions produced by a single cron job, newest first. + + Cron runs are flat, independent sessions whose id is + ``cron_{job_id}_{timestamp}`` (see ``cron/scheduler.run_job``). They are + never compression roots and never branch, so this deliberately skips the + ``list_sessions_rich`` recursive compression-chain CTE / leading-wildcard + ``id_query`` path — that path seeds from *every* ``source='cron'`` row in + the DB and only filters to one job's runs after the scan, so it scales + with the whole cron pile (a heavy history makes the desktop run-history + endpoint time out before it eventually populates). + + Instead this binds to one job with a ``[prefix, prefix_hi)`` range over + the id (an index range scan, not a ``%...%`` substring), filters + ``source='cron'``, and orders by ``started_at DESC``. Work scales with + the requested window, not the total cron history. + + Returns the same enriched row shape as ``list_sessions_rich`` (adds + ``preview`` + ``last_active``) so callers can reuse it. + """ + prefix = f"cron_{job_id}_" + # Half-open upper bound for an index range scan: increment the final + # byte of the prefix so the range covers exactly the ids that start + # with ``prefix`` and nothing else. ``prefix`` always ends in '_', but + # compute it generically rather than hardcoding the successor char. + prefix_hi = prefix[:-1] + chr(ord(prefix[-1]) + 1) + + query = f""" + SELECT s.*, + COALESCE( + (SELECT {_PREVIEW_RAW_SELECT} + FROM messages m + WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL + ORDER BY m.timestamp, m.id LIMIT 1), + '' + ) AS _preview_raw, + COALESCE( + (SELECT MAX(m2.timestamp) FROM messages m2 WHERE m2.session_id = s.id), + s.started_at + ) AS last_active + FROM sessions s + WHERE s.source = 'cron' AND s.id >= ? AND s.id < ? + ORDER BY s.started_at DESC, s.id DESC + LIMIT ? OFFSET ? + """ + with self._lock: + cursor = self._conn.execute(query, (prefix, prefix_hi, limit, offset)) + rows = cursor.fetchall() + + runs: List[Dict[str, Any]] = [] + for row in rows: + s = dict(row) + s["preview"] = _shape_preview(s.pop("_preview_raw", "")) + runs.append(s) + return runs + + def _get_session_rich_row(self, session_id: str, compact_rows: bool = False) -> Optional[Dict[str, Any]]: + """Fetch a single session with the same enriched columns as + ``list_sessions_rich`` (preview + last_active). Returns None if the + session doesn't exist. + + Pass ``compact_rows=True`` to omit the ``system_prompt`` blob (see + ``list_sessions_rich`` for details). + """ + # Same read-your-writes guarantee as list_sessions_rich. + self.flush_token_counts() + _sel = self._compact_session_cols() if compact_rows else "s.*" + query = f""" + SELECT {_sel}, + COALESCE( + (SELECT {_PREVIEW_RAW_SELECT} + FROM messages m + WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL + ORDER BY m.timestamp, m.id LIMIT 1), + '' + ) AS _preview_raw, + COALESCE( + (SELECT MAX(m2.timestamp) FROM messages m2 WHERE m2.session_id = s.id), + s.started_at + ) AS last_active + FROM sessions s + WHERE s.id = ? + """ + with self._lock: + cursor = self._conn.execute(query, (session_id,)) + row = cursor.fetchone() + if not row: + return None + s = dict(row) + s["preview"] = _shape_preview(s.pop("_preview_raw", "")) + return s + + def get_session_rich_row(self, session_id: str, compact_rows: bool = False) -> Optional[Dict[str, Any]]: + """Public wrapper for :meth:`_get_session_rich_row`. + + Exposes the single-session enriched row (same columns as + ``list_sessions_rich``: preview + last_active) for callers outside + this module, e.g. the web server's session-search hydration. + """ + return self._get_session_rich_row(session_id, compact_rows=compact_rows) + + def list_skill_scaffolded_sessions(self, limit: int = 200) -> List[Dict[str, Any]]: + """Titled sessions whose first user turn was a ``/skill`` invocation. + + Those titles were generated from the expanded message, which embeds the + whole skill body — so they describe the skill rather than the request. + Returns ``id``, ``title``, and the full first-turn ``content`` so a + caller can re-derive what the user typed. Newest first. + """ + with self._lock: + rows = self._conn.execute( + """ + SELECT s.id, s.title, m.content + FROM sessions s + JOIN messages m ON m.id = ( + SELECT m2.id FROM messages m2 + WHERE m2.session_id = s.id AND m2.role = 'user' + AND m2.content IS NOT NULL + ORDER BY m2.timestamp, m2.id LIMIT 1 + ) + WHERE s.title IS NOT NULL AND m.content LIKE ? + ORDER BY s.started_at DESC + LIMIT ? + """, + (SKILL_SCAFFOLD_SQL_LIKE, int(limit)), + ).fetchall() + return [dict(row) for row in rows] + + def get_first_assistant_text(self, session_id: str) -> str: + """The session's first assistant reply as plain text ('' when none). + + Pairs with :meth:`list_skill_scaffolded_sessions` so a re-title can feed + the titler the same (request, reply) shape the live path uses. + """ + with self._lock: + row = self._conn.execute( + "SELECT content FROM messages " + "WHERE session_id = ? AND role = 'assistant' AND content IS NOT NULL " + "ORDER BY timestamp, id LIMIT 1", + (session_id,), + ).fetchone() + if not row: + return "" + decoded = self._decode_content(row["content"]) + return decoded if isinstance(decoded, str) else "" + + def export_session(self, session_id: str) -> Optional[Dict[str, Any]]: + """Export a single session with all its messages as a dict.""" + session = self.get_session(session_id) + if not session: + return None + messages = self.get_messages(session_id) + return {**session, "messages": messages} + + def export_session_lineage(self, session_id: str) -> Optional[Dict[str, Any]]: + """Export a compression lineage as one logical session dict.""" + lineage_ids = self.get_compression_lineage(session_id) + if not lineage_ids: + return None + segments = [] + for sid in lineage_ids: + segment = self.export_session(sid) + if segment: + segments.append(segment) + if not segments: + return None + base = dict(segments[-1]) + total_messages = sum(len(seg.get("messages") or []) for seg in segments) + base["segments"] = segments + base["lineage_session_ids"] = [seg["id"] for seg in segments] + base["message_count"] = total_messages + base["messages"] = [msg for seg in segments for msg in (seg.get("messages") or [])] + return base + + def export_all(self, source: str = None) -> List[Dict[str, Any]]: + """ + Export all sessions (with messages) as a list of dicts. + Suitable for writing to a JSONL file for backup/analysis. + """ + sessions = self.search_sessions(source=source, limit=100000) + results = [] + for session in sessions: + messages = self.get_messages(session["id"]) + results.append({**session, "messages": messages}) + return results + + @staticmethod + def _import_text_or_none(value: Any, field: str) -> Optional[str]: + if value is None: + return None + if isinstance(value, str): + return value + raise ValueError(f"{field} must be a string") + + @staticmethod + def _import_json_object_or_none(value: Any, field: str) -> Optional[str]: + if value is None: + return None + if isinstance(value, str): + try: + parsed = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError(f"{field} must be valid JSON") from exc + if not isinstance(parsed, dict): + raise ValueError(f"{field} must be a JSON object") + return value + if not isinstance(value, dict): + raise ValueError(f"{field} must be a JSON object") + try: + return json.dumps(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{field} must be JSON serializable") from exc + + @staticmethod + def _float_or_none(value: Any) -> Optional[float]: + if value is None: + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + @staticmethod + def _import_int_or_none(value: Any, field: str) -> Optional[int]: + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{field} must be an integer") from exc + + @staticmethod + def _int_or_default(value: Any, default: int = 0) -> int: + if value is None: + return default + try: + return int(value) + except (TypeError, ValueError): + return default + + @staticmethod + def _reasoning_json_value(value: Any) -> Any: + if not isinstance(value, str): + return value + try: + return json.loads(value) + except (json.JSONDecodeError, TypeError): + return value + + @staticmethod + def _import_error(index: int, session_id: str, error: str) -> Dict[str, Any]: + item: Dict[str, Any] = {"index": index, "error": error} + if session_id: + item["session_id"] = session_id + return item + + def import_sessions(self, sessions: List[Dict[str, Any]]) -> Dict[str, Any]: + """Import sessions exported by :meth:`export_session` or ``export_all``. + + Existing session IDs are skipped. Imported child sessions keep their + parent only when that parent already exists or is included in the same + import payload; otherwise the child is detached so partial imports don't + fail foreign-key validation. Gateway routing, handoff, rewind, and other + live runtime state are intentionally reset: this restores conversation + history, not ownership of a live channel or process. + """ + if not isinstance(sessions, list): + raise ValueError("sessions must be a list") + if len(sessions) > self._IMPORT_MAX_SESSIONS: + raise ValueError( + f"sessions must contain at most {self._IMPORT_MAX_SESSIONS} entries" + ) + + normalized: List[Dict[str, Any]] = [] + errors: List[Dict[str, Any]] = [] + seen_ids: set[str] = set() + total_messages = 0 + total_bytes = 0 + session_text_fields = ( + "source", + "user_id", + "model", + "system_prompt", + "end_reason", + "cwd", + "git_branch", + "git_repo_root", + "billing_provider", + "billing_base_url", + "billing_mode", + "cost_status", + "cost_source", + "pricing_version", + "title", + ) + message_text_fields = ( + "role", + "tool_call_id", + "tool_name", + "effect_disposition", + "finish_reason", + "reasoning", + "reasoning_content", + "platform_message_id", + "message_id", + ) + + for index, raw in enumerate(sessions): + if not isinstance(raw, dict): + errors.append(self._import_error(index, "", "session must be an object")) + continue + session_id = str(raw.get("id") or "").strip() + if not session_id: + errors.append(self._import_error(index, "", "session id is required")) + continue + if session_id in seen_ids: + errors.append(self._import_error(index, session_id, "duplicate session id")) + continue + messages = raw.get("messages") or [] + if not isinstance(messages, list): + errors.append(self._import_error(index, session_id, "messages must be a list")) + continue + if len(messages) > self._IMPORT_MAX_MESSAGES_PER_SESSION: + errors.append( + self._import_error( + index, + session_id, + "messages exceeds the per-session import limit", + ) + ) + continue + if any(not isinstance(msg, dict) for msg in messages): + errors.append( + self._import_error( + index, + session_id, + "messages must contain only objects", + ) + ) + continue + + try: + session_bytes = len( + json.dumps(raw, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + ) + except (TypeError, ValueError): + errors.append( + self._import_error(index, session_id, "session must be JSON serializable") + ) + continue + if session_bytes > self._IMPORT_MAX_SESSION_BYTES: + errors.append( + self._import_error(index, session_id, "session exceeds the import size limit") + ) + continue + total_bytes += session_bytes + if total_bytes > self._IMPORT_MAX_TOTAL_BYTES: + errors.append( + self._import_error(index, session_id, "import exceeds the total size limit") + ) + continue + + try: + clean_session = dict(raw) + clean_session["id"] = session_id + clean_session["model_config"] = self._import_json_object_or_none( + clean_session.get("model_config"), "model_config" + ) + clean_session["parent_session_id"] = self._import_text_or_none( + clean_session.get("parent_session_id"), "parent_session_id" + ) + for field in session_text_fields: + clean_session[field] = self._import_text_or_none( + clean_session.get(field), field + ) + + clean_messages: List[Dict[str, Any]] = [] + for message_index, message in enumerate(messages): + clean_message = dict(message) + role = clean_message.get("role") + if not isinstance(role, str) or not role: + raise ValueError(f"messages[{message_index}].role must be a non-empty string") + for field in message_text_fields: + if field == "role": + continue + clean_message[field] = self._import_text_or_none( + clean_message.get(field), field + ) + clean_message["token_count"] = self._import_int_or_none( + clean_message.get("token_count"), "token_count" + ) + clean_messages.append(clean_message) + except ValueError as exc: + errors.append(self._import_error(index, session_id, str(exc))) + continue + + total_messages += len(clean_messages) + if total_messages > self._IMPORT_MAX_TOTAL_MESSAGES: + errors.append( + self._import_error( + index, + session_id, + "messages exceeds the total import limit", + ) + ) + continue + seen_ids.add(session_id) + normalized.append( + {"index": index, "session": clean_session, "messages": clean_messages} + ) + + if errors: + return { + "ok": False, + "imported": 0, + "skipped": 0, + "detached": 0, + "errors": errors, + } + + def _do(conn): + imported_ids: List[str] = [] + skipped_ids: List[str] = [] + parent_updates: List[tuple[str, str]] = [] + detached = 0 + + for item in normalized: + raw = item["session"] + messages = item["messages"] + session_id = str(raw.get("id") or "").strip() + exists = conn.execute( + "SELECT 1 FROM sessions WHERE id = ? LIMIT 1", + (session_id,), + ).fetchone() + if exists: + skipped_ids.append(session_id) + continue + + started_at = self._float_or_none(raw.get("started_at")) + if started_at is None: + started_at = time.time() + archived = 1 if raw.get("archived") else 0 + + conn.execute( + """INSERT INTO sessions ( + id, source, user_id, model, model_config, system_prompt, + parent_session_id, started_at, ended_at, end_reason, + message_count, tool_call_count, input_tokens, output_tokens, + cache_read_tokens, cache_write_tokens, reasoning_tokens, + cwd, git_branch, git_repo_root, + billing_provider, billing_base_url, billing_mode, + estimated_cost_usd, actual_cost_usd, cost_status, cost_source, + pricing_version, title, api_call_count, archived + ) + VALUES ( + :id, :source, :user_id, :model, :model_config, + :system_prompt, NULL, :started_at, :ended_at, + :end_reason, 0, 0, :input_tokens, :output_tokens, + :cache_read_tokens, :cache_write_tokens, + :reasoning_tokens, :cwd, :git_branch, :git_repo_root, + :billing_provider, :billing_base_url, :billing_mode, + :estimated_cost_usd, :actual_cost_usd, :cost_status, + :cost_source, :pricing_version, :title, + :api_call_count, :archived + )""", + { + "id": session_id, + "source": str(raw.get("source") or "import"), + "user_id": raw.get("user_id"), + "model": raw.get("model"), + "model_config": raw.get("model_config"), + "system_prompt": raw.get("system_prompt"), + "started_at": started_at, + "ended_at": self._float_or_none(raw.get("ended_at")), + "end_reason": raw.get("end_reason"), + "input_tokens": self._int_or_default(raw.get("input_tokens")), + "output_tokens": self._int_or_default(raw.get("output_tokens")), + "cache_read_tokens": self._int_or_default( + raw.get("cache_read_tokens") + ), + "cache_write_tokens": self._int_or_default( + raw.get("cache_write_tokens") + ), + "reasoning_tokens": self._int_or_default( + raw.get("reasoning_tokens") + ), + "cwd": raw.get("cwd"), + "git_branch": raw.get("git_branch"), + "git_repo_root": raw.get("git_repo_root"), + "billing_provider": raw.get("billing_provider"), + "billing_base_url": raw.get("billing_base_url"), + "billing_mode": raw.get("billing_mode"), + "estimated_cost_usd": self._float_or_none( + raw.get("estimated_cost_usd") + ), + "actual_cost_usd": self._float_or_none( + raw.get("actual_cost_usd") + ), + "cost_status": raw.get("cost_status"), + "cost_source": raw.get("cost_source"), + "pricing_version": raw.get("pricing_version"), + "title": raw.get("title"), + "api_call_count": self._int_or_default(raw.get("api_call_count")), + "archived": archived, + }, + ) + + sanitized_messages: List[Dict[str, Any]] = [] + for msg in messages: + clean = dict(msg) + for key in ( + "reasoning_details", + "codex_reasoning_items", + "codex_message_items", + ): + clean[key] = self._reasoning_json_value(clean.get(key)) + sanitized_messages.append(clean) + + total_messages, total_tool_calls = self._insert_message_rows( + conn, + session_id, + sanitized_messages, + ) + conn.execute( + "UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?", + (total_messages, total_tool_calls, session_id), + ) + + parent_id = str(raw.get("parent_session_id") or "").strip() + if parent_id: + parent_updates.append((session_id, parent_id)) + imported_ids.append(session_id) + + parent_by_child = dict(parent_updates) + + def _would_create_cycle(session_id: str, parent_id: str) -> bool: + seen = {session_id} + current = parent_id + while current: + if current in seen: + return True + seen.add(current) + if current in parent_by_child: + current = parent_by_child[current] + continue + row = conn.execute( + "SELECT parent_session_id FROM sessions WHERE id = ? LIMIT 1", + (current,), + ).fetchone() + if row is None: + return False + current = row["parent_session_id"] + return False + + for session_id, parent_id in parent_updates: + parent_exists = conn.execute( + "SELECT 1 FROM sessions WHERE id = ? LIMIT 1", + (parent_id,), + ).fetchone() + if parent_exists and not _would_create_cycle(session_id, parent_id): + conn.execute( + "UPDATE sessions SET parent_session_id = ? WHERE id = ?", + (parent_id, session_id), + ) + else: + # Drop only the closing edge. Later entries can still attach + # to this now-root session, preserving the acyclic portion + # of a malformed imported lineage. + parent_by_child.pop(session_id, None) + detached += 1 + + return { + "ok": True, + "imported": len(imported_ids), + "skipped": len(skipped_ids), + "detached": detached, + "imported_ids": imported_ids, + "skipped_ids": skipped_ids, + "errors": [], + } + + return self._execute_write(_do) diff --git a/hermes_state_schema.py b/hermes_state_schema.py new file mode 100644 index 00000000000..80a44943379 --- /dev/null +++ b/hermes_state_schema.py @@ -0,0 +1,779 @@ +"""Schema creation, column reconciliation, and FTS DDL management for SessionDB. + +Mixin contract: this is a plain mixin class consumed by +``hermes_state.SessionDB``. It defines no ``__init__`` and no state of its +own; methods access the host's attributes (``self._conn``, ``self.db_path``, +``self._execute_write`` and other SessionDB methods) established by +``SessionDB.__init__``. It must never import hermes_state (cycle) — shared +module-level constants live in hermes_state_common. +""" + +import logging +import json +import sqlite3 +from typing import Dict, Optional + +from hermes_constants import get_hermes_home +from hermes_state_common import ( + DEFERRED_INDEX_SQL, + FTS_SQL, + FTS_STORAGE_VERSION, + FTS_TRIGRAM_SQL, + LEGACY_FTS_SQL, + LEGACY_FTS_TRIGRAM_SQL, + SCHEMA_SQL, + SCHEMA_VERSION, + _FTS_TRIGGERS, + _ephemeral_child_sql, +) + +# Moved methods logged under the "hermes_state" logger before the split; +# keep that logger identity so log filtering/capture behavior is unchanged. +logger = logging.getLogger("hermes_state") + + +class SessionSchemaMixin: + """See module docstring — mixin for SessionDB (Schema cluster).""" + + def _sqlite_supports_fts5(self, cursor: sqlite3.Cursor) -> bool: + try: + cursor.execute("CREATE VIRTUAL TABLE temp._hermes_fts5_probe USING fts5(x)") + cursor.execute("DROP TABLE temp._hermes_fts5_probe") + return True + except sqlite3.OperationalError as exc: + if not self._is_fts5_unavailable_error(exc): + raise + self._warn_fts5_unavailable(exc) + return False + + @staticmethod + def _fts_trigger_count(cursor: sqlite3.Cursor) -> int: + placeholders = ",".join("?" for _ in _FTS_TRIGGERS) + row = cursor.execute( + f"SELECT COUNT(*) FROM sqlite_master " + f"WHERE type = 'trigger' AND name IN ({placeholders})", + _FTS_TRIGGERS, + ).fetchone() + return int(row[0] if not isinstance(row, sqlite3.Row) else row[0]) + + @staticmethod + def _rebuild_fts_indexes( + cursor: sqlite3.Cursor, + *, + include_trigram: bool = True, + ) -> None: + # Both FTS tables are external-content (v23+): the special 'rebuild' + # command wipes the inverted index and repopulates it from the + # content source (messages for the standard index, the tool-row- + # excluding messages_fts_trigram_src view for the trigram index). + cursor.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')") + if include_trigram: + cursor.execute( + "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('rebuild')" + ) + # 'rebuild' indexes EVERY row, so any deferred-backfill markers are + # now satisfied — clear them, otherwise the background worker would + # re-insert rows the rebuild already covered (duplicate entries). + cursor.execute( + "DELETE FROM state_meta WHERE key IN " + "('fts_rebuild_high_water', 'fts_rebuild_progress')" + ) + + @staticmethod + def _rebuild_legacy_fts_indexes( + cursor: sqlite3.Cursor, + *, + include_trigram: bool = True, + ) -> None: + """Rebuild the LEGACY inline FTS indexes (pre-v23) from messages. + + Used only to repair a legacy DB whose triggers degraded under an + earlier no-FTS5 runtime. Inline tables have no external-content + 'rebuild' source, so we DELETE + reinsert the concatenated content + the legacy triggers produced. Never touches the v23 shape. + """ + cursor.execute("DELETE FROM messages_fts") + cursor.execute( + "INSERT INTO messages_fts(rowid, content) " + "SELECT id, " + "COALESCE(content, '') || ' ' || " + "COALESCE(tool_name, '') || ' ' || " + "COALESCE(tool_calls, '') " + "FROM messages" + ) + if not include_trigram: + return + cursor.execute("DELETE FROM messages_fts_trigram") + cursor.execute( + "INSERT INTO messages_fts_trigram(rowid, content) " + "SELECT id, " + "COALESCE(content, '') || ' ' || " + "COALESCE(tool_name, '') || ' ' || " + "COALESCE(tool_calls, '') " + "FROM messages" + ) + + def _fts_table_probe(self, cursor: sqlite3.Cursor, table_name: str) -> Optional[bool]: + try: + cursor.execute(f"SELECT * FROM {table_name} LIMIT 0") + return True + except sqlite3.OperationalError as exc: + if self._is_fts5_unavailable_error(exc): + # Only disable FTS entirely when the whole module is missing. + # A missing trigram tokenizer only affects trigram searches. + if self._is_trigram_unavailable_error(exc): + self._warn_trigram_unavailable(exc) + else: + self._warn_fts5_unavailable(exc) + return None + if "no such table" in str(exc).lower(): + return False + raise + + @staticmethod + def _parse_schema_columns(schema_sql: str) -> Dict[str, Dict[str, str]]: + """Extract expected columns per table from SCHEMA_SQL. + + Uses an in-memory SQLite database to parse the SQL — SQLite itself + handles all syntax (DEFAULT expressions with commas, inline + REFERENCES, CHECK constraints, etc.) so there are zero regex + edge cases. The in-memory DB is opened, the schema DDL is + executed, and PRAGMA table_info extracts the column metadata. + + Adding a column to SCHEMA_SQL is all that's needed; the + reconciliation loop picks it up automatically. + """ + ref = sqlite3.connect(":memory:") + try: + ref.executescript(schema_sql) + table_columns: Dict[str, Dict[str, str]] = {} + for (tbl,) in ref.execute( + "SELECT name FROM sqlite_master " + "WHERE type='table' AND name NOT LIKE 'sqlite_%'" + ).fetchall(): + cols: Dict[str, str] = {} + for row in ref.execute( + f'PRAGMA table_info("{tbl}")' + ).fetchall(): + # row: (cid, name, type, notnull, dflt_value, pk) + col_name = row[1] + col_type = row[2] or "" + notnull = row[3] + default = row[4] + pk = row[5] + # Reconstruct the type expression for ALTER TABLE ADD COLUMN + parts = [col_type] if col_type else [] + if notnull and not pk: + parts.append("NOT NULL") + if default is not None: + parts.append(f"DEFAULT {default}") + cols[col_name] = " ".join(parts) + table_columns[tbl] = cols + return table_columns + finally: + ref.close() + + def _reconcile_columns(self, cursor: sqlite3.Cursor) -> None: + """Ensure live tables have every column declared in SCHEMA_SQL. + + Follows the Beets/sqlite-utils pattern: the CREATE TABLE definition + in SCHEMA_SQL is the single source of truth for the desired schema. + On every startup this method diffs the live columns (via PRAGMA + table_info) against the declared columns, and ADDs any that are + missing. + + This makes column additions a declarative operation — just add + the column to SCHEMA_SQL and it appears on the next startup. + Version-gated migration blocks are no longer needed for ADD COLUMN. + """ + expected = self._parse_schema_columns(SCHEMA_SQL) + for table_name, declared_cols in expected.items(): + # Get current columns from the live table + try: + rows = cursor.execute( + f'PRAGMA table_info("{table_name}")' + ).fetchall() + except sqlite3.OperationalError: + continue # Table doesn't exist yet (shouldn't happen after executescript) + live_cols = set() + for row in rows: + # PRAGMA table_info returns (cid, name, type, notnull, dflt_value, pk) + name = row[1] if isinstance(row, (tuple, list)) else row["name"] + live_cols.add(name) + + for col_name, col_type in declared_cols.items(): + if col_name not in live_cols: + safe_name = col_name.replace('"', '""') + try: + cursor.execute( + f'ALTER TABLE "{table_name}" ADD COLUMN "{safe_name}" {col_type}' + ) + except sqlite3.OperationalError as exc: + # Expected: "duplicate column name" from a race or + # re-run. Unexpected: "Cannot add a NOT NULL column + # with default value NULL" from a schema mistake. + # Log at DEBUG so it's visible in agent.log. + logger.debug( + "reconcile %s.%s: %s", table_name, col_name, exc, + ) + + def _heal_gateway_routing_pk(self, cursor: sqlite3.Cursor) -> None: + """Rebuild ``gateway_routing`` when its PRIMARY KEY predates scoping. + + Early builds of the routing-index migration (#59203) created the + table with ``session_key TEXT PRIMARY KEY`` and no ``scope`` column. + ``_reconcile_columns()`` ADDs the missing ``scope`` column on those + databases, but SQLite cannot ALTER a primary key, so the shipped + composite ``PRIMARY KEY (scope, session_key)`` never lands. On such + tables every write path is broken: + + * ``save_gateway_routing_entry`` fails with "ON CONFLICT clause does + not match any PRIMARY KEY or UNIQUE constraint" (its upsert targets + the composite key), and + * ``replace_gateway_routing_entries`` fails with "UNIQUE constraint + failed: gateway_routing.session_key" whenever the same session_key + exists under a different scope — the exact isolation the composite + key exists to provide. + + Each failed save logs a warning and falls back to sessions.json, + so a legacy-shaped table produces endless per-save warning spam. + Rebuild it once, preserving rows. On a session_key collision across + scopes (possible while the PK was wrong) the newest row wins. + """ + try: + rows = cursor.execute( + 'PRAGMA table_info("gateway_routing")' + ).fetchall() + except sqlite3.OperationalError: + return + if not rows: + return + + def _col(row, idx, name): + return row[idx] if isinstance(row, (tuple, list)) else row[name] + + pk_cols = [ + _col(r, 1, "name") + for r in sorted( + (r for r in rows if _col(r, 5, "pk")), + key=lambda r: _col(r, 5, "pk"), + ) + ] + if pk_cols == ["scope", "session_key"]: + return + + logger.info( + "gateway_routing has legacy primary key %r; rebuilding with " + "composite (scope, session_key) key", + pk_cols, + ) + cursor.execute( + "ALTER TABLE gateway_routing RENAME TO gateway_routing_legacy_pk" + ) + cursor.execute( + """CREATE TABLE gateway_routing ( + scope TEXT NOT NULL DEFAULT '', + session_key TEXT NOT NULL, + entry_json TEXT NOT NULL, + updated_at REAL NOT NULL, + PRIMARY KEY (scope, session_key) +)""" + ) + # INSERT OR REPLACE + updated_at ordering: if the broken PK ever let + # two scopes race over one session_key, keep the newest row per + # (scope, session_key) pair. + cursor.execute( + "INSERT OR REPLACE INTO gateway_routing " + "(scope, session_key, entry_json, updated_at) " + "SELECT COALESCE(scope, ''), session_key, entry_json, updated_at " + "FROM gateway_routing_legacy_pk ORDER BY updated_at ASC" + ) + cursor.execute("DROP TABLE gateway_routing_legacy_pk") + + def _init_schema(self): + """Create tables and FTS if they don't exist, reconcile columns. + + Schema management follows the declarative reconciliation pattern + (Beets, sqlite-utils): SCHEMA_SQL is the single source of truth. + On existing databases, _reconcile_columns() diffs live columns + against SCHEMA_SQL and ADDs any missing ones. This eliminates + the version-gated migration chain for column additions, making + it impossible for reordered or inserted migrations to skip columns. + + The schema_version table is retained for future data migrations + (transforming existing rows) which cannot be handled declaratively. + """ + cursor = self._conn.cursor() + + cursor.executescript(SCHEMA_SQL) + + # ── Declarative column reconciliation ────────────────────────── + # Diff live tables against SCHEMA_SQL and ADD any missing columns. + # This is idempotent and self-healing: even if a version-gated + # migration was skipped (e.g. due to version renumbering), the + # column gets created here. + self._reconcile_columns(cursor) + + # Rebuild gateway_routing if it still carries the pre-scope PRIMARY + # KEY (session_key alone). ADD COLUMN cannot fix a PK, so this is + # the one table-shape repair reconciliation can't express. + self._heal_gateway_routing_pk(cursor) + + # Indexes that reference reconciler-added columns must be created + # AFTER _reconcile_columns runs — declaring them in SCHEMA_SQL + # makes the initial executescript fail on legacy DBs (the index's + # WHERE clause references a column that doesn't exist yet). + try: + cursor.execute( + "CREATE INDEX IF NOT EXISTS idx_messages_platform_msg_id " + "ON messages(session_id, platform_message_id) " + "WHERE platform_message_id IS NOT NULL" + ) + except sqlite3.OperationalError as exc: + logger.debug("idx_messages_platform_msg_id create skipped: %s", exc) + + # Deferred indexes that reference the reconciler-added ``active`` + # column (idx_messages_session_active) — same ordering constraint. + cursor.executescript(DEFERRED_INDEX_SQL) + + # Heal NULL ``active`` rows unconditionally on every startup. + # On real-world DBs the reconciler-added ``active`` column can lack + # its NOT NULL DEFAULT 1 (older reconciler builds reconstructed the + # type without the default — see #51646: PRAGMA shows + # (17,'active','INTEGER',0,None,0) in the wild), so INSERTs that + # omitted the column wrote NULL and the ``WHERE active = 1`` + # transcript loaders hid the whole history. The INSERTs now set + # active=1 explicitly; this idempotent repair un-hides rows written + # before the fix. It was previously gated at ``current_version < + # 12`` which never re-ran for already-v12+ databases. + try: + cursor.execute( + "UPDATE messages SET active = 1 WHERE active IS NULL" + ) + except sqlite3.OperationalError: + pass + + fts5_available = self._sqlite_supports_fts5(cursor) + fts_migrations_complete = True + if not fts5_available: + # Existing FTS triggers can still fire on messages INSERT/UPDATE + # even though the current sqlite runtime cannot read the virtual + # tables they target. Drop only the triggers so core persistence + # continues; if a future runtime has FTS5, _ensure_fts_schema() + # recreates them. + self._drop_fts_triggers(cursor) + + # ── Schema version bookkeeping ───────────────────────────────── + # Bump to current so future data migrations (if any) can gate on + # version. No version-gated column additions remain. + cursor.execute("SELECT version FROM schema_version LIMIT 1") + row = cursor.fetchone() + if row is None: + cursor.execute( + "INSERT INTO schema_version (version) VALUES (?)", + (SCHEMA_VERSION,), + ) + else: + current_version = row["version"] if isinstance(row, sqlite3.Row) else row[0] + # Data migrations that can't be expressed declaratively (row + # backfills, index changes tied to a specific version step) stay + # in a version-gated chain. Column additions are handled by + # _reconcile_columns() above and no longer need entries here. + if current_version < 10 and SCHEMA_VERSION == 10: + # v10: trigram FTS5 table for CJK/substring search. The + # virtual table + triggers are created unconditionally via + # FTS_TRIGRAM_SQL below, but existing rows need a one-time + # backfill into the FTS index. + # + # Only run this when v10 itself is the target schema. Current + # v11+ code drops and rebuilds both FTS tables below, so doing + # the v10-only trigram backfill first only burns startup time + # and WAL space before v11 throws the work away. + if fts5_available: + _fts_trigram_exists = self._fts_table_probe( + cursor, "messages_fts_trigram" + ) + if _fts_trigram_exists is False: + if self._ensure_fts_schema( + cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL + ): + cursor.execute( + "INSERT INTO messages_fts_trigram(rowid, content) " + "SELECT id, content FROM messages WHERE content IS NOT NULL" + ) + else: + fts_migrations_complete = False + elif _fts_trigram_exists is None: + fts_migrations_complete = False + else: + fts_migrations_complete = False + if current_version < 11 and SCHEMA_VERSION < 23: + # v11 (SUPERSEDED by v23): re-index FTS5 tables to cover + # tool_name + tool_calls in inline mode (#16751). v23 drops + # and rebuilds both FTS tables in external-content form, so + # running the v11 inline backfill first would only burn + # startup time and WAL space before v23 throws the work + # away — and its inline INSERT shape no longer matches the + # current external-content FTS_SQL anyway. Kept only for + # source archaeology; unreachable while SCHEMA_VERSION >= 23. + pass + if current_version < 16: + # v16: tag delegate subagent rows so pickers stay clean after + # parent deletes that used to orphan them (parent_session_id → NULL). + try: + cursor.execute( + "UPDATE sessions SET model_config = json_set(" + "COALESCE(model_config, '{}'), '$._delegate_from', parent_session_id) " + f"WHERE parent_session_id IS NOT NULL " + "AND json_extract(COALESCE(model_config, '{}'), '$._delegate_from') IS NULL " + f"AND {_ephemeral_child_sql('sessions')}" + ) + cursor.execute( + "UPDATE sessions SET model_config = json_set(" + "COALESCE(model_config, '{}'), '$._delegate_from', '__orphaned__') " + "WHERE parent_session_id IS NULL " + "AND json_extract(COALESCE(model_config, '{}'), '$._delegate_from') IS NULL " + "AND json_extract(COALESCE(model_config, '{}'), '$._branched_from') IS NULL " + "AND title IS NULL " + "AND message_count <= 25 " + "AND EXISTS (SELECT 1 FROM messages m " + " WHERE m.session_id = sessions.id AND m.role = 'tool') " + "AND NOT EXISTS (SELECT 1 FROM sessions ch " + " WHERE ch.parent_session_id = sessions.id)" + ) + except sqlite3.OperationalError: + pass + if current_version < 18: + # v18: gateway metadata consolidation (#9006). Backfill + # display_name / origin_json / expiry_finalized from + # sessions.json so pre-migration gateway sessions are + # discoverable from state.db without the JSON index. + try: + self._backfill_gateway_metadata_from_sessions_json(cursor) + except Exception as exc: + # Backfill is best-effort: sessions.json may be absent, + # corrupted, or partially stale. Missing metadata simply + # means consumers fall back to sessions.json for those + # rows until the gateway rewrites them. + logger.debug("v18 gateway metadata backfill skipped: %s", exc) + if current_version < 20: + # v20: per-model usage attribution (issue #51607). Going + # forward update_token_counts() records each API call into + # session_model_usage keyed by the live model, but existing + # sessions only have their aggregate totals on the sessions + # row. Seed one usage row per historical session from those + # aggregates so insights reads uniformly from the new table. + # INSERT OR IGNORE keeps it idempotent: if newer code already + # wrote a (session_id, model, provider) row for a session, the + # PK conflict skips the stale aggregate rather than doubling it. + try: + cursor.execute( + """INSERT OR IGNORE INTO session_model_usage ( + session_id, model, billing_provider, + billing_base_url, billing_mode, + api_call_count, input_tokens, + output_tokens, cache_read_tokens, + cache_write_tokens, reasoning_tokens, + estimated_cost_usd, actual_cost_usd, + cost_status, cost_source, first_seen, last_seen + ) + SELECT id, COALESCE(model, 'unknown'), + COALESCE(billing_provider, ''), + COALESCE(billing_base_url, ''), + COALESCE(billing_mode, ''), + COALESCE(api_call_count, 0), + COALESCE(input_tokens, 0), + COALESCE(output_tokens, 0), + COALESCE(cache_read_tokens, 0), + COALESCE(cache_write_tokens, 0), + COALESCE(reasoning_tokens, 0), + COALESCE(estimated_cost_usd, 0), + COALESCE(actual_cost_usd, 0), + cost_status, cost_source, + started_at, COALESCE(ended_at, started_at) + FROM sessions + WHERE COALESCE(input_tokens, 0) + + COALESCE(output_tokens, 0) + + COALESCE(cache_read_tokens, 0) + + COALESCE(cache_write_tokens, 0) + + COALESCE(reasoning_tokens, 0) > 0""" + ) + except sqlite3.OperationalError: + pass + if current_version < 22: + # v22: task-dimension usage attribution (issue #23270). + # session_model_usage gains a ``task`` column ('' = main agent + # loop; 'vision'/'compression'/'title_generation'/... = + # auxiliary calls) so aux model spend is visible in analytics. + # The column participates in the PRIMARY KEY and SQLite cannot + # ALTER a PK, so rebuild the table. The reconciler will have + # already ADDed the plain column on legacy DBs (harmless); + # the rebuild bakes it into the PK properly. Existing rows are + # main-loop accounting by definition → task=''. + try: + legacy_pk = cursor.execute( + "SELECT COUNT(*) FROM pragma_table_info('session_model_usage') " + "WHERE name = 'task' AND pk > 0" + ).fetchone()[0] + if not legacy_pk: + cursor.execute("ALTER TABLE session_model_usage RENAME TO session_model_usage_v21") + cursor.execute( + """CREATE TABLE session_model_usage ( + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + model TEXT NOT NULL, + billing_provider TEXT NOT NULL DEFAULT '', + billing_base_url TEXT NOT NULL DEFAULT '', + billing_mode TEXT NOT NULL DEFAULT '', + task TEXT NOT NULL DEFAULT '', + api_call_count INTEGER NOT NULL DEFAULT 0, + input_tokens INTEGER NOT NULL DEFAULT 0, + output_tokens INTEGER NOT NULL DEFAULT 0, + cache_read_tokens INTEGER NOT NULL DEFAULT 0, + cache_write_tokens INTEGER NOT NULL DEFAULT 0, + reasoning_tokens INTEGER NOT NULL DEFAULT 0, + estimated_cost_usd REAL NOT NULL DEFAULT 0, + actual_cost_usd REAL NOT NULL DEFAULT 0, + cost_status TEXT, + cost_source TEXT, + first_seen REAL, + last_seen REAL, + PRIMARY KEY (session_id, model, billing_provider, billing_base_url, billing_mode, task) + )""" + ) + cursor.execute( + """INSERT INTO session_model_usage ( + session_id, model, billing_provider, billing_base_url, + billing_mode, task, api_call_count, input_tokens, + output_tokens, cache_read_tokens, cache_write_tokens, + reasoning_tokens, estimated_cost_usd, actual_cost_usd, + cost_status, cost_source, first_seen, last_seen + ) + SELECT session_id, model, billing_provider, billing_base_url, + billing_mode, '', api_call_count, input_tokens, + output_tokens, cache_read_tokens, cache_write_tokens, + reasoning_tokens, estimated_cost_usd, actual_cost_usd, + cost_status, cost_source, first_seen, last_seen + FROM session_model_usage_v21""" + ) + cursor.execute("DROP TABLE session_model_usage_v21") + cursor.execute( + "CREATE INDEX IF NOT EXISTS idx_session_model_usage_session " + "ON session_model_usage(session_id)" + ) + cursor.execute( + "CREATE INDEX IF NOT EXISTS idx_session_model_usage_model " + "ON session_model_usage(model)" + ) + except sqlite3.OperationalError as exc: + logger.debug("v22 session_model_usage rebuild skipped: %s", exc) + if current_version < 23: + # v23: FTS storage redesign (issues #22478, #43690, #55233). + # The v11 inline-mode FTS tables each store a full private + # copy of every message (content || tool_name || tool_calls), + # and the trigram index additionally covers role='tool' rows + # (~90% of message bytes: base64 payloads, file dumps) at + # ~2.6x amplification — together ~75% of state.db on heavy + # installs (observed: 18.9 GB of a 25 GB DB). + # + # OPT-IN, NOT AUTOMATIC. The transition (demote old vtables → + # new external-content schema → backfill → teardown → VACUUM) + # is disk-heavy (transient ~2x file size to fully reclaim via + # VACUUM) and long (~1-2h background on a 25 GB DB). Doing it + # silently on every big user's next open — with a completeness + # guarantee that depends on the process staying alive long + # enough — is the wrong default. So on an EXISTING install we + # touch nothing here: the v22 inline FTS keeps working exactly + # as before, and we only record a flag advertising that the + # optimization is available. `hermes sessions optimize-storage` + # performs the whole transition as one deliberate, disk-checked, + # progress-reported foreground operation. + # + # DECOUPLED VERSIONING. Crucially, this does NOT hold back the + # main schema_version. The FTS storage LAYOUT is tracked by an + # independent `fts_storage_version` marker (see + # _fts_storage_version / SETTLE below), so schema_version + # advances to SCHEMA_VERSION here like every other migration — + # future v24+ migrations land automatically for legacy-FTS + # users too. Only the FTS *layout* waits for opt-in. + if fts5_available and self._db_has_legacy_inline_fts(cursor): + self.set_meta("fts_optimize_available", "1", cursor=cursor) + + # The FTS storage layout is versioned independently of the main + # schema (see the v23 note above). Stamp the current layout so the + # main version can always advance: a fresh/optimized DB is at + # FTS_STORAGE_VERSION; a legacy DB is left at whatever it had + # (absent/0) until `optimize-storage` runs. An INTERRUPTED + # optimize (legacy vtables already demoted, but rebuild markers + # or demoted trash tables still present) is NOT stamped either — + # the marker is the source of truth for "fully optimized", and + # `fts_optimize_available()` keeps offering the resume until the + # transition actually completes. + if ( + fts5_available + and not self._db_has_legacy_inline_fts(cursor) + and cursor.execute( + "SELECT 1 FROM state_meta " + "WHERE key = 'fts_rebuild_high_water' LIMIT 1" + ).fetchone() is None + and not self._has_fts_trash(cursor) + ): + self.set_meta( + "fts_storage_version", str(FTS_STORAGE_VERSION), cursor=cursor + ) + + # Advance schema_version to current for ALL non-FTS-layout + # migrations. This is deliberately NOT gated on the FTS opt-in — + # holding the whole version back would block every future schema + # migration for a user who never optimizes. FTS5 being unavailable + # is the one case we skip (we can't have created the current FTS + # objects, so claiming the current schema would be a lie). + if ( + current_version < SCHEMA_VERSION + and fts_migrations_complete + and fts5_available + ): + cursor.execute( + "UPDATE schema_version SET version = ?", + (SCHEMA_VERSION,), + ) + + # Unique title index — always ensure it exists. Older databases may + # contain duplicate aliases from before the constraint was enforced; + # preserve every session while letting the newest one retain the alias. + title_index_sql = ( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_sessions_title_unique " + "ON sessions(title) WHERE title IS NOT NULL" + ) + try: + cursor.execute(title_index_sql) + except sqlite3.IntegrityError: + # The index is an optimization — its creation must never abort + # opening the database, so the repair itself is also guarded. + try: + cursor.execute( + """UPDATE sessions AS older + SET title = NULL + WHERE title IS NOT NULL + AND EXISTS ( + SELECT 1 FROM sessions AS newer + WHERE newer.title = older.title + AND newer.rowid > older.rowid + )""" + ) + logger.warning( + "Cleared %d duplicate session title(s) while restoring the unique index", + cursor.rowcount, + ) + cursor.execute(title_index_sql) + except sqlite3.Error: + logger.exception( + "Could not repair duplicate session titles; " + "unique title index not created" + ) + except sqlite3.OperationalError: + pass # Index already exists + + if fts5_available: + # FTS5 setup. Run the DDL even when the virtual table exists so + # CREATE TRIGGER IF NOT EXISTS repairs trigger-only degradation from + # an earlier no-FTS5 runtime. + # + # OPT-IN v23 boundary: a legacy v22 install (inline-content FTS, + # not yet opted into `hermes db optimize`) must keep its EXISTING + # inline schema + triggers. Running the v23 external-content DDL + # here would create the trigram source VIEW and leave the DB in a + # mixed inline/external state. So for a legacy DB we only ensure + # its inline triggers exist (via the legacy DDL), and skip the + # v23 view/external tables entirely. Fresh installs and opted-in + # DBs have no legacy inline FTS, so they get the v23 DDL. + if self._db_has_legacy_inline_fts(cursor): + triggers_need_repair = ( + self._fts_trigger_count(cursor) < len(_FTS_TRIGGERS) + ) + self._fts_enabled = self._ensure_fts_schema( + cursor, "messages_fts", LEGACY_FTS_SQL + ) + if self._fts_enabled: + trigram_enabled = self._ensure_fts_schema( + cursor, "messages_fts_trigram", LEGACY_FTS_TRIGRAM_SQL + ) + self._trigram_available = trigram_enabled + if triggers_need_repair: + self._rebuild_legacy_fts_indexes( + cursor, include_trigram=trigram_enabled + ) + else: + triggers_need_repair = ( + self._fts_trigger_count(cursor) < len(_FTS_TRIGGERS) + ) + self._fts_enabled = self._ensure_fts_schema( + cursor, "messages_fts", FTS_SQL + ) + + # Trigram FTS5 for CJK/substring search. This is optional + # relative to the main FTS table; if it cannot be created, + # CJK search falls back to LIKE. + if self._fts_enabled: + trigram_enabled = self._ensure_fts_schema( + cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL + ) + self._trigram_available = trigram_enabled + if triggers_need_repair: + self._rebuild_fts_indexes( + cursor, + include_trigram=trigram_enabled, + ) + # CJK-bigram index (cjk_unicode61). Strictly additive to + # the surfaces above and gated on the loadable tokenizer: + self._ensure_fts_cjk_schema(cursor) + + self._conn.commit() + + def _backfill_gateway_metadata_from_sessions_json( + self, cursor: sqlite3.Cursor + ) -> None: + """One-time v18 backfill of gateway metadata from sessions.json. + + Existing gateway sessions predate the display_name / origin_json / + expiry_finalized columns; copy what sessions.json knows so consumers + can switch to state.db without losing pre-migration sessions. + Only fills NULL columns — never overwrites data written by newer code. + """ + sessions_file = get_hermes_home() / "sessions" / "sessions.json" + if not sessions_file.exists(): + return + with open(sessions_file, "r", encoding="utf-8") as f: + data = json.load(f) + if not isinstance(data, dict): + return + for key, entry in data.items(): + if str(key).startswith("_") or not isinstance(entry, dict): + continue + session_id = entry.get("session_id") + if not session_id: + continue + origin = entry.get("origin") + cursor.execute( + """UPDATE sessions + SET session_key = COALESCE(session_key, ?), + chat_id = COALESCE(chat_id, ?), + chat_type = COALESCE(chat_type, ?), + thread_id = COALESCE(thread_id, ?), + display_name = COALESCE(display_name, ?), + origin_json = COALESCE(origin_json, ?), + expiry_finalized = CASE + WHEN COALESCE(expiry_finalized, 0) = 0 AND ? = 1 THEN 1 + ELSE expiry_finalized + END + WHERE id = ?""", + ( + entry.get("session_key") or key, + (origin or {}).get("chat_id") if isinstance(origin, dict) else None, + entry.get("chat_type"), + (origin or {}).get("thread_id") if isinstance(origin, dict) else None, + entry.get("display_name"), + json.dumps(origin) if isinstance(origin, dict) else None, + 1 if entry.get("expiry_finalized") or entry.get("memory_flushed") else 0, + str(session_id), + ), + ) diff --git a/hermes_state_search.py b/hermes_state_search.py new file mode 100644 index 00000000000..e4da6bed745 --- /dev/null +++ b/hermes_state_search.py @@ -0,0 +1,1907 @@ +"""Full-text / trigram / CJK message search and FTS maintenance for SessionDB. + +Mixin contract: this is a plain mixin class consumed by +``hermes_state.SessionDB``. It defines no ``__init__`` and no state of its +own; methods access the host's attributes (``self._conn``, ``self.db_path``, +``self._execute_write`` and other SessionDB methods) established by +``SessionDB.__init__``. It must never import hermes_state (cycle) — shared +module-level constants live in hermes_state_common. +""" + +import logging +import json +import os +import re +import sqlite3 +import time +from typing import Any, Callable, Dict, List, Optional, Tuple + +from agent.skill_commands import describe_skill_invocation +from hermes_state_common import ( + FTS_CJK_STALE_KEY, + FTS_SQL, + FTS_STORAGE_VERSION, + FTS_TRIGRAM_SQL, + MAX_FTS5_QUERY_CHARS, + SCHEMA_VERSION, + _FTS_CJK_TRIGGERS, +) + +# Moved methods logged under the "hermes_state" logger before the split; +# keep that logger identity so log filtering/capture behavior is unchanged. +logger = logging.getLogger("hermes_state") + + +class SessionSearchMixin: + """See module docstring — mixin for SessionDB (Search cluster).""" + + def _try_incremental_merge_fts(self) -> None: + """Run one bounded FTS5 merge pass without failing the completed write.""" + if not self._fts_enabled: + return + try: + self._merge_fts_incrementally( + max_pages=self._FTS_MERGE_MAX_PAGES_PER_INDEX + ) + except sqlite3.Error as exc: + # Routine maintenance is best effort, but unexpected SQLite errors + # must remain visible instead of being silently mistaken for an + # optional missing index. + logger.warning("FTS incremental merge failed: %s", exc) + + def fts_rebuild_status(self) -> Optional[Dict[str, Any]]: + """Return deferred-rebuild progress, or None when no rebuild pending. + + Shape: {"pending": True, "total": , + "indexed": , "percent": <0-100 int>}. + Consumed by search_messages() notes and by status surfaces + (dashboard/desktop can poll this to render a progress indicator). + + Reads state_meta directly via _read_ctx instead of calling + get_meta() (which takes self._lock) so search_messages doesn't + block on the writer lock when checking rebuild status. + """ + with self._read_ctx() as conn: + row = conn.execute( + "SELECT key, value FROM state_meta WHERE key IN (?, ?)", + ("fts_rebuild_high_water", "fts_rebuild_progress"), + ).fetchall() + meta = {r["key"]: r["value"] for r in row} + high_water = meta.get("fts_rebuild_high_water") + if high_water is None: + return None + progress = int(meta.get("fts_rebuild_progress") or 0) + total = int(high_water) + if total <= 0: + return None + pct = min(100, int(100 * progress / total)) + return {"pending": True, "total": total, "indexed": progress, "percent": pct} + + def _fts_rebuild_finish(self) -> None: + """Finalize the deferred rebuild: boundary sweep + clear markers. + + The sweep is cheap insurance against any write that slipped through + the migration-boundary instant (between high_water capture and + trigger activation): re-index any row near the boundary that the + index is missing. docsize has one row per indexed doc, so the + anti-join is exact and runs on a narrow id range. + """ + def _do(conn): + hw_row = conn.execute( + "SELECT value FROM state_meta WHERE key = 'fts_rebuild_high_water'" + ).fetchone() + if hw_row is not None: + hw = int(hw_row[0]) + # Sweep a generous window around the boundary. + lo, hi = hw - 1000, hw + 1000 + conn.execute( + "INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) " + "SELECT m.id, m.content, m.tool_name, m.tool_calls " + "FROM messages m " + "WHERE m.id > ? AND m.id <= ? " + "AND NOT EXISTS (SELECT 1 FROM messages_fts_docsize d WHERE d.id = m.id)", + (lo, hi), + ) + conn.execute( + "INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) " + "SELECT m.id, m.content, m.tool_name, m.tool_calls " + "FROM messages m " + "WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' " + "AND NOT EXISTS (SELECT 1 FROM messages_fts_trigram_docsize d WHERE d.id = m.id)", + (lo, hi), + ) + conn.execute( + "DELETE FROM state_meta WHERE key IN " + "('fts_rebuild_high_water', 'fts_rebuild_progress')" + ) + self._execute_write(_do) + logger.info("Deferred FTS rebuild complete — all messages indexed.") + + def _fts_teardown_trash_step(self) -> bool: + """Tear down one chunk of a demoted v22 FTS shadow table. + + The trash tables are PLAIN tables (their vtable parent was demoted + away during the migration), so chunked DELETE + final DROP involve + no FTS5 machinery at all. Returns True while teardown work remains. + """ + with self._lock: + trash = [ + r[0] for r in self._conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' " + "AND name LIKE ? ESCAPE '\\'", + (self._FTS_TRASH_PREFIX.replace("_", "\\_") + "%",), + ).fetchall() + ] + if not trash: + return False + + tbl = trash[0] + + def _do(conn): + pk_cols = [ + r[1] for r in conn.execute(f"PRAGMA table_info({tbl})") + if r[5] > 0 + ] + key = ", ".join(pk_cols) if pk_cols else "rowid" + cur = conn.execute( + f"DELETE FROM {tbl} WHERE ({key}) IN " + f"(SELECT {key} FROM {tbl} LIMIT {self._FTS_REBUILD_CHUNK_ROWS})" + ) + if cur.rowcount == 0: + # Empty — the DROP is cheap now. + conn.execute(f"DROP TABLE IF EXISTS {tbl}") + logger.info("Old FTS shadow table %s torn down.", tbl) + return True # re-check: more trash tables / chunks may remain + + try: + return bool(self._execute_write(_do)) + except sqlite3.OperationalError as exc: + logger.debug("FTS trash teardown chunk failed (will retry): %s", exc) + return True + + def fts_rebuild_step(self) -> bool: + """Backfill one chunk of the deferred FTS rebuild. + + Returns True when more work remains, False when the rebuild is + complete (or none is pending). Safe to call from any process at any + time; chunks are claimed atomically inside the write transaction, so + concurrent callers interleave instead of duplicating rows. + """ + if not self._fts_enabled: + return False + high_water_raw = self.get_meta("fts_rebuild_high_water") + if high_water_raw is None: + return False + high_water = int(high_water_raw) + include_trigram = self._trigram_available + chunk = self._FTS_REBUILD_CHUNK_ROWS + + def _do(conn): + # Re-read progress inside the write transaction (BEGIN IMMEDIATE + # is already held by _execute_write) — this is the claim: two + # workers can't read the same progress value concurrently. + row = conn.execute( + "SELECT value FROM state_meta WHERE key = 'fts_rebuild_progress'" + ).fetchone() + if row is None: + return False # finished (or cleared) by another process + progress = int(row[0]) + if progress >= high_water: + return False + + # The chunk upper bound is an id, not a row count, so gaps from + # deleted rows don't shrink chunks below the claimed range. + upper = min(progress + chunk, high_water) + conn.execute( + "INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) " + "SELECT id, content, tool_name, tool_calls FROM messages " + "WHERE id > ? AND id <= ?", + (progress, upper), + ) + if include_trigram: + conn.execute( + "INSERT INTO messages_fts_trigram" + "(rowid, content, tool_name, tool_calls) " + "SELECT id, content, tool_name, tool_calls FROM messages " + "WHERE id > ? AND id <= ? AND role <> 'tool'", + (progress, upper), + ) + # Publish progress in the same transaction as the rows it + # covers — crash-atomic: either both land or neither does. + conn.execute( + "UPDATE state_meta SET value = ? " + "WHERE key = 'fts_rebuild_progress'", + (str(upper),), + ) + return upper < high_water + + try: + more = self._execute_write(_do) + except sqlite3.OperationalError as exc: + logger.debug("FTS rebuild chunk failed (will retry): %s", exc) + return True # transient (lock contention) — caller retries + if more is False: + status = self.fts_rebuild_status() + if status is not None and status["indexed"] >= status["total"]: + self._fts_rebuild_finish() + return False + return bool(more) + + def fts_cjk_rebuild_status(self) -> Optional[Dict[str, Any]]: + """CJK-index backfill progress, or None when none is pending.""" + with self._read_ctx() as conn: + row = conn.execute( + "SELECT key, value FROM state_meta WHERE key IN (?, ?)", + ("fts_cjk_rebuild_high_water", "fts_cjk_rebuild_progress"), + ).fetchall() + meta = {r["key"]: r["value"] for r in row} + high_water = meta.get("fts_cjk_rebuild_high_water") + if high_water is None: + return None + progress = int(meta.get("fts_cjk_rebuild_progress") or 0) + total = int(high_water) + if total <= 0: + return None + pct = min(100, int(100 * progress / total)) + return {"pending": True, "total": total, "indexed": progress, "percent": pct} + + def fts_cjk_rebuild_step(self) -> bool: + """Backfill one chunk of the CJK index. True while work remains.""" + if not self._fts_enabled or not self._fts_cjk_loaded: + return False + high_water_raw = self.get_meta("fts_cjk_rebuild_high_water") + if high_water_raw is None: + return False + high_water = int(high_water_raw) + chunk = self._FTS_REBUILD_CHUNK_ROWS + + def _do(conn): + row = conn.execute( + "SELECT value FROM state_meta " + "WHERE key = 'fts_cjk_rebuild_progress'" + ).fetchone() + if row is None: + return False # finished (or cleared) by another process + progress = int(row[0]) + if progress >= high_water: + return False + upper = min(progress + chunk, high_water) + conn.execute( + "INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) " + "SELECT id, content, tool_name, tool_calls FROM messages " + "WHERE id > ? AND id <= ? AND role <> 'tool'", + (progress, upper), + ) + conn.execute( + "UPDATE state_meta SET value = ? " + "WHERE key = 'fts_cjk_rebuild_progress'", + (str(upper),), + ) + return upper < high_water + + try: + more = self._execute_write(_do) + except sqlite3.OperationalError as exc: + logger.debug("CJK FTS rebuild chunk failed (will retry): %s", exc) + return True + if more is False: + status = self.fts_cjk_rebuild_status() + if status is not None and status["indexed"] >= status["total"]: + self._fts_cjk_rebuild_finish() + return False + return bool(more) + + def _fts_cjk_rebuild_finish(self) -> None: + """Boundary sweep + clear the cjk markers; index becomes servable.""" + def _do(conn): + hw_row = conn.execute( + "SELECT value FROM state_meta " + "WHERE key = 'fts_cjk_rebuild_high_water'" + ).fetchone() + if hw_row is not None: + hw = int(hw_row[0]) + lo, hi = hw - 1000, hw + 1000 + conn.execute( + "INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) " + "SELECT m.id, m.content, m.tool_name, m.tool_calls " + "FROM messages m " + "WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' " + "AND NOT EXISTS (SELECT 1 FROM messages_fts_cjk_docsize d WHERE d.id = m.id)", + (lo, hi), + ) + conn.execute( + "DELETE FROM state_meta WHERE key IN " + "('fts_cjk_rebuild_high_water', 'fts_cjk_rebuild_progress')" + ) + self._execute_write(_do) + self._fts_cjk_available = True + logger.info("CJK FTS index backfill complete — serving CJK search.") + + def _fts_cjk_reset_if_stale(self) -> None: + """Rebuild path for a stale cjk index (triggers were dropped). + + The gap's extent is unknown, so the only safe recovery is a from- + scratch rebuild: drop the table + triggers, clear the breadcrumb, + recreate via ``_ensure_fts_cjk_schema`` (which sets fresh backfill + markers on a populated DB). Called from ``optimize_fts_storage`` on + a tokenizer-capable host; no-op when not stale. + """ + if not self._fts_cjk_loaded: + return + + def _do(conn): + stale = conn.execute( + "SELECT 1 FROM state_meta WHERE key = ?", + (FTS_CJK_STALE_KEY,), + ).fetchone() + if not stale: + return False + for trig in _FTS_CJK_TRIGGERS: + conn.execute(f"DROP TRIGGER IF EXISTS {trig}") + conn.execute("DROP TABLE IF EXISTS messages_fts_cjk") + conn.execute("DROP VIEW IF EXISTS messages_fts_cjk_src") + conn.execute( + "DELETE FROM state_meta WHERE key IN " + f"('{FTS_CJK_STALE_KEY}', 'fts_cjk_rebuild_high_water', " + "'fts_cjk_rebuild_progress')" + ) + return True + was_stale = self._execute_write(_do) + if was_stale: + # Recreate outside the write transaction — _ensure_fts_cjk_schema + # uses executescript(), which implicitly commits any pending + # transaction and must not run inside _execute_write's BEGIN + # IMMEDIATE. Sets fresh backfill markers on a populated DB. + with self._lock: + self._ensure_fts_cjk_schema(self._conn) + self._conn.commit() + + def fts_optimize_available(self) -> bool: + """True when `optimize_fts_storage()` has work to do: either this DB + is a legacy inline-FTS install that can be optimized to the v23 + external-content schema, or a previous optimize run was interrupted + (legacy vtables already demoted, but backfill markers and/or trash + tables remain) and re-running would resume it, or the CJK-bigram + index needs a backfill/rebuild on this tokenizer-capable host. + False for fresh and fully-optimized installs (and when FTS5 is + unavailable).""" + if not self._fts_enabled or self.read_only: + return False + with self._lock: + if self._db_has_legacy_inline_fts(self._conn): + return True + # Interrupted optimize: demotion already removed the legacy + # vtables (so the check above is False), but the transition is + # unfinished until the backfill markers are cleared and the + # demoted trash tables are torn down. Search stays complete + # through the gap supplement meanwhile; re-running resumes. + if self._conn.execute( + "SELECT 1 FROM state_meta " + "WHERE key = 'fts_rebuild_high_water' LIMIT 1" + ).fetchone(): + return True + # CJK-bigram index work — only offerable when THIS process can + # tokenize: a pending backfill (markers set at creation on a + # populated DB) or a stale index awaiting a from-scratch rebuild. + if self._fts_cjk_loaded and self._conn.execute( + "SELECT 1 FROM state_meta WHERE key IN " + f"('fts_cjk_rebuild_high_water', '{FTS_CJK_STALE_KEY}') LIMIT 1" + ).fetchone(): + return True + return self._has_fts_trash(self._conn) + + def _demote_legacy_fts_to_trash(self) -> int: + """Demote the legacy inline FTS vtables and stage their shadow tables + for chunked teardown. Returns MAX(messages.id) as the rebuild high + water. O(1) schema surgery — the heavy delete is deferred to the + chunked teardown, exactly as the validated auto path did.""" + def _do(conn): + self._drop_fts_triggers(conn) + conn.execute("DROP VIEW IF EXISTS messages_fts_trigram_src") + had = bool(conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' " + "AND name IN ('messages_fts', 'messages_fts_trigram') " + "AND sql LIKE 'CREATE VIRTUAL TABLE%' LIMIT 1" + ).fetchone()) + if had: + conn.execute("PRAGMA writable_schema=ON") + conn.execute( + "DELETE FROM sqlite_master WHERE type = 'table' " + "AND name IN ('messages_fts', 'messages_fts_trigram') " + "AND sql LIKE 'CREATE VIRTUAL TABLE%'" + ) + conn.execute("PRAGMA writable_schema=RESET") + shadows = [ + r[0] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' " + "AND (name LIKE 'messages_fts_%' ESCAPE '\\' " + "OR name LIKE 'messages_fts_trigram_%' ESCAPE '\\')" + ).fetchall() + ] + for sh in shadows: + conn.execute(f"ALTER TABLE {sh} RENAME TO fts_v22_trash_{sh}") + # Create the new v23 empty schema + set the backfill markers. + self._ensure_fts_schema(conn, "messages_fts", FTS_SQL) + self._ensure_fts_schema(conn, "messages_fts_trigram", FTS_TRIGRAM_SQL) + hw = conn.execute("SELECT COALESCE(MAX(id), 0) FROM messages").fetchone()[0] + for k, v in ( + ("fts_rebuild_high_water", str(hw)), + ("fts_rebuild_progress", "0"), + ): + conn.execute( + "INSERT INTO state_meta (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (k, v), + ) + conn.execute("DELETE FROM state_meta WHERE key = 'fts_optimize_available'") + return hw + return int(self._execute_write(_do)) + + def optimize_fts_storage( + self, + *, + progress_cb: Optional[Callable[[Dict[str, Any]], None]] = None, + vacuum: bool = True, + ) -> Dict[str, Any]: + """Migrate a legacy v22 inline-FTS DB to the v23 external-content + schema, foreground and to completion. Safe to re-run: if a previous + attempt was interrupted it resumes from the progress marker. + + ``progress_cb`` receives {"phase", "percent", "indexed", "total"} + dicts for a CLI progress bar. Returns a summary dict. + + The trigram tokenizer being unavailable is not fatal — the base index + is still rebuilt (CJK falls back to LIKE), mirroring normal startup. + """ + if not self._fts_enabled: + return {"ok": False, "reason": "fts5_unavailable"} + if self.read_only: + return {"ok": False, "reason": "read_only"} + + # Only demote if we're actually still on the legacy shape. If a prior + # run already demoted (markers/trash present), skip straight to + # finishing the backfill + teardown — this is what makes re-running + # after an interruption safe. + with self._lock: + legacy = self._db_has_legacy_inline_fts(self._conn) + pending = self.get_meta("fts_rebuild_high_water") is not None + if legacy and not pending: + self._demote_legacy_fts_to_trash() + + # A stale CJK index (triggers dropped by a tokenizer-less process) + # can only be recovered from scratch — reset it now so the cjk + # backfill phase below rebuilds it. No-op without the tokenizer. + self._fts_cjk_reset_if_stale() + # An optimized v23 DB gaining the cjk index for the first time (no + # legacy work left, tokenizer newly installed): ensure the table + + # markers exist so the backfill phase has work to claim. + if self._fts_cjk_loaded: + with self._lock: + self._ensure_fts_cjk_schema(self._conn) + self._conn.commit() + + def _emit(phase: str) -> None: + if progress_cb is None: + return + st = self.fts_rebuild_status() + if st is None: + st = self.fts_cjk_rebuild_status() + progress_cb({ + "phase": phase, + "percent": st["percent"] if st else 100, + "indexed": st["indexed"] if st else 0, + "total": st["total"] if st else 0, + }) + + def _pause(chunk_seconds: float) -> None: + """Inter-chunk throttle (see the chunk-engine note above). + + The chunk methods themselves never sleep, so this loop is the + single place the duty cycle is enforced: without it, back-to-back + BEGIN IMMEDIATE chunks starve any live gateway/CLI process + sharing the DB out of its lock retries (the measured ~85% + write-lock ownership that froze concurrent sessions). + """ + time.sleep(max( + self._FTS_REBUILD_MIN_PAUSE, + chunk_seconds * self._FTS_REBUILD_DUTY_FACTOR, + )) + + # Phase 1: backfill (foreground, throttled between chunks so a live + # gateway sharing the DB stays responsive). + _emit("backfill") + while True: + _t0 = time.monotonic() + if not self.fts_rebuild_step(): + break + _emit("backfill") + _pause(time.monotonic() - _t0) + _emit("backfill") + + # Phase 1b: backfill the CJK-bigram index (its own marker pair; a + # no-op when the tokenizer isn't loadable or nothing is pending). + while True: + _t0 = time.monotonic() + if not self.fts_cjk_rebuild_step(): + break + _emit("backfill") + _pause(time.monotonic() - _t0) + + # Phase 2: tear down the demoted legacy shadow tables in chunks. + _emit("teardown") + while True: + _t0 = time.monotonic() + if not self._fts_teardown_trash_step(): + break + _emit("teardown") + _pause(time.monotonic() - _t0) + + # Phase 3: reclaim freed pages to the OS. + vacuum_ok = None + if vacuum: + _emit("vacuum") + try: + with self._lock: + self._conn.execute("VACUUM") + vacuum_ok = True + except sqlite3.OperationalError as exc: + # Most common cause: not enough free disk for VACUUM's temp + # copy. The optimization still succeeded; space just isn't + # reclaimed until a later VACUUM. Non-fatal. + logger.warning("VACUUM after FTS optimize failed: %s", exc) + vacuum_ok = False + # Best-effort: fold the WAL back into the main file so the on-disk + # size settles now rather than at close(). NOTE this is REFUSED + # (SQLITE_BUSY) while any other connection holds a WAL read-mark — + # e.g. a live gateway sharing the DB — so it is not sufficient on + # its own. Callers must therefore NOT size the result by stat()ing + # the file; use :meth:`logical_size_bytes`, which is truthful + # immediately regardless of readers. + try: + with self._lock: + self._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + except Exception as exc: + logger.debug( + "WAL checkpoint (TRUNCATE) after optimize VACUUM failed: %s", + exc, + ) + + # Phase 4: stamp the FTS storage layout as current, clear the "available" + # flag, and advance schema_version if it was somehow still behind (the + # main version normally advances on open now, but bump defensively so a + # DB opened only by pre-decoupling code still settles). The FTS-layout + # marker is the source of truth for "is this DB optimized". + def _settle(conn): + conn.execute( + "INSERT INTO state_meta (key, value) VALUES ('fts_storage_version', ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (str(FTS_STORAGE_VERSION),), + ) + conn.execute("DELETE FROM state_meta WHERE key = 'fts_optimize_available'") + conn.execute( + "UPDATE schema_version SET version = ? WHERE version < ?", + (SCHEMA_VERSION, SCHEMA_VERSION), + ) + self._execute_write(_settle) + _emit("done") + logger.info( + "FTS storage optimization complete (layout v%d).", FTS_STORAGE_VERSION + ) + return {"ok": True, "vacuumed": vacuum_ok} + + def get_anchored_view( + self, + session_id: str, + around_message_id: int, + window: int = 5, + bookend: int = 3, + keep_roles: Optional[Tuple[str, ...]] = ("user", "assistant"), + ) -> Dict[str, Any]: + """Return an anchored window plus session bookends. + + Built on top of ``get_messages_around``. Three slices: + + - ``window``: messages immediately surrounding the anchor. Filtered + to ``keep_roles`` (tool-response noise dropped by default), EXCEPT + the anchor itself is always preserved regardless of role. + - ``bookend_start``: first ``bookend`` user/assistant messages of the + session — but only those whose id is strictly before the window's + first message id. Empty when the window already overlaps the + session head. Empty-content messages (tool-call-only assistant + turns) are skipped so they don't crowd out actual prose openings. + - ``bookend_end``: last ``bookend`` user/assistant messages of the + session, same non-overlap rule at the tail. + + Bookends let an FTS5 hit anywhere in a long session yield the goal + (opening) and the resolution (closing) on a single call — without + loading the whole transcript. + + Returns ``{"window": [], "messages_before": 0, "messages_after": 0, + "bookend_start": [], "bookend_end": []}`` when the anchor isn't in + the session. + + ``keep_roles=None`` disables role filtering (raw window + raw + bookends). + """ + if bookend < 0: + bookend = 0 + + # Reuse the primitive — handles anchor-existence, content decoding, + # tool_calls deserialisation, and boundary counts. + primitive = self.get_messages_around( + session_id, around_message_id, window=window + ) + window_rows = primitive["window"] + if not window_rows: + return { + "window": [], + "messages_before": 0, + "messages_after": 0, + "bookend_start": [], + "bookend_end": [], + } + + # Apply role filter to the window, but never drop the anchor itself. + if keep_roles is not None: + keep_set = set(keep_roles) + filtered_window = [ + m for m in window_rows + if m.get("id") == around_message_id or m.get("role") in keep_set + ] + else: + filtered_window = window_rows + + window_min_id = window_rows[0]["id"] + window_max_id = window_rows[-1]["id"] + + # Fetch bookends only when there's room outside the window. SQL filters + # by id range, role, and non-empty content — tool-call-only assistant + # turns (content='' with tool_calls populated) are excluded so they + # don't crowd out actual prose openings/closings. + bookend_start_rows: List[Any] = [] + bookend_end_rows: List[Any] = [] + if bookend > 0: + with self._read_ctx() as conn: + role_clause = "" + role_params: list = [] + if keep_roles is not None: + role_placeholders = ",".join("?" for _ in keep_roles) + role_clause = f" AND role IN ({role_placeholders})" + role_params = list(keep_roles) + + bookend_start_rows = conn.execute( + f"SELECT * FROM messages " + f"WHERE session_id = ? AND id < ?{role_clause} " + f"AND length(content) > 0 " + f"ORDER BY id ASC LIMIT ?", + (session_id, window_min_id, *role_params, bookend), + ).fetchall() + + bookend_end_rows = conn.execute( + f"SELECT * FROM messages " + f"WHERE session_id = ? AND id > ?{role_clause} " + f"AND length(content) > 0 " + f"ORDER BY id DESC LIMIT ?", + (session_id, window_max_id, *role_params, bookend), + ).fetchall() + # End rows came back DESC for the LIMIT cap; flip to ASC. + bookend_end_rows = list(reversed(bookend_end_rows)) + + def _hydrate(row) -> Dict[str, Any]: + msg = dict(row) + if "content" in msg: + msg["content"] = self._decode_content(msg["content"]) + if msg.get("tool_calls"): + try: + msg["tool_calls"] = json.loads(msg["tool_calls"]) + except (json.JSONDecodeError, TypeError): + logger.warning( + "Failed to deserialize tool_calls in get_anchored_view, falling back to []" + ) + msg["tool_calls"] = [] + if msg.get("display_metadata") is not None: + msg["display_metadata"] = self._decode_display_metadata(msg["display_metadata"]) + return msg + + return { + "window": filtered_window, + "messages_before": primitive["messages_before"], + "messages_after": primitive["messages_after"], + "bookend_start": [_hydrate(r) for r in bookend_start_rows], + "bookend_end": [_hydrate(r) for r in bookend_end_rows], + } + + def list_recent_user_messages( + self, + session_id: str, + limit: int = 20, + include_inactive: bool = False, + ) -> List[Dict[str, Any]]: + """Return the *limit* most-recent user messages, newest first. + + Each entry is a dict with keys ``id``, ``timestamp``, ``preview``. + ``preview`` is the first 80 characters of the message content + (with line breaks collapsed to spaces). Used by the /rewind + slash command picker, CLI/TUI/gateway ``/undo [N]``, and any other + caller that needs real user-turn targets. + + Bookkeeping timeline rows (``display_kind`` set — e.g. model_switch, + async_delegation_complete, auto_continue, hidden) are excluded. They + are durable ``role='user'`` rows for the API transcript, but no client + counts them as user turns (desktop demotes them to system / drops them; + the CLI already uses ``not m.get("display_kind")``). Including them here + made ``/undo`` soft-delete from a marker instead of the last real turn — + same class of index skew as the prompt.submit ordinal bug. + + By default only active messages are returned. + """ + active_clause = "" if include_inactive else " AND active = 1" + # Match CLI/desktop: only real user turns, not timeline bookkeeping. + display_clause = " AND (display_kind IS NULL OR display_kind = '')" + with self._lock: + cursor = self._conn.execute( + "SELECT id, timestamp, content FROM messages " + "WHERE session_id = ? AND role = 'user'" + f"{active_clause}{display_clause} " + "ORDER BY id DESC LIMIT ?", + (session_id, int(limit)), + ) + rows = cursor.fetchall() + + result: List[Dict[str, Any]] = [] + for row in rows: + decoded = self._decode_content(row["content"]) + if isinstance(decoded, list): + # Multimodal — flatten text parts. + text_parts = [ + p.get("text", "") for p in decoded + if isinstance(p, dict) and p.get("type") == "text" + ] + preview = " ".join(t for t in text_parts if t).strip() + if not preview: + preview = "[multimodal content]" + elif isinstance(decoded, str): + # A /skill turn embeds the whole skill body; show what the user + # typed instead of the skill's opening prose. + preview = describe_skill_invocation(decoded) or decoded + else: + preview = "" + preview = " ".join(preview.split()) # collapse whitespace + if len(preview) > 80: + preview = preview[:77] + "..." + result.append( + { + "id": row["id"], + "timestamp": row["timestamp"], + "preview": preview, + } + ) + return result + + @staticmethod + def _sanitize_fts5_query(query: str) -> str: + """Sanitize user input for safe use in FTS5 MATCH queries. + + FTS5 has its own query syntax where characters like ``"``, ``(``, ``)``, + ``+``, ``*``, ``{``, ``}``, the column-filter operator ``:`` and bare + boolean operators (``AND``, ``OR``, ``NOT``) have special meaning. + Passing raw user input directly to MATCH can cause + ``sqlite3.OperationalError``. + + Strategy: + - Preserve properly paired quoted phrases (``"exact phrase"``) + - Strip unmatched FTS5-special characters that would cause errors + - Wrap unquoted hyphenated and dotted terms in quotes so FTS5 + matches them as exact phrases instead of splitting on the + hyphen/dot (e.g. ``chat-send``, ``P2.2``, ``my-app.config.ts``) + """ + # Cap user-controlled FTS input before any regex processing. Search + # queries do not need to be arbitrarily large, and bounding them keeps + # sanitizer/runtime behavior predictable under adversarial input. + query = query[:MAX_FTS5_QUERY_CHARS] + + # Step 1: Extract balanced double-quoted phrases and protect them + # from further processing via numbered placeholders. Do this with a + # single linear scan rather than a regex so pathological quote runs + # cannot induce backtracking. + _quoted_parts: list = [] + pieces: list[str] = [] + i = 0 + while i < len(query): + ch = query[i] + if ch != '"': + pieces.append(ch) + i += 1 + continue + end = query.find('"', i + 1) + if end == -1: + # Unmatched quote: replace with whitespace like the old + # sanitizer's special-char stripping step. + pieces.append(" ") + i += 1 + continue + _quoted_parts.append(query[i:end + 1]) + pieces.append(f"\x00Q{len(_quoted_parts) - 1}\x00") + i = end + 1 + + sanitized = "".join(pieces) + + # Step 2: Strip remaining (unmatched) FTS5-special characters. ``:`` is + # FTS5's column-filter operator (``col:term``); since the FTS table has a + # single ``content`` column, an unquoted colon query like ``TODO: fix`` + # parses as ``column:term`` and raises "no such column" — swallowed at + # the execute site into zero results. Strip it like the others. + sanitized = re.sub(r'[+{}():\"^]', " ", sanitized) + + # Step 3: Collapse repeated * (e.g. "***") into a single one, + # and remove leading * (prefix-only needs at least one char before *) + sanitized = re.sub(r"\*+", "*", sanitized) + sanitized = re.sub(r"(^|\s)\*", r"\1", sanitized) + + # Step 4: Remove dangling boolean operators at start/end that would + # cause syntax errors (e.g. "hello AND" or "OR world") + sanitized = re.sub(r"(?i)^(AND|OR|NOT)\b\s*", "", sanitized.strip()) + sanitized = re.sub(r"(?i)\s+(AND|OR|NOT)\s*$", "", sanitized.strip()) + + # Step 5: Wrap unquoted dotted and/or hyphenated terms in double + # quotes. FTS5's tokenizer splits on dots and hyphens, turning + # ``chat-send`` into ``chat AND send`` and ``P2.2`` into ``p2 AND 2``. + # Quoting preserves phrase semantics. A single pass avoids the + # double-quoting bug that would occur if dotted, hyphenated and underscored + # patterns were applied sequentially (e.g. ``my-app.config``). + sanitized = re.sub(r"\b(\w+(?:[._-]\w+)+)\b", r'"\1"', sanitized) + + # Step 6: Restore preserved quoted phrases + for i, quoted in enumerate(_quoted_parts): + sanitized = sanitized.replace(f"\x00Q{i}\x00", quoted) + + return sanitized.strip() + + @staticmethod + def _is_cjk_codepoint(cp: int) -> bool: + return (0x4E00 <= cp <= 0x9FFF or # CJK Unified Ideographs + 0x3400 <= cp <= 0x4DBF or # CJK Extension A + 0x20000 <= cp <= 0x2A6DF or # CJK Extension B + 0x3000 <= cp <= 0x303F or # CJK Symbols + 0x3040 <= cp <= 0x309F or # Hiragana + 0x30A0 <= cp <= 0x30FF or # Katakana + 0xAC00 <= cp <= 0xD7AF) # Hangul Syllables + + @staticmethod + def _contains_cjk(text: str) -> bool: + """Check if text contains CJK (Chinese, Japanese, Korean) characters.""" + for ch in text: + cp = ord(ch) + if (0x4E00 <= cp <= 0x9FFF or # CJK Unified Ideographs + 0x3400 <= cp <= 0x4DBF or # CJK Extension A + 0x20000 <= cp <= 0x2A6DF or # CJK Extension B + 0x3000 <= cp <= 0x303F or # CJK Symbols + 0x3040 <= cp <= 0x309F or # Hiragana + 0x30A0 <= cp <= 0x30FF or # Katakana + 0xAC00 <= cp <= 0xD7AF): # Hangul Syllables + return True + return False + + @classmethod + def _count_cjk(cls, text: str) -> int: + """Count CJK characters in text.""" + return sum(1 for ch in text if cls._is_cjk_codepoint(ord(ch))) + + @classmethod + def _has_lone_cjk_run(cls, query: str) -> bool: + """True when any maximal CJK run in the query is a single char. + + The cjk-bigram index stores bigrams for runs >=2 chars and unigrams + only for isolated chars, so a 1-char CJK term can't match inside + longer runs there — those queries keep the LIKE substring route. + """ + run = 0 + for ch in query: + if cls._is_cjk_codepoint(ord(ch)): + run += 1 + else: + if run == 1: + return True + run = 0 + return run == 1 + + @staticmethod + def _trigram_eligible_tokens(query: str) -> bool: + """True when every non-operator token is long enough for the trigram + tokenizer to match (>=3 chars). + + The trigram tokenizer indexes overlapping 3-character sequences, so a + token shorter than 3 chars produces no trigrams and can never match. + With FTS5's implicit-AND between tokens, a single short token makes the + whole MATCH return nothing, so the trigram path is only worth taking + when every searchable token qualifies. + """ + tokens = [ + t for t in query.strip('"').strip().split() + if t.upper() not in {"AND", "OR", "NOT"} + ] + return bool(tokens) and all(len(t) >= 3 for t in tokens) + + def _run_trigram_search( + self, + raw_query: str, + *, + table: str = "messages_fts_trigram", + order_by_sql: str, + include_inactive: bool, + source_filter: List[str] = None, + exclude_sources: List[str] = None, + role_filter: List[str] = None, + limit: int = 20, + offset: int = 0, + ) -> Optional[List[Dict[str, Any]]]: + """Run a search against a substring-capable FTS index. + + ``table`` is ``messages_fts_trigram`` (default) or + ``messages_fts_cjk``. The trigram tokenizer indexes overlapping + 3-byte sequences, so it matches substrings regardless of word + boundaries — both CJK phrases the unicode61 tokenizer splits into + single characters and Latin runs the unicode61 tokenizer fuses onto + adjacent CJK (e.g. ``修改youer服务端``). The cjk-bigram tokenizer + splits Latin runs off adjacent CJK, giving the same recovery as an + exact ranked token match. Each non-operator token is quoted to + neutralise FTS5 special characters while boolean operators + (AND/OR/NOT) are preserved. + + Returns the matching rows, or ``None`` when the query cannot be + executed (e.g. the tokenizer is unavailable at runtime) so the + caller can fall back to another strategy. + """ + tokens = raw_query.split() + parts = [] + for tok in tokens: + if tok.upper() in {"AND", "OR", "NOT"}: + parts.append(tok) + else: + parts.append('"' + tok.replace('"', '""') + '"') + trigram_query = " ".join(parts) + tri_where = [f"{table} MATCH ?"] + tri_params: list = [trigram_query] + if not include_inactive: + tri_where.append("(m.active = 1 OR m.compacted = 1)") + if source_filter is not None: + tri_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") + tri_params.extend(source_filter) + if exclude_sources is not None: + tri_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") + tri_params.extend(exclude_sources) + if role_filter: + tri_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") + tri_params.extend(role_filter) + tri_sql = f""" + SELECT + m.id, + m.session_id, + m.role, + snippet({table}, -1, '>>>', '<<<', '...', 40) AS snippet, + m.content, + m.timestamp, + m.tool_name, + s.source, + s.model, + s.started_at AS session_started + FROM {table} + JOIN messages m ON m.id = {table}.rowid + JOIN sessions s ON s.id = m.session_id + WHERE {' AND '.join(tri_where)} + {order_by_sql} + LIMIT ? OFFSET ? + """ + tri_params.extend([limit, offset]) + with self._read_ctx() as conn: + try: + tri_cursor = conn.execute(tri_sql, tri_params) + except sqlite3.OperationalError: + # Query failed at runtime — let the caller fall back. + return None + return [dict(row) for row in tri_cursor.fetchall()] + + def search_messages( + self, + query: str, + source_filter: List[str] = None, + exclude_sources: List[str] = None, + role_filter: List[str] = None, + limit: int = 20, + offset: int = 0, + sort: str = None, + include_inactive: bool = False, + ) -> List[Dict[str, Any]]: + """Instrumented wrapper around :meth:`_search_messages_impl`. + + Logs one line per slow search with the routing path taken, so + production latency stays attributable per query shape (the 2026-07 + session_search investigation needed trace archaeology to discover + the LIKE full scans; this makes the next regression a grep). + Threshold: HERMES_SEARCH_SLOW_MS (default 1000; 0 logs every call). + """ + started = time.time() + rows = None + try: + rows = self._search_messages_impl( + query, + source_filter=source_filter, + exclude_sources=exclude_sources, + role_filter=role_filter, + limit=limit, + offset=offset, + sort=sort, + include_inactive=include_inactive, + ) + return rows + finally: + try: + threshold = float(os.getenv("HERMES_SEARCH_SLOW_MS", "1000")) + except (TypeError, ValueError): + threshold = 1000.0 + elapsed_ms = (time.time() - started) * 1000.0 + if elapsed_ms >= threshold: + logger.info( + "slow session search: path=%s elapsed=%.0fms rows=%s query=%r", + self._describe_search_path(query), + elapsed_ms, + len(rows) if rows is not None else "err", + query[:200], + ) + + def _describe_search_path(self, query: str) -> str: + """Best-effort name of the routing path a query takes (log-only).""" + try: + sanitized = self._sanitize_fts5_query(query or "") + if not sanitized: + return "empty" + if not self._contains_cjk(sanitized): + return "fts5" + raw = sanitized.strip('"').strip() + if self._fts_cjk_available and not self._has_lone_cjk_run(raw): + return "fts_cjk" + tokens = [ + t for t in raw.split() + if t.upper() not in {"AND", "OR", "NOT"} and self._contains_cjk(t) + ] + short = any(self._count_cjk(t) < 3 for t in tokens) + if self._count_cjk(raw) >= 3 and not short and self._trigram_available: + return "trigram" + return "like_scan" + except Exception: + return "unknown" + + def _search_messages_impl( + self, + query: str, + source_filter: List[str] = None, + exclude_sources: List[str] = None, + role_filter: List[str] = None, + limit: int = 20, + offset: int = 0, + sort: str = None, + include_inactive: bool = False, + ) -> List[Dict[str, Any]]: + """ + Full-text search across session messages using FTS5. + + Supports FTS5 query syntax: + - Simple keywords: "docker deployment" + - Phrases: '"exact phrase"' + - Boolean: "docker OR kubernetes", "python NOT java" + - Prefix: "deploy*" + + Returns matching messages with session metadata, content snippet, + and surrounding context (1 message before and after the match). + + ``sort`` controls temporal ordering: + - ``None`` (default): FTS5 BM25 relevance only. Time-neutral. + - ``"newest"``: order by message timestamp DESC, then by rank. + - ``"oldest"``: order by message timestamp ASC, then by rank. + + The short-CJK LIKE fallback already orders by timestamp DESC and + ignores ``sort``. The trigram CJK path honours ``sort`` like the main + FTS5 path. + + Rewound (``active=0``, ``compacted=0``) rows are excluded by default — + the user took those back. Compaction-archived rows (``active=0``, + ``compacted=1``) ARE included by default: they were summarized away from + the live context but remain part of the conversation's record, so the + pre-compaction transcript stays discoverable after in-place compaction + (#38763). Pass ``include_inactive=True`` to search every row regardless. + """ + if not self._fts_enabled: + return [] + + if not query or not query.strip(): + return [] + + query = self._sanitize_fts5_query(query) + if not query: + return [] + + # Normalise sort. Anything not in the allowed set falls back to None + # (FTS5 rank-only) so callers can pass through user input without + # validation. + if isinstance(sort, str): + sort_norm = sort.strip().lower() + if sort_norm not in ("newest", "oldest"): + sort_norm = None + else: + sort_norm = None + + # ORDER BY shared across the main FTS5 path and trigram CJK path. + # With sort set, timestamp is primary and rank is the tiebreaker. + if sort_norm == "newest": + order_by_sql = "ORDER BY m.timestamp DESC, rank" + elif sort_norm == "oldest": + order_by_sql = "ORDER BY m.timestamp ASC, rank" + else: + order_by_sql = "ORDER BY rank" + + # Build WHERE clauses dynamically + where_clauses = ["messages_fts MATCH ?"] + params: list = [query] + if not include_inactive: + # Live rows (active=1) AND compaction-archived rows (compacted=1) + # are discoverable; only rewind/undo rows (active=0, compacted=0) + # are hidden. See archive_and_compact() / #38763. + where_clauses.append("(m.active = 1 OR m.compacted = 1)") + + if source_filter is not None: + source_placeholders = ",".join("?" for _ in source_filter) + where_clauses.append(f"s.source IN ({source_placeholders})") + params.extend(source_filter) + + if exclude_sources is not None: + exclude_placeholders = ",".join("?" for _ in exclude_sources) + where_clauses.append(f"s.source NOT IN ({exclude_placeholders})") + params.extend(exclude_sources) + + if role_filter: + role_placeholders = ",".join("?" for _ in role_filter) + where_clauses.append(f"m.role IN ({role_placeholders})") + params.extend(role_filter) + + where_sql = " AND ".join(where_clauses) + params.extend([limit, offset]) + + sql = f""" + SELECT + m.id, + m.session_id, + m.role, + snippet(messages_fts, -1, '>>>', '<<<', '...', 40) AS snippet, + m.content, + m.timestamp, + m.tool_name, + s.source, + s.model, + s.started_at AS session_started + FROM messages_fts + JOIN messages m ON m.id = messages_fts.rowid + JOIN sessions s ON s.id = m.session_id + WHERE {where_sql} + {order_by_sql} + LIMIT ? OFFSET ? + """ + + # CJK queries bypass the unicode61 FTS5 table. The default tokenizer + # splits CJK characters into individual tokens, so "大别山项目" becomes + # "大 AND 别 AND 山 AND 项 AND 目" — producing false positives and + # missing exact phrase matches. + # + # For queries with 3+ CJK characters, we use the trigram FTS5 table + # (indexed substring matching with ranking and snippets). For shorter + # CJK queries (1-2 chars), trigram can't match (it needs ≥9 UTF-8 + # bytes = 3 CJK chars), so we fall back to LIKE. + is_cjk = self._contains_cjk(query) + if is_cjk: + raw_query = query.strip('"').strip() + cjk_count = self._count_cjk(raw_query) + + # Per-token CJK length check (#20494): trigram needs >=3 CJK chars + # per token. A query like "广西 OR 桂林 OR 漓江" has cjk_count=6 + # (>=3) but each individual token is only 2 chars — trigram returns 0. + # Route to LIKE when any non-operator CJK token is <3 CJK chars. + _tokens_for_check = [ + t for t in raw_query.split() + if t.upper() not in {"AND", "OR", "NOT"} and self._contains_cjk(t) + ] + _any_short_cjk = any( + self._count_cjk(t) < 3 for t in _tokens_for_check + ) + + _trigram_succeeded = False + # Tool rows are excluded from the trigram index (they're ~90% of + # message bytes and machine noise — see FTS_TRIGRAM_SQL). A CJK + # query explicitly filtering on role='tool' must therefore use + # the LIKE fallback, which scans the base table directly. + _wants_tool_rows = bool(role_filter) and "tool" in role_filter + + # ── CJK-bigram route (messages_fts_cjk, cjk_unicode61) ────── + # When the bigram index is available it serves EVERY CJK query + # shape the legacy code split between trigram (>=3 chars/token) + # and LIKE full scans (1-2 char tokens) — the whole point of the + # index (PR #65544). Exceptions stay on the legacy routes: + # - role_filter=['tool'] queries (tool rows aren't in the cjk + # index, same exclusion as trigram), + # - queries containing a LONE 1-char CJK run: the index stores + # bigrams for runs >=2, so a single-char term can only match + # isolated chars — LIKE substring semantics are broader. + if ( + self._fts_cjk_available + and not _wants_tool_rows + and not self._has_lone_cjk_run(raw_query) + ): + tokens = raw_query.split() + parts = [] + for tok in tokens: + if tok.upper() in {"AND", "OR", "NOT"}: + parts.append(tok) + else: + parts.append('"' + tok.replace('"', '""') + '"') + cjk_query = " ".join(parts) + cjk_where = ["messages_fts_cjk MATCH ?"] + cjk_params: list = [cjk_query] + if not include_inactive: + cjk_where.append("(m.active = 1 OR m.compacted = 1)") + if source_filter is not None: + cjk_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") + cjk_params.extend(source_filter) + if exclude_sources is not None: + cjk_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") + cjk_params.extend(exclude_sources) + if role_filter: + cjk_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") + cjk_params.extend(role_filter) + cjk_sql = f""" + SELECT + m.id, + m.session_id, + m.role, + snippet(messages_fts_cjk, -1, '>>>', '<<<', '...', 40) AS snippet, + m.content, + m.timestamp, + m.tool_name, + s.source, + s.model, + s.started_at AS session_started + FROM messages_fts_cjk + JOIN messages m ON m.id = messages_fts_cjk.rowid + JOIN sessions s ON s.id = m.session_id + WHERE {' AND '.join(cjk_where)} + {order_by_sql} + LIMIT ? OFFSET ? + """ + cjk_params.extend([limit, offset]) + try: + with self._read_ctx() as conn: + cjk_cursor = conn.execute(cjk_sql, cjk_params) + matches = [dict(row) for row in cjk_cursor.fetchall()] + _trigram_succeeded = True + except sqlite3.OperationalError: + # Tokenizer missing on this connection / query syntax — + # the trigram + LIKE routes below still answer. + logger.debug( + "messages_fts_cjk query failed; falling back to " + "trigram/LIKE", exc_info=True, + ) + except sqlite3.DatabaseError as exc: + # Same corruption class as the other FTS reads: rebuild + # in place once and retry; on refusal/failure fall back. + if self._try_runtime_fts_rebuild(exc): + try: + with self._read_ctx() as conn: + cjk_cursor = conn.execute( + cjk_sql, cjk_params + ) + matches = [ + dict(row) for row in cjk_cursor.fetchall() + ] + _trigram_succeeded = True + except sqlite3.DatabaseError: + logger.warning( + "CJK-bigram FTS search still failing after " + "in-place rebuild; falling back to " + "trigram/LIKE." + ) + else: + logger.warning( + "CJK-bigram FTS search hit a corruption error " + "(%s) and no in-place rebuild was possible; " + "falling back to trigram/LIKE.", exc, + ) + + if ( + not _trigram_succeeded + and cjk_count >= 3 + and not _any_short_cjk + and self._trigram_available + and not _wants_tool_rows + ): + # Trigram FTS5 path — quote each non-operator token to handle + # FTS5 special chars (%, *, etc.) while preserving boolean + # operators (AND, OR, NOT) for multi-term queries. + tokens = raw_query.split() + parts = [] + for tok in tokens: + if tok.upper() in {"AND", "OR", "NOT"}: + parts.append(tok) + else: + parts.append('"' + tok.replace('"', '""') + '"') + trigram_query = " ".join(parts) + tri_where = ["messages_fts_trigram MATCH ?"] + tri_params: list = [trigram_query] + if not include_inactive: + tri_where.append("(m.active = 1 OR m.compacted = 1)") + if source_filter is not None: + tri_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") + tri_params.extend(source_filter) + if exclude_sources is not None: + tri_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") + tri_params.extend(exclude_sources) + if role_filter: + tri_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") + tri_params.extend(role_filter) + tri_sql = f""" + SELECT + m.id, + m.session_id, + m.role, + snippet(messages_fts_trigram, -1, '>>>', '<<<', '...', 40) AS snippet, + m.content, + m.timestamp, + m.tool_name, + s.source, + s.model, + s.started_at AS session_started + FROM messages_fts_trigram + JOIN messages m ON m.id = messages_fts_trigram.rowid + JOIN sessions s ON s.id = m.session_id + WHERE {' AND '.join(tri_where)} + {order_by_sql} + LIMIT ? OFFSET ? + """ + tri_params.extend([limit, offset]) + try: + with self._read_ctx() as conn: + tri_cursor = conn.execute(tri_sql, tri_params) + matches = [dict(row) for row in tri_cursor.fetchall()] + _trigram_succeeded = True + except sqlite3.OperationalError: + # Trigram query failed at runtime — fall through to LIKE. + pass + except sqlite3.DatabaseError as exc: + # Same corruption class the main FTS5 MATCH branch + # self-heals above: a corrupt trigram shadow table raises + # malformed / "fts5: corrupt structure record", which is a + # DatabaseError (parent of the OperationalError syntax arm + # caught first). Rebuild once outside the lock — the lock + # is released here so rebuild_fts() can re-acquire it — + # and retry the trigram query. If the rebuild is refused + # (already attempted / FTS disabled / different error + # class) or the retry fails again, fall through to the + # LIKE substring path, which reads only the canonical + # messages table, so CJK search stays available. + if self._try_runtime_fts_rebuild(exc): + try: + with self._read_ctx() as conn: + tri_cursor = conn.execute( + tri_sql, tri_params + ) + matches = [ + dict(row) for row in tri_cursor.fetchall() + ] + _trigram_succeeded = True + except sqlite3.DatabaseError: + logger.warning( + "Trigram FTS search still failing after " + "in-place rebuild; falling back to LIKE." + ) + else: + logger.warning( + "Trigram FTS search hit a corruption error (%s) " + "and no in-place rebuild was possible; falling " + "back to LIKE.", exc, + ) + if not _trigram_succeeded: + # Short / mixed CJK query, trigram unavailable, or trigram + # <3 CJK chars. Fall back to LIKE substring search. + # For multi-token OR queries (e.g. "广西 OR 桂林 OR 漓江"), + # build one LIKE condition per non-operator token so each term + # is matched independently (#20494). + non_op_tokens = [ + t for t in raw_query.split() + if t.upper() not in {"AND", "OR", "NOT"} + ] or [raw_query] + token_clauses = [] + like_params: list = [] + for tok in non_op_tokens: + esc = tok.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + token_clauses.append( + "(m.content LIKE ? ESCAPE '\\' OR m.tool_name LIKE ? ESCAPE '\\' OR m.tool_calls LIKE ? ESCAPE '\\')" + ) + like_params += [f"%{esc}%", f"%{esc}%", f"%{esc}%"] + like_where = [f"({' OR '.join(token_clauses)})"] + if not include_inactive: + # Same visibility rule as the FTS5 paths: live rows and + # compaction-archived rows are discoverable; rewind/undo + # rows (active=0, compacted=0) are hidden (#38763). + like_where.append("(m.active = 1 OR m.compacted = 1)") + if source_filter is not None: + like_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") + like_params.extend(source_filter) + if exclude_sources is not None: + like_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") + like_params.extend(exclude_sources) + if role_filter: + like_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") + like_params.extend(role_filter) + like_sql = f""" + SELECT m.id, m.session_id, m.role, + substr(m.content, + max(1, instr(m.content, ?) - 40), + 120) AS snippet, + m.content, m.timestamp, m.tool_name, + s.source, s.model, s.started_at AS session_started + FROM messages m + JOIN sessions s ON s.id = m.session_id + WHERE {' AND '.join(like_where)} + ORDER BY m.timestamp DESC + LIMIT ? OFFSET ? + """ + like_params.extend([limit, offset]) + # instr() for snippet uses first search token + like_params = [non_op_tokens[0]] + like_params + with self._read_ctx() as conn: + like_cursor = conn.execute(like_sql, like_params) + matches = [dict(row) for row in like_cursor.fetchall()] + else: + try: + with self._read_ctx() as conn: + cursor = conn.execute(sql, params) + matches = [dict(row) for row in cursor.fetchall()] + except sqlite3.OperationalError: + # FTS5 query syntax error despite sanitization — return empty + return [] + except sqlite3.DatabaseError as exc: + # A corrupt FTS index raises the malformed / "fts5: corrupt + # structure record" class on the MATCH read, the same class the + # write path self-heals (#66296). OperationalError (query + # syntax) is a subclass caught above; this arm is the corruption + # parent. Rebuild the index in place once — the read context + # holds no writer lock, so rebuild_fts() can acquire it — and + # retry, so search self-heals for read-only sessions (cron/CLI + # history search) that never trigger a write to repair it first. + if not self._try_runtime_fts_rebuild(exc): + raise + with self._read_ctx() as conn: + cursor = conn.execute(sql, params) + matches = [dict(row) for row in cursor.fetchall()] + + # Deferred-rebuild supplement (schema v23): while the background + # backfill is pending, the FTS indexes only cover rows outside the + # (progress, high_water] gap. Top the results up with a bounded LIKE + # scan over just that id range so search never silently loses old + # messages mid-rebuild. The range shrinks as the backfill advances, + # so this cost decays to zero. The CJK LIKE-fallback path above + # already scans the whole base table and needs no supplement. + rebuild_status = self.fts_rebuild_status() + if rebuild_status is not None and len(matches) < limit: + try: + gap_matches = self._search_unindexed_gap( + query, + limit - len(matches), + include_inactive=include_inactive, + source_filter=source_filter, + exclude_sources=exclude_sources, + role_filter=role_filter, + ) + seen_ids = {m["id"] for m in matches} + matches.extend(m for m in gap_matches if m["id"] not in seen_ids) + except sqlite3.OperationalError as exc: + logger.debug("Unindexed-gap supplement skipped: %s", exc) + + # Pure-Latin queries run against the unicode61 ``messages_fts`` table, + # whose tokenizer does not insert a boundary between Latin letters and + # adjacent CJK characters: "修改youer服务端" is indexed as one token, + # so MATCH "youer" finds nothing even though the substring is present + # (#54242). When the exact-token search returns nothing, retry on the + # substring-capable indexes. Preference order: + # 1. messages_fts_cjk (when built): its tokenizer splits Latin runs + # off adjacent CJK, so "youer" is an exact ranked token match. + # 2. messages_fts_trigram: substring matching, needs >=3-char + # tokens (shorter tokens produce no trigrams). + # Gated on a zero-result miss so successful Latin searches keep their + # unicode61 ranking — strictly additive, never reorders existing + # hits. Trade-off on the trigram leg: any zero-result Latin query + # gains substring semantics (e.g. "cat" can then match + # "concatenate"). Genuinely absent terms still return []. Skipped for + # role_filter=['tool'] queries — both fallback indexes exclude tool + # rows (v23), so a retry could never add hits. + if ( + not matches + and not is_cjk + and not (bool(role_filter) and "tool" in role_filter) + ): + _fb_query = query.strip('"').strip() + if self._fts_cjk_available: + cjk_fb = self._run_trigram_search( + _fb_query, + table="messages_fts_cjk", + order_by_sql=order_by_sql, + include_inactive=include_inactive, + source_filter=source_filter, + exclude_sources=exclude_sources, + role_filter=role_filter, + limit=limit, + offset=offset, + ) + if cjk_fb: + matches = cjk_fb + if ( + not matches + and self._trigram_available + and self._trigram_eligible_tokens(query) + ): + tri_matches = self._run_trigram_search( + _fb_query, + order_by_sql=order_by_sql, + include_inactive=include_inactive, + source_filter=source_filter, + exclude_sources=exclude_sources, + role_filter=role_filter, + limit=limit, + offset=offset, + ) + if tri_matches: + matches = tri_matches + + # Add surrounding context (1 message before + after each match). + # Each query takes its own fresh read transaction via _read_ctx, so + # we never hold a lock across N sequential queries. + for match in matches: + try: + with self._read_ctx() as conn: + ctx_cursor = conn.execute( + """WITH target AS ( + SELECT session_id, timestamp, id + FROM messages + WHERE id = ? + ) + SELECT role, content + FROM ( + SELECT m.id, m.timestamp, m.role, m.content + FROM messages m + JOIN target t ON t.session_id = m.session_id + WHERE (m.timestamp < t.timestamp) + OR (m.timestamp = t.timestamp AND m.id < t.id) + ORDER BY m.timestamp DESC, m.id DESC + LIMIT 1 + ) + UNION ALL + SELECT role, content + FROM messages + WHERE id = ? + UNION ALL + SELECT role, content + FROM ( + SELECT m.id, m.timestamp, m.role, m.content + FROM messages m + JOIN target t ON t.session_id = m.session_id + WHERE (m.timestamp > t.timestamp) + OR (m.timestamp = t.timestamp AND m.id > t.id) + ORDER BY m.timestamp ASC, m.id ASC + LIMIT 1 + )""", + (match["id"], match["id"]), + ) + context_msgs = [] + for r in ctx_cursor.fetchall(): + raw = r["content"] + decoded = self._decode_content(raw) + # Multimodal context: render a compact text-only + # summary for search previews. + if isinstance(decoded, list): + text_parts = [ + p.get("text", "") for p in decoded + if isinstance(p, dict) and p.get("type") == "text" + ] + text = " ".join(t for t in text_parts if t).strip() + preview = text or "[multimodal content]" + elif isinstance(decoded, str): + preview = decoded + else: + preview = "" + context_msgs.append( + {"role": r["role"], "content": preview[:200]} + ) + match["context"] = context_msgs + except Exception: + match["context"] = [] + + # Remove full content from result (snippet is enough, saves tokens) + for match in matches: + match.pop("content", None) + + return matches + + def _search_unindexed_gap( + self, + fts_query: str, + limit: int, + *, + include_inactive: bool = False, + source_filter: Optional[List[str]] = None, + exclude_sources: Optional[List[str]] = None, + role_filter: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + """LIKE-scan the rows the deferred rebuild hasn't indexed yet. + + Only touches ids in (fts_rebuild_progress, fts_rebuild_high_water] — + a range that shrinks to nothing as the backfill advances. The FTS + query is degraded to per-token substring terms (AND-joined; quoted + phrases kept whole), which is deliberately recall-over-precision: + temporary results beat silently missing ones mid-rebuild. + """ + status = self.fts_rebuild_status() + if status is None or limit <= 0: + return [] + progress, high_water = status["indexed"], status["total"] + + # Degrade the FTS query to LIKE terms: strip operators/wildcards, + # keep quoted phrases intact, AND the rest. + terms: List[str] = [] + for raw_tok in re.findall(r'"[^"]+"|\S+', fts_query): + tok = raw_tok.strip('"').strip("*").strip() + if not tok or tok.upper() in {"AND", "OR", "NOT", "NEAR"}: + continue + terms.append(tok) + if not terms: + return [] + + where = ["m.id > ? AND m.id <= ?"] + params: list = [progress, high_water] + for term in terms: + esc = term.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + where.append( + "(m.content LIKE ? ESCAPE '\\' OR m.tool_name LIKE ? ESCAPE '\\' " + "OR m.tool_calls LIKE ? ESCAPE '\\')" + ) + params += [f"%{esc}%"] * 3 + if not include_inactive: + where.append("(m.active = 1 OR m.compacted = 1)") + if source_filter is not None: + where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") + params.extend(source_filter) + if exclude_sources is not None: + where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") + params.extend(exclude_sources) + if role_filter: + where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") + params.extend(role_filter) + + sql = f""" + SELECT m.id, m.session_id, m.role, + substr(m.content, + max(1, instr(m.content, ?) - 40), + 120) AS snippet, + m.content, m.timestamp, m.tool_name, + s.source, s.model, s.started_at AS session_started + FROM messages m + JOIN sessions s ON s.id = m.session_id + WHERE {' AND '.join(where)} + ORDER BY m.timestamp DESC + LIMIT ? + """ + params = [terms[0]] + params + [limit] + with self._read_ctx() as conn: + rows = conn.execute(sql, params).fetchall() + return [dict(r) for r in rows] + + def search_sessions_by_id( + self, + query: str, + limit: int = 20, + include_archived: bool = True, + source: str = None, + sources: List[str] = None, + exclude_sources: List[str] = None, + ) -> List[Dict[str, Any]]: + """Search surfaced sessions by exact/prefix/substring session id. + + Desktop search uses this alongside FTS message search so users can paste + a session id from logs, CLI output, or another Hermes surface and jump + straight to that conversation. Matching also checks ``_lineage_root_id`` + for projected compression-chain tips, so an old root id still resolves to + the live continuation row. + """ + needle = (query or "").strip().lower() + if not needle or limit <= 0: + return [] + + # SQL-bounded: list_sessions_rich pushes the id LIKE filter into the + # query (matching the row's own id AND any id in its forward + # compression chain), so we only materialize matching rows instead of + # scanning every session. Fetch a small multiple of `limit` so the + # in-Python exact/prefix/substring ranking below has enough candidates + # to order, then truncate. + candidates = self.list_sessions_rich( + source=source, + sources=sources, + exclude_sources=exclude_sources, + limit=max(limit * 4, limit), + offset=0, + include_archived=include_archived, + order_by_last_active=True, + id_query=needle, + ) + + def score(row: Dict[str, Any]) -> int: + ids = [str(row.get("id") or ""), str(row.get("_lineage_root_id") or "")] + normalized = [value.lower() for value in ids if value] + if any(value == needle for value in normalized): + return 0 + if any(value.startswith(needle) for value in normalized): + return 1 + return 2 + + ranked = sorted( + enumerate(candidates), + key=lambda item: (score(item[1]), item[0]), + ) + return [row for _, row in ranked[:limit]] + + def _fts_table_exists(self, name: str) -> bool: + """True if an FTS5 virtual table is queryable in this DB.""" + try: + self._conn.execute(f"SELECT 1 FROM {name} LIMIT 0") + return True + except sqlite3.DatabaseError: + # OperationalError ("no such table") or the broader + # DatabaseError class ("vtable constructor failed", raised when + # e.g. a required tokenizer is missing or the table is mid- + # teardown) — in every case the table is not queryable. + return False + + def optimize_fts(self) -> int: + """Merge fragmented FTS5 b-tree segments into one per index. + + FTS5 indexes grow as a series of incremental segments — one per + ``INSERT`` batch driven by the message triggers. Over tens of + thousands of messages these segments accumulate, which both bloats + the ``*_data`` shadow tables and slows ``MATCH`` queries that must + scan every segment. The special ``'optimize'`` command rewrites each + index as a single merged segment. + + This is purely a maintenance operation — it changes neither search + results nor ``snippet()`` output, only on-disk layout and query + speed. It is complementary to VACUUM: ``optimize`` compacts the FTS + index internally, then VACUUM returns the freed pages to the OS. + + Skips any FTS table that does not exist (e.g. the trigram index when + disabled via ``HERMES_DISABLE_FTS_TRIGRAM`` or not yet created), so + it is safe to call unconditionally. + + Returns the number of FTS indexes that were optimized. + """ + optimized = 0 + with self._lock: + for tbl in self._FTS_TABLES: + if not self._fts_table_exists(tbl): + continue + try: + # The column name in the INSERT must match the table name + # for FTS5 special commands. + self._conn.execute( + f"INSERT INTO {tbl}({tbl}) VALUES('optimize')" + ) + optimized += 1 + except sqlite3.OperationalError as exc: + logger.warning( + "FTS optimize failed for %s: %s", tbl, exc + ) + return optimized + + def rebuild_fts(self) -> int: + """Rebuild FTS5 indexes from the canonical ``messages`` table. + + Uses the FTS5 ``'rebuild'`` command, which rewrites the internal + b-tree segments from the content rows. This is the documented + recovery for a corrupt FTS index that rejects message writes while + reads still succeed (issue #50502). Unlike ``optimize_fts`` (which + merges existing segments), ``rebuild`` discards and recreates the + index data entirely. + + Safe to call when FTS tables don't exist (skips them). + Returns the number of FTS indexes that were rebuilt. + """ + rebuilt = 0 + with self._lock: + for tbl in self._FTS_TABLES: + if not self._fts_table_exists(tbl): + continue + try: + self._conn.execute( + f"INSERT INTO {tbl}({tbl}) VALUES('rebuild')" + ) + self._conn.commit() + rebuilt += 1 + except sqlite3.OperationalError as exc: + self._conn.rollback() + logger.warning( + "FTS rebuild failed for %s: %s", tbl, exc + ) + return rebuilt + + def _merge_fts_incrementally( + self, *, max_pages: int, max_commands: Optional[int] = None + ) -> int: + """Run bounded FTS5 ``'merge'`` commands against each present index. + + A positive merge rank tells SQLite to stop after approximately that + many output pages, so each command holds the write lock for + milliseconds regardless of index size — unlike ``'optimize'``, which + rewrites the whole index in one transaction (measured 9-18 s per + index on a 10 GB production DB, long enough to exhaust a competing + writer's entire lock-retry patience). + + Protocol (SQLite FTS5 §6.8-6.9): + + - ``usermerge`` is lowered to its minimum of 2 (persisted in the + ``%_config`` shadow table, applied once per instance) so a + positive merge acts on ANY level holding >= 2 segments. With the + default of 4, levels below that threshold are never merged by a + positive-rank command and a fragmented index cannot converge. + - Up to *max_commands* merge commands run per index, stopping early + on the documented no-progress signal: the delta in + ``total_changes`` is < 2 (the command's own INSERT accounts + for 1 change; >= 2 means real merge work happened). + + Each command is its own implicit transaction (the connection runs + with ``isolation_level=None``), so the SQLite write lock is released + between commands and competing processes can interleave writes + mid-pass. Missing tables are valid schema variants (FTS variants are + optional, and ``optimize_fts_storage`` legitimately drops + backfills + these tables while writers keep running) and are skipped, mirroring + ``optimize_fts``. Other SQLite errors propagate to the caller. + + Returns the number of merge commands executed. + """ + if isinstance(max_pages, bool) or not isinstance(max_pages, int): + raise TypeError("max_pages must be an integer") + if max_pages <= 0: + raise ValueError("max_pages must be greater than zero") + if max_commands is None: + max_commands = self._FTS_MERGE_COMMANDS_PER_PASS + if isinstance(max_commands, bool) or not isinstance(max_commands, int): + raise TypeError("max_commands must be an integer") + if max_commands <= 0: + raise ValueError("max_commands must be greater than zero") + + executed = 0 + with self._lock: + for tbl in self._FTS_TABLES: + if not self._fts_table_exists(tbl): + continue + # One-time (per instance) usermerge floor; the value is + # persisted in the index's config shadow table so future + # connections inherit it. Setting config is a metadata-only + # write — it never touches segment data. + if not getattr(self, "_fts_usermerge_floor_applied", False): + self._conn.execute( + f"INSERT INTO {tbl}({tbl}, rank) " + "VALUES('usermerge', 2)" + ) + for _ in range(max_commands): + before = self._conn.total_changes + self._conn.execute( + f"INSERT INTO {tbl}({tbl}, rank) VALUES('merge', ?)", + (max_pages,), + ) + executed += 1 + if self._conn.total_changes - before < 2: + break + self._fts_usermerge_floor_applied = True + return executed