mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-30 19:09:28 +00:00
779 lines
39 KiB
Python
779 lines
39 KiB
Python
"""Schema creation, column reconciliation, and FTS DDL management for SessionDB.
|
|
|
|
Mixin contract: this is a plain mixin class consumed by
|
|
``hermes_state.SessionDB``. It defines no ``__init__`` and no state of its
|
|
own; methods access the host's attributes (``self._conn``, ``self.db_path``,
|
|
``self._execute_write`` and other SessionDB methods) established by
|
|
``SessionDB.__init__``. It must never import hermes_state (cycle) — shared
|
|
module-level constants live in hermes_state_common.
|
|
"""
|
|
|
|
import logging
|
|
import json
|
|
import sqlite3
|
|
from typing import Dict, Optional
|
|
|
|
from hermes_constants import get_hermes_home
|
|
from hermes_state_common import (
|
|
DEFERRED_INDEX_SQL,
|
|
FTS_SQL,
|
|
FTS_STORAGE_VERSION,
|
|
FTS_TRIGRAM_SQL,
|
|
LEGACY_FTS_SQL,
|
|
LEGACY_FTS_TRIGRAM_SQL,
|
|
SCHEMA_SQL,
|
|
SCHEMA_VERSION,
|
|
_FTS_TRIGGERS,
|
|
_ephemeral_child_sql,
|
|
)
|
|
|
|
# Moved methods logged under the "hermes_state" logger before the split;
|
|
# keep that logger identity so log filtering/capture behavior is unchanged.
|
|
logger = logging.getLogger("hermes_state")
|
|
|
|
|
|
class SessionSchemaMixin:
|
|
"""See module docstring — mixin for SessionDB (Schema cluster)."""
|
|
|
|
def _sqlite_supports_fts5(self, cursor: sqlite3.Cursor) -> bool:
|
|
try:
|
|
cursor.execute("CREATE VIRTUAL TABLE temp._hermes_fts5_probe USING fts5(x)")
|
|
cursor.execute("DROP TABLE temp._hermes_fts5_probe")
|
|
return True
|
|
except sqlite3.OperationalError as exc:
|
|
if not self._is_fts5_unavailable_error(exc):
|
|
raise
|
|
self._warn_fts5_unavailable(exc)
|
|
return False
|
|
|
|
@staticmethod
|
|
def _fts_trigger_count(cursor: sqlite3.Cursor) -> int:
|
|
placeholders = ",".join("?" for _ in _FTS_TRIGGERS)
|
|
row = cursor.execute(
|
|
f"SELECT COUNT(*) FROM sqlite_master "
|
|
f"WHERE type = 'trigger' AND name IN ({placeholders})",
|
|
_FTS_TRIGGERS,
|
|
).fetchone()
|
|
return int(row[0] if not isinstance(row, sqlite3.Row) else row[0])
|
|
|
|
@staticmethod
|
|
def _rebuild_fts_indexes(
|
|
cursor: sqlite3.Cursor,
|
|
*,
|
|
include_trigram: bool = True,
|
|
) -> None:
|
|
# Both FTS tables are external-content (v23+): the special 'rebuild'
|
|
# command wipes the inverted index and repopulates it from the
|
|
# content source (messages for the standard index, the tool-row-
|
|
# excluding messages_fts_trigram_src view for the trigram index).
|
|
cursor.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')")
|
|
if include_trigram:
|
|
cursor.execute(
|
|
"INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('rebuild')"
|
|
)
|
|
# 'rebuild' indexes EVERY row, so any deferred-backfill markers are
|
|
# now satisfied — clear them, otherwise the background worker would
|
|
# re-insert rows the rebuild already covered (duplicate entries).
|
|
cursor.execute(
|
|
"DELETE FROM state_meta WHERE key IN "
|
|
"('fts_rebuild_high_water', 'fts_rebuild_progress')"
|
|
)
|
|
|
|
@staticmethod
|
|
def _rebuild_legacy_fts_indexes(
|
|
cursor: sqlite3.Cursor,
|
|
*,
|
|
include_trigram: bool = True,
|
|
) -> None:
|
|
"""Rebuild the LEGACY inline FTS indexes (pre-v23) from messages.
|
|
|
|
Used only to repair a legacy DB whose triggers degraded under an
|
|
earlier no-FTS5 runtime. Inline tables have no external-content
|
|
'rebuild' source, so we DELETE + reinsert the concatenated content
|
|
the legacy triggers produced. Never touches the v23 shape.
|
|
"""
|
|
cursor.execute("DELETE FROM messages_fts")
|
|
cursor.execute(
|
|
"INSERT INTO messages_fts(rowid, content) "
|
|
"SELECT id, "
|
|
"COALESCE(content, '') || ' ' || "
|
|
"COALESCE(tool_name, '') || ' ' || "
|
|
"COALESCE(tool_calls, '') "
|
|
"FROM messages"
|
|
)
|
|
if not include_trigram:
|
|
return
|
|
cursor.execute("DELETE FROM messages_fts_trigram")
|
|
cursor.execute(
|
|
"INSERT INTO messages_fts_trigram(rowid, content) "
|
|
"SELECT id, "
|
|
"COALESCE(content, '') || ' ' || "
|
|
"COALESCE(tool_name, '') || ' ' || "
|
|
"COALESCE(tool_calls, '') "
|
|
"FROM messages"
|
|
)
|
|
|
|
def _fts_table_probe(self, cursor: sqlite3.Cursor, table_name: str) -> Optional[bool]:
|
|
try:
|
|
cursor.execute(f"SELECT * FROM {table_name} LIMIT 0")
|
|
return True
|
|
except sqlite3.OperationalError as exc:
|
|
if self._is_fts5_unavailable_error(exc):
|
|
# Only disable FTS entirely when the whole module is missing.
|
|
# A missing trigram tokenizer only affects trigram searches.
|
|
if self._is_trigram_unavailable_error(exc):
|
|
self._warn_trigram_unavailable(exc)
|
|
else:
|
|
self._warn_fts5_unavailable(exc)
|
|
return None
|
|
if "no such table" in str(exc).lower():
|
|
return False
|
|
raise
|
|
|
|
@staticmethod
|
|
def _parse_schema_columns(schema_sql: str) -> Dict[str, Dict[str, str]]:
|
|
"""Extract expected columns per table from SCHEMA_SQL.
|
|
|
|
Uses an in-memory SQLite database to parse the SQL — SQLite itself
|
|
handles all syntax (DEFAULT expressions with commas, inline
|
|
REFERENCES, CHECK constraints, etc.) so there are zero regex
|
|
edge cases. The in-memory DB is opened, the schema DDL is
|
|
executed, and PRAGMA table_info extracts the column metadata.
|
|
|
|
Adding a column to SCHEMA_SQL is all that's needed; the
|
|
reconciliation loop picks it up automatically.
|
|
"""
|
|
ref = sqlite3.connect(":memory:")
|
|
try:
|
|
ref.executescript(schema_sql)
|
|
table_columns: Dict[str, Dict[str, str]] = {}
|
|
for (tbl,) in ref.execute(
|
|
"SELECT name FROM sqlite_master "
|
|
"WHERE type='table' AND name NOT LIKE 'sqlite_%'"
|
|
).fetchall():
|
|
cols: Dict[str, str] = {}
|
|
for row in ref.execute(
|
|
f'PRAGMA table_info("{tbl}")'
|
|
).fetchall():
|
|
# row: (cid, name, type, notnull, dflt_value, pk)
|
|
col_name = row[1]
|
|
col_type = row[2] or ""
|
|
notnull = row[3]
|
|
default = row[4]
|
|
pk = row[5]
|
|
# Reconstruct the type expression for ALTER TABLE ADD COLUMN
|
|
parts = [col_type] if col_type else []
|
|
if notnull and not pk:
|
|
parts.append("NOT NULL")
|
|
if default is not None:
|
|
parts.append(f"DEFAULT {default}")
|
|
cols[col_name] = " ".join(parts)
|
|
table_columns[tbl] = cols
|
|
return table_columns
|
|
finally:
|
|
ref.close()
|
|
|
|
def _reconcile_columns(self, cursor: sqlite3.Cursor) -> None:
|
|
"""Ensure live tables have every column declared in SCHEMA_SQL.
|
|
|
|
Follows the Beets/sqlite-utils pattern: the CREATE TABLE definition
|
|
in SCHEMA_SQL is the single source of truth for the desired schema.
|
|
On every startup this method diffs the live columns (via PRAGMA
|
|
table_info) against the declared columns, and ADDs any that are
|
|
missing.
|
|
|
|
This makes column additions a declarative operation — just add
|
|
the column to SCHEMA_SQL and it appears on the next startup.
|
|
Version-gated migration blocks are no longer needed for ADD COLUMN.
|
|
"""
|
|
expected = self._parse_schema_columns(SCHEMA_SQL)
|
|
for table_name, declared_cols in expected.items():
|
|
# Get current columns from the live table
|
|
try:
|
|
rows = cursor.execute(
|
|
f'PRAGMA table_info("{table_name}")'
|
|
).fetchall()
|
|
except sqlite3.OperationalError:
|
|
continue # Table doesn't exist yet (shouldn't happen after executescript)
|
|
live_cols = set()
|
|
for row in rows:
|
|
# PRAGMA table_info returns (cid, name, type, notnull, dflt_value, pk)
|
|
name = row[1] if isinstance(row, (tuple, list)) else row["name"]
|
|
live_cols.add(name)
|
|
|
|
for col_name, col_type in declared_cols.items():
|
|
if col_name not in live_cols:
|
|
safe_name = col_name.replace('"', '""')
|
|
try:
|
|
cursor.execute(
|
|
f'ALTER TABLE "{table_name}" ADD COLUMN "{safe_name}" {col_type}'
|
|
)
|
|
except sqlite3.OperationalError as exc:
|
|
# Expected: "duplicate column name" from a race or
|
|
# re-run. Unexpected: "Cannot add a NOT NULL column
|
|
# with default value NULL" from a schema mistake.
|
|
# Log at DEBUG so it's visible in agent.log.
|
|
logger.debug(
|
|
"reconcile %s.%s: %s", table_name, col_name, exc,
|
|
)
|
|
|
|
def _heal_gateway_routing_pk(self, cursor: sqlite3.Cursor) -> None:
|
|
"""Rebuild ``gateway_routing`` when its PRIMARY KEY predates scoping.
|
|
|
|
Early builds of the routing-index migration (#59203) created the
|
|
table with ``session_key TEXT PRIMARY KEY`` and no ``scope`` column.
|
|
``_reconcile_columns()`` ADDs the missing ``scope`` column on those
|
|
databases, but SQLite cannot ALTER a primary key, so the shipped
|
|
composite ``PRIMARY KEY (scope, session_key)`` never lands. On such
|
|
tables every write path is broken:
|
|
|
|
* ``save_gateway_routing_entry`` fails with "ON CONFLICT clause does
|
|
not match any PRIMARY KEY or UNIQUE constraint" (its upsert targets
|
|
the composite key), and
|
|
* ``replace_gateway_routing_entries`` fails with "UNIQUE constraint
|
|
failed: gateway_routing.session_key" whenever the same session_key
|
|
exists under a different scope — the exact isolation the composite
|
|
key exists to provide.
|
|
|
|
Each failed save logs a warning and falls back to sessions.json,
|
|
so a legacy-shaped table produces endless per-save warning spam.
|
|
Rebuild it once, preserving rows. On a session_key collision across
|
|
scopes (possible while the PK was wrong) the newest row wins.
|
|
"""
|
|
try:
|
|
rows = cursor.execute(
|
|
'PRAGMA table_info("gateway_routing")'
|
|
).fetchall()
|
|
except sqlite3.OperationalError:
|
|
return
|
|
if not rows:
|
|
return
|
|
|
|
def _col(row, idx, name):
|
|
return row[idx] if isinstance(row, (tuple, list)) else row[name]
|
|
|
|
pk_cols = [
|
|
_col(r, 1, "name")
|
|
for r in sorted(
|
|
(r for r in rows if _col(r, 5, "pk")),
|
|
key=lambda r: _col(r, 5, "pk"),
|
|
)
|
|
]
|
|
if pk_cols == ["scope", "session_key"]:
|
|
return
|
|
|
|
logger.info(
|
|
"gateway_routing has legacy primary key %r; rebuilding with "
|
|
"composite (scope, session_key) key",
|
|
pk_cols,
|
|
)
|
|
cursor.execute(
|
|
"ALTER TABLE gateway_routing RENAME TO gateway_routing_legacy_pk"
|
|
)
|
|
cursor.execute(
|
|
"""CREATE TABLE gateway_routing (
|
|
scope TEXT NOT NULL DEFAULT '',
|
|
session_key TEXT NOT NULL,
|
|
entry_json TEXT NOT NULL,
|
|
updated_at REAL NOT NULL,
|
|
PRIMARY KEY (scope, session_key)
|
|
)"""
|
|
)
|
|
# INSERT OR REPLACE + updated_at ordering: if the broken PK ever let
|
|
# two scopes race over one session_key, keep the newest row per
|
|
# (scope, session_key) pair.
|
|
cursor.execute(
|
|
"INSERT OR REPLACE INTO gateway_routing "
|
|
"(scope, session_key, entry_json, updated_at) "
|
|
"SELECT COALESCE(scope, ''), session_key, entry_json, updated_at "
|
|
"FROM gateway_routing_legacy_pk ORDER BY updated_at ASC"
|
|
)
|
|
cursor.execute("DROP TABLE gateway_routing_legacy_pk")
|
|
|
|
def _init_schema(self):
|
|
"""Create tables and FTS if they don't exist, reconcile columns.
|
|
|
|
Schema management follows the declarative reconciliation pattern
|
|
(Beets, sqlite-utils): SCHEMA_SQL is the single source of truth.
|
|
On existing databases, _reconcile_columns() diffs live columns
|
|
against SCHEMA_SQL and ADDs any missing ones. This eliminates
|
|
the version-gated migration chain for column additions, making
|
|
it impossible for reordered or inserted migrations to skip columns.
|
|
|
|
The schema_version table is retained for future data migrations
|
|
(transforming existing rows) which cannot be handled declaratively.
|
|
"""
|
|
cursor = self._conn.cursor()
|
|
|
|
cursor.executescript(SCHEMA_SQL)
|
|
|
|
# ── Declarative column reconciliation ──────────────────────────
|
|
# Diff live tables against SCHEMA_SQL and ADD any missing columns.
|
|
# This is idempotent and self-healing: even if a version-gated
|
|
# migration was skipped (e.g. due to version renumbering), the
|
|
# column gets created here.
|
|
self._reconcile_columns(cursor)
|
|
|
|
# Rebuild gateway_routing if it still carries the pre-scope PRIMARY
|
|
# KEY (session_key alone). ADD COLUMN cannot fix a PK, so this is
|
|
# the one table-shape repair reconciliation can't express.
|
|
self._heal_gateway_routing_pk(cursor)
|
|
|
|
# Indexes that reference reconciler-added columns must be created
|
|
# AFTER _reconcile_columns runs — declaring them in SCHEMA_SQL
|
|
# makes the initial executescript fail on legacy DBs (the index's
|
|
# WHERE clause references a column that doesn't exist yet).
|
|
try:
|
|
cursor.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_messages_platform_msg_id "
|
|
"ON messages(session_id, platform_message_id) "
|
|
"WHERE platform_message_id IS NOT NULL"
|
|
)
|
|
except sqlite3.OperationalError as exc:
|
|
logger.debug("idx_messages_platform_msg_id create skipped: %s", exc)
|
|
|
|
# Deferred indexes that reference the reconciler-added ``active``
|
|
# column (idx_messages_session_active) — same ordering constraint.
|
|
cursor.executescript(DEFERRED_INDEX_SQL)
|
|
|
|
# Heal NULL ``active`` rows unconditionally on every startup.
|
|
# On real-world DBs the reconciler-added ``active`` column can lack
|
|
# its NOT NULL DEFAULT 1 (older reconciler builds reconstructed the
|
|
# type without the default — see #51646: PRAGMA shows
|
|
# (17,'active','INTEGER',0,None,0) in the wild), so INSERTs that
|
|
# omitted the column wrote NULL and the ``WHERE active = 1``
|
|
# transcript loaders hid the whole history. The INSERTs now set
|
|
# active=1 explicitly; this idempotent repair un-hides rows written
|
|
# before the fix. It was previously gated at ``current_version <
|
|
# 12`` which never re-ran for already-v12+ databases.
|
|
try:
|
|
cursor.execute(
|
|
"UPDATE messages SET active = 1 WHERE active IS NULL"
|
|
)
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
|
|
fts5_available = self._sqlite_supports_fts5(cursor)
|
|
fts_migrations_complete = True
|
|
if not fts5_available:
|
|
# Existing FTS triggers can still fire on messages INSERT/UPDATE
|
|
# even though the current sqlite runtime cannot read the virtual
|
|
# tables they target. Drop only the triggers so core persistence
|
|
# continues; if a future runtime has FTS5, _ensure_fts_schema()
|
|
# recreates them.
|
|
self._drop_fts_triggers(cursor)
|
|
|
|
# ── Schema version bookkeeping ─────────────────────────────────
|
|
# Bump to current so future data migrations (if any) can gate on
|
|
# version. No version-gated column additions remain.
|
|
cursor.execute("SELECT version FROM schema_version LIMIT 1")
|
|
row = cursor.fetchone()
|
|
if row is None:
|
|
cursor.execute(
|
|
"INSERT INTO schema_version (version) VALUES (?)",
|
|
(SCHEMA_VERSION,),
|
|
)
|
|
else:
|
|
current_version = row["version"] if isinstance(row, sqlite3.Row) else row[0]
|
|
# Data migrations that can't be expressed declaratively (row
|
|
# backfills, index changes tied to a specific version step) stay
|
|
# in a version-gated chain. Column additions are handled by
|
|
# _reconcile_columns() above and no longer need entries here.
|
|
if current_version < 10 and SCHEMA_VERSION == 10:
|
|
# v10: trigram FTS5 table for CJK/substring search. The
|
|
# virtual table + triggers are created unconditionally via
|
|
# FTS_TRIGRAM_SQL below, but existing rows need a one-time
|
|
# backfill into the FTS index.
|
|
#
|
|
# Only run this when v10 itself is the target schema. Current
|
|
# v11+ code drops and rebuilds both FTS tables below, so doing
|
|
# the v10-only trigram backfill first only burns startup time
|
|
# and WAL space before v11 throws the work away.
|
|
if fts5_available:
|
|
_fts_trigram_exists = self._fts_table_probe(
|
|
cursor, "messages_fts_trigram"
|
|
)
|
|
if _fts_trigram_exists is False:
|
|
if self._ensure_fts_schema(
|
|
cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL
|
|
):
|
|
cursor.execute(
|
|
"INSERT INTO messages_fts_trigram(rowid, content) "
|
|
"SELECT id, content FROM messages WHERE content IS NOT NULL"
|
|
)
|
|
else:
|
|
fts_migrations_complete = False
|
|
elif _fts_trigram_exists is None:
|
|
fts_migrations_complete = False
|
|
else:
|
|
fts_migrations_complete = False
|
|
if current_version < 11 and SCHEMA_VERSION < 23:
|
|
# v11 (SUPERSEDED by v23): re-index FTS5 tables to cover
|
|
# tool_name + tool_calls in inline mode (#16751). v23 drops
|
|
# and rebuilds both FTS tables in external-content form, so
|
|
# running the v11 inline backfill first would only burn
|
|
# startup time and WAL space before v23 throws the work
|
|
# away — and its inline INSERT shape no longer matches the
|
|
# current external-content FTS_SQL anyway. Kept only for
|
|
# source archaeology; unreachable while SCHEMA_VERSION >= 23.
|
|
pass
|
|
if current_version < 16:
|
|
# v16: tag delegate subagent rows so pickers stay clean after
|
|
# parent deletes that used to orphan them (parent_session_id → NULL).
|
|
try:
|
|
cursor.execute(
|
|
"UPDATE sessions SET model_config = json_set("
|
|
"COALESCE(model_config, '{}'), '$._delegate_from', parent_session_id) "
|
|
f"WHERE parent_session_id IS NOT NULL "
|
|
"AND json_extract(COALESCE(model_config, '{}'), '$._delegate_from') IS NULL "
|
|
f"AND {_ephemeral_child_sql('sessions')}"
|
|
)
|
|
cursor.execute(
|
|
"UPDATE sessions SET model_config = json_set("
|
|
"COALESCE(model_config, '{}'), '$._delegate_from', '__orphaned__') "
|
|
"WHERE parent_session_id IS NULL "
|
|
"AND json_extract(COALESCE(model_config, '{}'), '$._delegate_from') IS NULL "
|
|
"AND json_extract(COALESCE(model_config, '{}'), '$._branched_from') IS NULL "
|
|
"AND title IS NULL "
|
|
"AND message_count <= 25 "
|
|
"AND EXISTS (SELECT 1 FROM messages m "
|
|
" WHERE m.session_id = sessions.id AND m.role = 'tool') "
|
|
"AND NOT EXISTS (SELECT 1 FROM sessions ch "
|
|
" WHERE ch.parent_session_id = sessions.id)"
|
|
)
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
if current_version < 18:
|
|
# v18: gateway metadata consolidation (#9006). Backfill
|
|
# display_name / origin_json / expiry_finalized from
|
|
# sessions.json so pre-migration gateway sessions are
|
|
# discoverable from state.db without the JSON index.
|
|
try:
|
|
self._backfill_gateway_metadata_from_sessions_json(cursor)
|
|
except Exception as exc:
|
|
# Backfill is best-effort: sessions.json may be absent,
|
|
# corrupted, or partially stale. Missing metadata simply
|
|
# means consumers fall back to sessions.json for those
|
|
# rows until the gateway rewrites them.
|
|
logger.debug("v18 gateway metadata backfill skipped: %s", exc)
|
|
if current_version < 20:
|
|
# v20: per-model usage attribution (issue #51607). Going
|
|
# forward update_token_counts() records each API call into
|
|
# session_model_usage keyed by the live model, but existing
|
|
# sessions only have their aggregate totals on the sessions
|
|
# row. Seed one usage row per historical session from those
|
|
# aggregates so insights reads uniformly from the new table.
|
|
# INSERT OR IGNORE keeps it idempotent: if newer code already
|
|
# wrote a (session_id, model, provider) row for a session, the
|
|
# PK conflict skips the stale aggregate rather than doubling it.
|
|
try:
|
|
cursor.execute(
|
|
"""INSERT OR IGNORE INTO session_model_usage (
|
|
session_id, model, billing_provider,
|
|
billing_base_url, billing_mode,
|
|
api_call_count, input_tokens,
|
|
output_tokens, cache_read_tokens,
|
|
cache_write_tokens, reasoning_tokens,
|
|
estimated_cost_usd, actual_cost_usd,
|
|
cost_status, cost_source, first_seen, last_seen
|
|
)
|
|
SELECT id, COALESCE(model, 'unknown'),
|
|
COALESCE(billing_provider, ''),
|
|
COALESCE(billing_base_url, ''),
|
|
COALESCE(billing_mode, ''),
|
|
COALESCE(api_call_count, 0),
|
|
COALESCE(input_tokens, 0),
|
|
COALESCE(output_tokens, 0),
|
|
COALESCE(cache_read_tokens, 0),
|
|
COALESCE(cache_write_tokens, 0),
|
|
COALESCE(reasoning_tokens, 0),
|
|
COALESCE(estimated_cost_usd, 0),
|
|
COALESCE(actual_cost_usd, 0),
|
|
cost_status, cost_source,
|
|
started_at, COALESCE(ended_at, started_at)
|
|
FROM sessions
|
|
WHERE COALESCE(input_tokens, 0)
|
|
+ COALESCE(output_tokens, 0)
|
|
+ COALESCE(cache_read_tokens, 0)
|
|
+ COALESCE(cache_write_tokens, 0)
|
|
+ COALESCE(reasoning_tokens, 0) > 0"""
|
|
)
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
if current_version < 22:
|
|
# v22: task-dimension usage attribution (issue #23270).
|
|
# session_model_usage gains a ``task`` column ('' = main agent
|
|
# loop; 'vision'/'compression'/'title_generation'/... =
|
|
# auxiliary calls) so aux model spend is visible in analytics.
|
|
# The column participates in the PRIMARY KEY and SQLite cannot
|
|
# ALTER a PK, so rebuild the table. The reconciler will have
|
|
# already ADDed the plain column on legacy DBs (harmless);
|
|
# the rebuild bakes it into the PK properly. Existing rows are
|
|
# main-loop accounting by definition → task=''.
|
|
try:
|
|
legacy_pk = cursor.execute(
|
|
"SELECT COUNT(*) FROM pragma_table_info('session_model_usage') "
|
|
"WHERE name = 'task' AND pk > 0"
|
|
).fetchone()[0]
|
|
if not legacy_pk:
|
|
cursor.execute("ALTER TABLE session_model_usage RENAME TO session_model_usage_v21")
|
|
cursor.execute(
|
|
"""CREATE TABLE session_model_usage (
|
|
session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
|
|
model TEXT NOT NULL,
|
|
billing_provider TEXT NOT NULL DEFAULT '',
|
|
billing_base_url TEXT NOT NULL DEFAULT '',
|
|
billing_mode TEXT NOT NULL DEFAULT '',
|
|
task TEXT NOT NULL DEFAULT '',
|
|
api_call_count INTEGER NOT NULL DEFAULT 0,
|
|
input_tokens INTEGER NOT NULL DEFAULT 0,
|
|
output_tokens INTEGER NOT NULL DEFAULT 0,
|
|
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
|
|
cache_write_tokens INTEGER NOT NULL DEFAULT 0,
|
|
reasoning_tokens INTEGER NOT NULL DEFAULT 0,
|
|
estimated_cost_usd REAL NOT NULL DEFAULT 0,
|
|
actual_cost_usd REAL NOT NULL DEFAULT 0,
|
|
cost_status TEXT,
|
|
cost_source TEXT,
|
|
first_seen REAL,
|
|
last_seen REAL,
|
|
PRIMARY KEY (session_id, model, billing_provider, billing_base_url, billing_mode, task)
|
|
)"""
|
|
)
|
|
cursor.execute(
|
|
"""INSERT INTO session_model_usage (
|
|
session_id, model, billing_provider, billing_base_url,
|
|
billing_mode, task, api_call_count, input_tokens,
|
|
output_tokens, cache_read_tokens, cache_write_tokens,
|
|
reasoning_tokens, estimated_cost_usd, actual_cost_usd,
|
|
cost_status, cost_source, first_seen, last_seen
|
|
)
|
|
SELECT session_id, model, billing_provider, billing_base_url,
|
|
billing_mode, '', api_call_count, input_tokens,
|
|
output_tokens, cache_read_tokens, cache_write_tokens,
|
|
reasoning_tokens, estimated_cost_usd, actual_cost_usd,
|
|
cost_status, cost_source, first_seen, last_seen
|
|
FROM session_model_usage_v21"""
|
|
)
|
|
cursor.execute("DROP TABLE session_model_usage_v21")
|
|
cursor.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_session_model_usage_session "
|
|
"ON session_model_usage(session_id)"
|
|
)
|
|
cursor.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_session_model_usage_model "
|
|
"ON session_model_usage(model)"
|
|
)
|
|
except sqlite3.OperationalError as exc:
|
|
logger.debug("v22 session_model_usage rebuild skipped: %s", exc)
|
|
if current_version < 23:
|
|
# v23: FTS storage redesign (issues #22478, #43690, #55233).
|
|
# The v11 inline-mode FTS tables each store a full private
|
|
# copy of every message (content || tool_name || tool_calls),
|
|
# and the trigram index additionally covers role='tool' rows
|
|
# (~90% of message bytes: base64 payloads, file dumps) at
|
|
# ~2.6x amplification — together ~75% of state.db on heavy
|
|
# installs (observed: 18.9 GB of a 25 GB DB).
|
|
#
|
|
# OPT-IN, NOT AUTOMATIC. The transition (demote old vtables →
|
|
# new external-content schema → backfill → teardown → VACUUM)
|
|
# is disk-heavy (transient ~2x file size to fully reclaim via
|
|
# VACUUM) and long (~1-2h background on a 25 GB DB). Doing it
|
|
# silently on every big user's next open — with a completeness
|
|
# guarantee that depends on the process staying alive long
|
|
# enough — is the wrong default. So on an EXISTING install we
|
|
# touch nothing here: the v22 inline FTS keeps working exactly
|
|
# as before, and we only record a flag advertising that the
|
|
# optimization is available. `hermes sessions optimize-storage`
|
|
# performs the whole transition as one deliberate, disk-checked,
|
|
# progress-reported foreground operation.
|
|
#
|
|
# DECOUPLED VERSIONING. Crucially, this does NOT hold back the
|
|
# main schema_version. The FTS storage LAYOUT is tracked by an
|
|
# independent `fts_storage_version` marker (see
|
|
# _fts_storage_version / SETTLE below), so schema_version
|
|
# advances to SCHEMA_VERSION here like every other migration —
|
|
# future v24+ migrations land automatically for legacy-FTS
|
|
# users too. Only the FTS *layout* waits for opt-in.
|
|
if fts5_available and self._db_has_legacy_inline_fts(cursor):
|
|
self.set_meta("fts_optimize_available", "1", cursor=cursor)
|
|
|
|
# The FTS storage layout is versioned independently of the main
|
|
# schema (see the v23 note above). Stamp the current layout so the
|
|
# main version can always advance: a fresh/optimized DB is at
|
|
# FTS_STORAGE_VERSION; a legacy DB is left at whatever it had
|
|
# (absent/0) until `optimize-storage` runs. An INTERRUPTED
|
|
# optimize (legacy vtables already demoted, but rebuild markers
|
|
# or demoted trash tables still present) is NOT stamped either —
|
|
# the marker is the source of truth for "fully optimized", and
|
|
# `fts_optimize_available()` keeps offering the resume until the
|
|
# transition actually completes.
|
|
if (
|
|
fts5_available
|
|
and not self._db_has_legacy_inline_fts(cursor)
|
|
and cursor.execute(
|
|
"SELECT 1 FROM state_meta "
|
|
"WHERE key = 'fts_rebuild_high_water' LIMIT 1"
|
|
).fetchone() is None
|
|
and not self._has_fts_trash(cursor)
|
|
):
|
|
self.set_meta(
|
|
"fts_storage_version", str(FTS_STORAGE_VERSION), cursor=cursor
|
|
)
|
|
|
|
# Advance schema_version to current for ALL non-FTS-layout
|
|
# migrations. This is deliberately NOT gated on the FTS opt-in —
|
|
# holding the whole version back would block every future schema
|
|
# migration for a user who never optimizes. FTS5 being unavailable
|
|
# is the one case we skip (we can't have created the current FTS
|
|
# objects, so claiming the current schema would be a lie).
|
|
if (
|
|
current_version < SCHEMA_VERSION
|
|
and fts_migrations_complete
|
|
and fts5_available
|
|
):
|
|
cursor.execute(
|
|
"UPDATE schema_version SET version = ?",
|
|
(SCHEMA_VERSION,),
|
|
)
|
|
|
|
# Unique title index — always ensure it exists. Older databases may
|
|
# contain duplicate aliases from before the constraint was enforced;
|
|
# preserve every session while letting the newest one retain the alias.
|
|
title_index_sql = (
|
|
"CREATE UNIQUE INDEX IF NOT EXISTS idx_sessions_title_unique "
|
|
"ON sessions(title) WHERE title IS NOT NULL"
|
|
)
|
|
try:
|
|
cursor.execute(title_index_sql)
|
|
except sqlite3.IntegrityError:
|
|
# The index is an optimization — its creation must never abort
|
|
# opening the database, so the repair itself is also guarded.
|
|
try:
|
|
cursor.execute(
|
|
"""UPDATE sessions AS older
|
|
SET title = NULL
|
|
WHERE title IS NOT NULL
|
|
AND EXISTS (
|
|
SELECT 1 FROM sessions AS newer
|
|
WHERE newer.title = older.title
|
|
AND newer.rowid > older.rowid
|
|
)"""
|
|
)
|
|
logger.warning(
|
|
"Cleared %d duplicate session title(s) while restoring the unique index",
|
|
cursor.rowcount,
|
|
)
|
|
cursor.execute(title_index_sql)
|
|
except sqlite3.Error:
|
|
logger.exception(
|
|
"Could not repair duplicate session titles; "
|
|
"unique title index not created"
|
|
)
|
|
except sqlite3.OperationalError:
|
|
pass # Index already exists
|
|
|
|
if fts5_available:
|
|
# FTS5 setup. Run the DDL even when the virtual table exists so
|
|
# CREATE TRIGGER IF NOT EXISTS repairs trigger-only degradation from
|
|
# an earlier no-FTS5 runtime.
|
|
#
|
|
# OPT-IN v23 boundary: a legacy v22 install (inline-content FTS,
|
|
# not yet opted into `hermes db optimize`) must keep its EXISTING
|
|
# inline schema + triggers. Running the v23 external-content DDL
|
|
# here would create the trigram source VIEW and leave the DB in a
|
|
# mixed inline/external state. So for a legacy DB we only ensure
|
|
# its inline triggers exist (via the legacy DDL), and skip the
|
|
# v23 view/external tables entirely. Fresh installs and opted-in
|
|
# DBs have no legacy inline FTS, so they get the v23 DDL.
|
|
if self._db_has_legacy_inline_fts(cursor):
|
|
triggers_need_repair = (
|
|
self._fts_trigger_count(cursor) < len(_FTS_TRIGGERS)
|
|
)
|
|
self._fts_enabled = self._ensure_fts_schema(
|
|
cursor, "messages_fts", LEGACY_FTS_SQL
|
|
)
|
|
if self._fts_enabled:
|
|
trigram_enabled = self._ensure_fts_schema(
|
|
cursor, "messages_fts_trigram", LEGACY_FTS_TRIGRAM_SQL
|
|
)
|
|
self._trigram_available = trigram_enabled
|
|
if triggers_need_repair:
|
|
self._rebuild_legacy_fts_indexes(
|
|
cursor, include_trigram=trigram_enabled
|
|
)
|
|
else:
|
|
triggers_need_repair = (
|
|
self._fts_trigger_count(cursor) < len(_FTS_TRIGGERS)
|
|
)
|
|
self._fts_enabled = self._ensure_fts_schema(
|
|
cursor, "messages_fts", FTS_SQL
|
|
)
|
|
|
|
# Trigram FTS5 for CJK/substring search. This is optional
|
|
# relative to the main FTS table; if it cannot be created,
|
|
# CJK search falls back to LIKE.
|
|
if self._fts_enabled:
|
|
trigram_enabled = self._ensure_fts_schema(
|
|
cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL
|
|
)
|
|
self._trigram_available = trigram_enabled
|
|
if triggers_need_repair:
|
|
self._rebuild_fts_indexes(
|
|
cursor,
|
|
include_trigram=trigram_enabled,
|
|
)
|
|
# CJK-bigram index (cjk_unicode61). Strictly additive to
|
|
# the surfaces above and gated on the loadable tokenizer:
|
|
self._ensure_fts_cjk_schema(cursor)
|
|
|
|
self._conn.commit()
|
|
|
|
def _backfill_gateway_metadata_from_sessions_json(
|
|
self, cursor: sqlite3.Cursor
|
|
) -> None:
|
|
"""One-time v18 backfill of gateway metadata from sessions.json.
|
|
|
|
Existing gateway sessions predate the display_name / origin_json /
|
|
expiry_finalized columns; copy what sessions.json knows so consumers
|
|
can switch to state.db without losing pre-migration sessions.
|
|
Only fills NULL columns — never overwrites data written by newer code.
|
|
"""
|
|
sessions_file = get_hermes_home() / "sessions" / "sessions.json"
|
|
if not sessions_file.exists():
|
|
return
|
|
with open(sessions_file, "r", encoding="utf-8") as f:
|
|
data = json.load(f)
|
|
if not isinstance(data, dict):
|
|
return
|
|
for key, entry in data.items():
|
|
if str(key).startswith("_") or not isinstance(entry, dict):
|
|
continue
|
|
session_id = entry.get("session_id")
|
|
if not session_id:
|
|
continue
|
|
origin = entry.get("origin")
|
|
cursor.execute(
|
|
"""UPDATE sessions
|
|
SET session_key = COALESCE(session_key, ?),
|
|
chat_id = COALESCE(chat_id, ?),
|
|
chat_type = COALESCE(chat_type, ?),
|
|
thread_id = COALESCE(thread_id, ?),
|
|
display_name = COALESCE(display_name, ?),
|
|
origin_json = COALESCE(origin_json, ?),
|
|
expiry_finalized = CASE
|
|
WHEN COALESCE(expiry_finalized, 0) = 0 AND ? = 1 THEN 1
|
|
ELSE expiry_finalized
|
|
END
|
|
WHERE id = ?""",
|
|
(
|
|
entry.get("session_key") or key,
|
|
(origin or {}).get("chat_id") if isinstance(origin, dict) else None,
|
|
entry.get("chat_type"),
|
|
(origin or {}).get("thread_id") if isinstance(origin, dict) else None,
|
|
entry.get("display_name"),
|
|
json.dumps(origin) if isinstance(origin, dict) else None,
|
|
1 if entry.get("expiry_finalized") or entry.get("memory_flushed") else 0,
|
|
str(session_id),
|
|
),
|
|
)
|