mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-30 19:09:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6
hermes-agent/hermes_state.py
Jorkey Liu 05afea65f4 fix(session): resolve default state DB path at call time 
DEFAULT_DB_PATH in hermes_state.py is computed at import time, freezing
the developer's real ~/.hermes even when a test fixture (or runtime
profile switch) later redirects HERMES_HOME. Any default SessionDB() —
e.g. gateway SessionStore — then opened the real state.db.

Add _default_db_path(): resolves get_hermes_home() fresh at call time,
while a deliberately re-pointed DEFAULT_DB_PATH (the established
monkeypatch escape hatch) still wins via an import-time snapshot
comparison, preserving existing test behavior. SessionDB.__init__ and
session_search's requirement check now use the resolver; explicit
db_path arguments are untouched.

Reimplemented from PR #11875 by @JorkeyLiu (original diff predates the
hermes_state rewrite); regression test ported and modernized.
2026-07-29 18:55:10 -07:00

8351 lines
366 KiB
Python
Raw Blame History

#!/usr/bin/env python3
"""
SQLite State Store for Hermes Agent.
Provides persistent session storage with FTS5 full-text search, replacing
the per-session JSONL file approach. Stores session metadata, full message
history, and model configuration for CLI and gateway sessions.
Key design decisions:
- WAL mode for concurrent readers + one writer (gateway multi-platform)
- FTS5 virtual table for fast text search across all session messages
- Compression-triggered session splitting via parent_session_id chains
- Batch runner and RL trajectories are NOT stored here (separate systems)
- Session source tagging ('cli', 'telegram', 'discord', etc.) for filtering
"""
import asyncio
import atexit
import json
import logging
import os
import random
import re
import sqlite3
import sys
import threading
import time
from collections import deque
from contextlib import contextmanager
from pathlib import Path
from agent.memory_manager import sanitize_context
from agent.message_sanitization import _sanitize_surrogates
from agent.skill_commands import (
SKILL_EXCERPT_JOINT,
SKILL_SCAFFOLD_SQL_LIKE,
describe_skill_invocation,
)
from hermes_constants import get_hermes_home
from hermes_cli.sqlite_runtime import (
is_sqlite_wal_reset_vulnerable as _is_sqlite_wal_reset_vulnerable,
)
from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar
from hermes_state_common import ( # noqa: F401 (re-exported for back-compat)
_BRANCH_CHILD_SQL,
_COMPRESSION_CHILD_SQL,
_FTS_CJK_TRIGGERS,
_FTS_TRIGGERS,
_LISTABLE_CHILD_SQL,
_PREVIEW_RAW_SELECT,
_ephemeral_child_sql,
_shape_preview,
DEFERRED_INDEX_SQL,
FTS_CJK_STALE_KEY,
FTS_SQL,
FTS_STORAGE_VERSION,
FTS_TRIGRAM_SQL,
LEGACY_FTS_SQL,
LEGACY_FTS_TRIGRAM_SQL,
MAX_FTS5_QUERY_CHARS,
SCHEMA_SQL,
SCHEMA_VERSION,
_PREVIEW_CONTENT_SQL,
_PREVIEW_HEAD_CHARS,
_PREVIEW_MAX_CHARS,
_PREVIEW_SCAFFOLD_WINDOW,
_PREVIEW_SCAFFOLDED_SQL,
)
from hermes_state_portability import SessionPortabilityMixin
from hermes_state_schema import SessionSchemaMixin
from hermes_state_search import SessionSearchMixin
try: # Hard dependency, but tolerate scaffold-phase imports before pip install.
import psutil
except ImportError: # pragma: no cover - stripped/scaffold installs only
psutil = None # type: ignore[assignment]
logger = logging.getLogger(__name__)
_COMPRESSION_LOCK_HOLDER_PID_RE = re.compile(r"(?:^|:)pid=(\d+)(?::|$)")
def _compression_lock_holder_process_is_dead(holder: str) -> bool:
"""Return True only when a structured lock holder's local PID is gone.
Compression locks are stored in a host-local SQLite database and holder
IDs created by ``conversation_compression`` start with ``pid=<n>``. A
process killed during gateway shutdown cannot release its lease, so waiting
for the full TTL makes every new turn repeatedly attempt compaction. Reclaim
only when the kernel proves that PID no longer exists; legacy/unstructured
holders, same-process holders, permission errors, and any probe doubt
remain protected until normal TTL expiry (conservative: PID reuse must
never steal a live lease, and a wrongly-kept lease self-heals via TTL).
"""
match = _COMPRESSION_LOCK_HOLDER_PID_RE.search(holder or "")
if match is None:
return False
try:
pid = int(match.group(1))
except (TypeError, ValueError):
return False
if pid <= 0:
return False
if pid == os.getpid():
# Same-process holder (e.g. another thread's live lease): never
# self-reclaim — the lease refresher and release path own it.
return False
if psutil is not None:
try:
# psutil is the canonical cross-platform liveness answer
# (CONTRIBUTING.md "Critical rules" #1). pid_exists() reports
# recycled PIDs as alive — conservative, the TTL still applies.
return not psutil.pid_exists(pid)
except Exception:
return False # any doubt → keep the lease until TTL expiry
# Scaffold-phase fallback only (psutil missing), and POSIX-only: stdlib
# os.kill(pid, 0) is NOT a no-op probe on Windows (bpo-14484 — sig=0 maps
# to CTRL_C_EVENT and can kill the target's console group). Without psutil
# a Windows host stays TTL-only; the lease TTL remains the recovery path.
if os.name == "nt":
return False
try:
os.kill(pid, 0) # windows-footgun: ok — nt early-returns just above
except ProcessLookupError:
return True
except (PermissionError, OSError, OverflowError):
return False
return False
def _scrub_surrogates(value: Any) -> Any:
"""Replace lone surrogates when *value* is text; pass anything else through.
sqlite3 encodes bound ``str`` parameters as UTF-8 and raises
``UnicodeEncodeError`` on lone surrogates (U+D800..U+DFFF), so a single
such code point anywhere in a message aborts the whole write. No-op for
well-formed text.
"""
return _sanitize_surrogates(value) if isinstance(value, str) else value
def workspace_key(row: Dict[str, Any]) -> Optional[str]:
"""A session's workspace grouping key: its git repo root when known, else
its cwd.
Branch is deliberately excluded so checking out a new branch doesn't
fragment a workspace's session history. Returns None for cwd-less (unbound)
sessions. Both fields are already recorded on ``sessions`` — this just picks
the coarser identity for grouping/filtering.
"""
root = (row.get("git_repo_root") or "").strip()
if root:
return root
cwd = (row.get("cwd") or "").strip()
return cwd or None
def _delegate_from_json(col: str = "model_config") -> str:
return f"json_extract(COALESCE({col}, '{{}}'), '$._delegate_from')"
def _cwd_prefix_clause(cwd_prefix: str) -> Tuple[str, List[str]]:
prefix = cwd_prefix.rstrip("/\\") or cwd_prefix
return "(s.cwd = ? OR s.cwd LIKE ? OR s.cwd LIKE ?)", [prefix, f"{prefix}/%", f"{prefix}\\%"]
def _workspace_key_clause(key: str) -> Tuple[str, List[str]]:
"""Match sessions whose ``workspace_key(row)`` equals ``key``.
Mirrors :func:`workspace_key`: a session belongs to workspace ``key``
when its recorded ``git_repo_root`` equals ``key``, or — for rows that
predate per-session git metadata — when its ``cwd`` is at or under
``key`` (so a session started in ``repo/src`` still groups with ``repo``).
Used by ``hermes -c``/``--resume`` to continue the most recent session in
the *current* workspace rather than the global MRU.
"""
prefix = key.rstrip("/\\") or key
cwd_clause, cwd_params = _cwd_prefix_clause(prefix)
return (
f"(s.git_repo_root = ? OR (COALESCE(s.git_repo_root, '') = '' AND {cwd_clause}))",
[prefix, *cwd_params],
)
def _collect_delegate_child_ids(conn, parent_ids: List[str]) -> List[str]:
"""Delegate-subagent ids to cascade-delete with *parent_ids*.
Only rows carrying the ``_delegate_from`` marker (set at creation, and
backfilled by the v16 migration) — generic untagged children keep the
orphan-don't-delete contract. Walks marker chains recursively so an
orchestrator subagent's own delegate children go too (FK safety).
"""
df = _delegate_from_json()
seeds = {sid for sid in parent_ids if sid}
# Seed the visited set with the parents themselves. A delegation marker
# chain can loop back onto a parent — a cycle, or a parent that is also
# another parent's delegate child when several ids are deleted at once —
# and without this guard that parent would be collected as one of its own
# descendants and cascade-deleted along with all of its messages. Callers
# delete the parents separately, so parents must never appear in the
# returned child set. (#49148)
found: set[str] = set(seeds)
frontier = list(seeds)
while frontier:
ph = ",".join("?" * len(frontier))
cursor = conn.execute(
f"SELECT id FROM sessions WHERE {df} IN ({ph}) "
f"OR (parent_session_id IN ({ph}) AND {df} IS NOT NULL)",
frontier + frontier,
)
frontier = [row["id"] for row in cursor.fetchall() if row["id"] not in found]
found.update(frontier)
# Return only the discovered children — never the parents themselves.
return [sid for sid in found if sid not in seeds]
def _delete_delegate_children(conn, parent_ids: List[str]) -> List[str]:
ids = _collect_delegate_child_ids(conn, parent_ids)
if ids:
ph = ",".join("?" * len(ids))
conn.execute(f"DELETE FROM messages WHERE session_id IN ({ph})", ids)
# FK safety: orphan any untagged stragglers pointing at a doomed row.
conn.execute(
f"UPDATE sessions SET parent_session_id = NULL "
f"WHERE parent_session_id IN ({ph})",
ids,
)
conn.execute(f"DELETE FROM sessions WHERE id IN ({ph})", ids)
return ids
T = TypeVar("T")
DEFAULT_DB_PATH = get_hermes_home() / "state.db"
# Import-time snapshot used by _default_db_path() to detect a deliberately
# re-pointed DEFAULT_DB_PATH (tests monkeypatch the constant directly).
_IMPORT_DEFAULT_DB_PATH = DEFAULT_DB_PATH
def _default_db_path() -> Path:
"""Resolve the default state DB path at call time.
``DEFAULT_DB_PATH`` is computed when this module is first imported, which
freezes the developer's real ``~/.hermes`` even when a test fixture later
redirects ``HERMES_HOME`` — importing this module during collection was
enough to point every default ``SessionDB()`` at the real state.db.
Precedence:
1. A deliberately re-pointed ``DEFAULT_DB_PATH`` (differs from the
import-time snapshot — the established test escape hatch) wins.
2. Otherwise resolve ``get_hermes_home()`` fresh so a runtime
``HERMES_HOME`` redirect takes effect regardless of import order.
"""
if DEFAULT_DB_PATH != _IMPORT_DEFAULT_DB_PATH:
return DEFAULT_DB_PATH
return get_hermes_home() / "state.db"
# ---------------------------------------------------------------------------
# WAL-compatibility fallback
# ---------------------------------------------------------------------------
# SQLite's WAL mode requires shared-memory (mmap) coordination and fcntl
# byte-range locks that don't reliably work on network filesystems (NFS,
# SMB/CIFS, some FUSE mounts, WSL1). Upstream documents this explicitly:
# https://www.sqlite.org/wal.html#sometimes_queries_return_sqlite_busy_in_wal_mode
#
# On those filesystems ``PRAGMA journal_mode=WAL`` raises
# ``sqlite3.OperationalError: locking protocol`` (SQLITE_PROTOCOL). If we
# propagate that, every feature backed by state.db / kanban.db breaks
# silently — /resume, /title, /history, /branch, kanban dispatcher, etc.
#
# ZFS is a separate case: its COW + mmap semantics can corrupt the WAL
# shared-memory (-shm) file under concurrent connection bursts, presenting
# as ``disk I/O error`` rather than ``locking protocol``.
#
# Instead, fall back to ``journal_mode=DELETE`` (the pre-WAL default) which
# works on NFS and ZFS. Concurrency drops — concurrent readers are blocked
# during a write — but the feature works.
#
# Separately, SQLite's WAL-reset bug can corrupt multi-process WAL databases
# on unfixed library builds (issue #69784). See:
# https://sqlite.org/wal.html#walresetbug
# Fixed in 3.51.3+ with backports 3.50.7 and 3.44.6. On vulnerable builds we
# refuse to *enable* WAL for fresh / non-WAL databases (prefer DELETE). We do
# NOT live-downgrade an on-disk WAL database — other gateway/cron/worker
# connections may still hold it open, and flipping journal_mode under them is
# unsafe (same invariant as the NFS path below).
_WAL_INCOMPAT_MARKERS = (
"locking protocol", # SQLITE_PROTOCOL on NFS/SMB
"not authorized", # Some FUSE mounts block WAL pragma outright
"disk i/o error", # ZFS SHM corruption under concurrent connections
)
# Last SessionDB() init error, per-process. Surfaced in /resume and
# related slash-command error strings so users know WHY the DB is
# unavailable instead of getting a bare "Session database not available."
# Only SessionDB.__init__ writes to this; kanban_db.connect() failures
# do not update it (by design — kanban failures are reported via their
# own caller's error handling, not via /resume-style slash commands).
_last_init_error: Optional[str] = None
_last_init_error_lock = threading.Lock()
# Paths for which we've already logged a WAL-fallback WARNING. Without
# this, kanban_db.connect() (called on every kanban operation — see
# hermes_cli/kanban_db.py for ~30 call sites) would re-log the same
# filesystem-incompat warning on every connection, filling errors.log.
_wal_fallback_warned_paths: set[str] = set()
_wal_fallback_warned_lock = threading.Lock()
# Dedup WARNING for the WAL-reset vulnerability fallback (issue #69784).
_wal_reset_bug_warned_paths: set[str] = set()
_wal_reset_bug_warned_lock = threading.Lock()
def _set_last_init_error(msg: Optional[str]) -> None:
"""Record (or clear) the most recent state.db init failure.
Thread-safe via _last_init_error_lock. Callers pass a message to
record a failure or None to clear. SessionDB.__init__ only calls
this to SET on failure — it deliberately does NOT clear on success,
because in a multi-threaded caller (e.g. gateway / web_server per-
request SessionDB() instantiation), a concurrent successful open
racing past a different thread's failure would erase the cause
string that thread's /resume handler is about to format. Explicit
clears (e.g. test fixtures) are still supported by passing None.
"""
global _last_init_error
with _last_init_error_lock:
_last_init_error = msg
def get_last_init_error() -> Optional[str]:
"""Return the most recent state.db init failure, if any.
Slash-command handlers (``/resume``, ``/title``, ``/history``, ``/branch``)
call this to surface the underlying cause in their error messages when
``_session_db is None``. Returns ``None`` if SessionDB initialized
successfully (or hasn't been attempted).
"""
return _last_init_error
# Distinctive opening shared by both background-review harness prompts
# (_SKILL_REVIEW_PROMPT and _MEMORY_REVIEW_PROMPT in agent/background_review.py).
# Matched case-sensitively against the leading content of a user/system message.
_REVIEW_HARNESS_PREFIXES = (
"Review the conversation above and update the skill library",
"Review the conversation above and consider saving to memory",
)
def _is_background_review_harness_message(msg: Dict[str, Any]) -> bool:
"""True when ``msg`` is a persisted background-review harness prompt.
These are user/system turns the forked skill/memory review agent wrote into
a real session in older builds (before the ``_persist_disabled`` isolation
fix). They instruct the agent to act as the curator under a hard tool
restriction, so replaying them as live history hijacks the session.
"""
if not isinstance(msg, dict):
return False
if msg.get("role") not in {"user", "system"}:
return False
content = msg.get("content")
if not isinstance(content, str):
return False
head = content.lstrip()
return any(head.startswith(p) for p in _REVIEW_HARNESS_PREFIXES)
def _strip_background_review_harness(
messages: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
"""Drop background-review harness messages and the curator-mode assistant
reply that immediately followed each one.
Walk the list once; when a harness user/system message is found, skip it and
also skip the next message if it is the assistant turn that answered it.
Everything else passes through untouched and in order.
"""
if not messages:
return messages
out: List[Dict[str, Any]] = []
skip_next_assistant = False
for msg in messages:
if _is_background_review_harness_message(msg):
skip_next_assistant = True
continue
if skip_next_assistant:
skip_next_assistant = False
if isinstance(msg, dict) and msg.get("role") == "assistant":
# The curator-mode reply to the harness prompt — drop it.
continue
out.append(msg)
return out
def format_session_db_unavailable(prefix: str = "Session database not available") -> str:
"""Format a user-facing 'session DB unavailable' message with cause.
When ``SessionDB()`` init fails, callers set ``_session_db = None`` and
several slash commands (/resume, /title, /history, /branch) previously
responded with a bare ``"Session database not available."`` — no
indication of WHY. This helper includes the captured cause (typically
``"locking protocol"`` from NFS/SMB) and points users at the known
culprit so they can fix it themselves.
Example output:
Session database not available: locking protocol (state.db may be
on NFS/SMB — see https://www.sqlite.org/wal.html).
"""
cause = get_last_init_error()
if not cause:
return f"{prefix}."
hint = ""
if any(marker in cause.lower() for marker in _WAL_INCOMPAT_MARKERS):
hint = " (state.db may be on NFS/SMB/FUSE/ZFS — see https://www.sqlite.org/wal.html)"
return f"{prefix}: {cause}{hint}."
def _on_disk_journal_mode(conn: sqlite3.Connection) -> Optional[str]:
"""Read the journal mode from the SQLite DB header on disk.
Returns the mode string (e.g. ``"wal"``, ``"delete"``), or ``None``
if the value cannot be determined (new DB, or PRAGMA read failed).
"""
try:
row = conn.execute("PRAGMA journal_mode").fetchone()
except sqlite3.OperationalError:
return None
if row is None:
return None
mode = row[0]
if isinstance(mode, bytes): # defensive: sqlite3 occasionally returns bytes
try:
mode = mode.decode("ascii")
except UnicodeDecodeError:
return None
return str(mode).strip().lower() if mode is not None else None
def _apply_macos_checkpoint_barrier(conn: sqlite3.Connection) -> None:
"""Enable ``PRAGMA checkpoint_fullfsync`` on macOS (no-op elsewhere).
On Darwin, ``synchronous=FULL`` (the WAL default) issues a plain
``fsync()``, which Apple documents does *not* guarantee that data
has reached stable storage or that writes are not reordered — see
the ``fsync(2)`` man page. SQLite's WAL corruption-safety guarantee
assumes the OS honors the fsync write barrier; macOS does not unless
the app uses ``F_FULLFSYNC``.
During a launchd *system* shutdown/reboot the OS page cache is
dropped (effectively a power-loss event for in-flight pages), so a
WAL checkpoint whose ``fsync()`` "reported" durable may never have
hit the platter — corrupting ``state.db`` with a malformed image.
This is the trigger in issue #30636 ("SIGTERM during launchd
shutdown under high load"), distinct from a plain in-session kill
(which the page cache survives and SQLite recovers from).
``checkpoint_fullfsync=1`` forces an ``F_FULLFSYNC`` barrier only at
checkpoint boundaries — where WAL frames land in the main DB — so the
cost amortizes to roughly +0.1 ms/commit (vs ~+4 ms for the broader
``fullfsync=1`` that flushes on every commit's WAL sync). Guarded by
``sys.platform == "darwin"`` because ``F_FULLFSYNC`` is macOS-only;
on other platforms the PRAGMA is a no-op, so we skip it entirely.
Best-effort: never raises.
"""
if sys.platform != "darwin":
return
try:
conn.execute("PRAGMA checkpoint_fullfsync=1")
except sqlite3.OperationalError:
pass
def _enforce_macos_synchronous_full(conn: sqlite3.Connection) -> None:
"""Enforce ``PRAGMA synchronous=FULL`` on macOS to prevent btree corruption.
On Darwin, the default ``synchronous=NORMAL`` only calls ``fsync()``,
which Apple's fsync(2) man page explicitly states does *not* guarantee
data-on-platter or write-ordering. During a WAL checkpoint race with
process termination (e.g., launchd shutdown), this can leave the main
DB with half-written btree pages → ``btreeInitPage error 11``.
WAL mode's durability guarantee assumes the OS honors fsync barriers;
macOS does not unless we explicitly set ``synchronous=FULL``, which issues
a real ``fsync()`` on every transaction commit. The ``F_FULLFSYNC``
barrier at checkpoint boundaries is handled separately by
:func:`_apply_macos_checkpoint_barrier`.
This function is called after any successful WAL activation (either
from ``apply_wal_with_fallback()`` setting a fresh WAL or when probing
an existing WAL mode). It ensures macOS connections always use FULL
synchronous mode, even if a prior connection set ``synchronous=NORMAL``.
Best-effort: never raises.
"""
if sys.platform != "darwin":
return
try:
conn.execute("PRAGMA synchronous=FULL")
except sqlite3.OperationalError:
pass
def is_sqlite_wal_reset_vulnerable(
version_info: Optional[tuple] = None,
) -> bool:
"""Return True when the linked SQLite library has the WAL-reset bug.
Upstream documents the bug in versions 3.7.0 through 3.51.2, fixed in
3.51.3+, with backports 3.50.7 and 3.44.6:
https://sqlite.org/wal.html#walresetbug
Pre-WAL libraries (< 3.7.0) cannot hit the race and are treated as safe.
"""
info = version_info if version_info is not None else sqlite3.sqlite_version_info
return _is_sqlite_wal_reset_vulnerable(info)
def sqlite_source_id() -> str:
"""Return ``sqlite_source_id()``, or an empty string when unavailable."""
try:
conn = sqlite3.connect(":memory:")
try:
row = conn.execute("SELECT sqlite_source_id()").fetchone()
finally:
conn.close()
except sqlite3.Error:
return ""
if not row or row[0] is None:
return ""
return str(row[0])
def resolve_journal_mode() -> str:
"""Return the configured journal mode (``wal`` or ``delete``).
``database.journal_mode`` in config.yaml is the canonical operator
setting. ``wal`` remains the default; use ``delete`` when the backing
filesystem does not provide WAL-safe durability (for example macOS
virtiofs, NFS, or SMB). Invalid or malformed values fail safely to the
existing default.
"""
try:
from hermes_cli.config import load_config_readonly
config = load_config_readonly() or {}
database = config.get("database", {})
if not isinstance(database, dict):
return "wal"
raw = database.get("journal_mode", "wal")
except Exception:
return "wal"
if not isinstance(raw, str):
return "wal"
mode = raw.strip().lower()
return mode if mode in ("wal", "delete") else "wal"
class WalUnsupportedError(sqlite3.OperationalError):
"""Raised by :func:`apply_wal_with_fallback` when ``require_wal=True`` and
the filesystem cannot provide WAL journal mode.
Covers both shapes of WAL refusal on network filesystems (NFS / SMB / FUSE
/ the AgentFS NFS overlay): SQLite *raising* ``SQLITE_PROTOCOL`` ("locking
protocol"), and the quieter macOS-NFS case where ``PRAGMA journal_mode=WAL``
silently returns the still-effective mode without raising. Subclasses
``sqlite3.OperationalError`` so existing ``except sqlite3.OperationalError``
DB-init handling still catches it, while callers that specifically mandate
WAL can catch this narrower type.
"""
def apply_wal_with_fallback(
conn: sqlite3.Connection,
*,
db_label: str = "state.db",
require_wal: bool = False,
) -> str:
"""Set ``journal_mode=WAL`` on ``conn``, falling back to DELETE on failure.
Returns the journal mode actually set (``"wal"`` or ``"delete"``).
On WAL-incompatible filesystems (NFS, SMB, some FUSE, ZFS), SQLite either
raises ``OperationalError("locking protocol")`` /
``OperationalError("disk I/O error")`` or — on macOS NFS / SMB /
the AgentFS NFS overlay — silently refuses the switch and leaves the DB in
DELETE. Either way the degradation is logged at ERROR level (it is a real
loss of concurrency — a write blocks concurrent readers — not a cosmetic
warning) and, by default, the function falls back to DELETE (the pre-WAL
default, which works on NFS and ZFS) so the feature keeps working.
On SQLite builds that still contain the WAL-reset corruption bug
(issue #69784), refuse to enable WAL on fresh / non-WAL databases
(prefer DELETE). If the on-disk DB is already WAL, keep WAL and warn
— never live-downgrade under possible concurrent openers.
This gate (#70055) is deliberately RETAINED. An earlier revision of the
lock-cancellation fix (#71724) reverted it on the theory that DELETE was
"the mode that corrupts", but that comparison was confounded: the clean
WAL result came from SQLite 3.53.1, which carries BOTH the WAL-reset fix
AND 3.51.0's defenses against close()-broken POSIX locks, so it says
nothing about 3.50.4. Re-measured on the actually-bundled 3.50.4 with
the lock fix in place, WAL and DELETE are both clean (0/3 each) — i.e.
there is no evidence that WAL is safer here, and upstream still documents
the WAL-reset bug as real through 3.51.2 with serious consequences. Until
a fixed runtime is delivered, keep new databases out of WAL.
Callers that genuinely require WAL concurrency (and would rather fail loudly
than run silently degraded) pass ``require_wal=True``; the function then
raises :class:`WalUnsupportedError` instead of returning ``"delete"``. All
current callers deliberately keep the default ``require_wal=False`` so
NFS-homed installs keep working.
The ERROR is deduplicated per ``db_label``: repeated connections to the
same underlying DB (e.g. kanban_db.connect() which is called on every
kanban operation) log once per process, not once per call. Different
db_labels log independently, so state.db and kanban.db each get one error
on the same NFS mount.
Shared by :class:`SessionDB` and ``hermes_cli.kanban_db.connect`` so
both databases get identical fallback behavior.
Never downgrades to DELETE if the on-disk DB header reports WAL — see
_on_disk_journal_mode. That holds for both the NFS path and the
WAL-reset vulnerability path.
"""
configured = resolve_journal_mode()
# Vulnerable SQLite: do not enable WAL on new/non-WAL files. Resolve the
# operator setting first so an explicit DELETE request still verifies that
# SQLite actually accepted DELETE rather than silently returning MEMORY or
# another connection-specific mode.
if is_sqlite_wal_reset_vulnerable():
return _apply_delete_for_wal_reset_bug(
conn,
db_label=db_label,
require_delete=configured == "delete",
)
# Read-only probe — no flock, no checkpoint, no WAL/SHM unlink.
# Skipping the set-pragma prevents WAL-init from unlinking files other connections hold open.
try:
current_mode = conn.execute("PRAGMA journal_mode").fetchone()
if current_mode and current_mode[0] == "wal":
_apply_macos_checkpoint_barrier(conn)
_enforce_macos_synchronous_full(conn)
return "wal"
except sqlite3.OperationalError:
pass
# #68545: honor the canonical database.journal_mode setting. Existing
# on-disk WAL databases were returned above and are never live-downgraded.
if configured == "delete":
row = conn.execute("PRAGMA journal_mode=DELETE").fetchone()
actual = str(row[0]).lower() if row else ""
if actual != "delete":
raise sqlite3.OperationalError(
f"could not set configured journal_mode=delete (got {actual or 'no result'})"
)
return actual
try:
# ``PRAGMA journal_mode=WAL`` is a query-that-sets: it RETURNS the
# resulting journal mode. Network filesystems that refuse WAL by
# *raising* SQLITE_PROTOCOL ("locking protocol") are handled in the
# except branch below. But macOS NFS — and SMB/CIFS, and the AgentFS
# NFS overlay — refuse the switch WITHOUT raising: the pragma simply
# returns the still-effective mode (e.g. ``delete``). Trust the
# returned row, not the mere absence of an exception; otherwise we
# report a false ``"wal"`` AND skip the fallback WARNING, leaving the
# DB silently in DELETE (reader-blocks-writer) with no signal.
row = conn.execute("PRAGMA journal_mode=WAL").fetchone()
mode = str(row[0]).strip().lower() if row and row[0] is not None else ""
if mode == "wal":
_apply_macos_checkpoint_barrier(conn)
_enforce_macos_synchronous_full(conn)
return "wal"
# Silent refusal (macOS NFS / SMB / AgentFS overlay): WAL was not
# honored, but nothing raised.
silent_exc = WalUnsupportedError(
f"journal_mode=WAL refused without raising (still {mode!r})"
)
if require_wal:
raise silent_exc
_log_wal_fallback_once(db_label, silent_exc)
return mode or "delete"
except sqlite3.OperationalError as exc:
# The require_wal silent-refusal raise above is a WalUnsupportedError
# (an OperationalError subclass) and lands here — propagate it
# unchanged rather than re-running it through the marker logic.
if isinstance(exc, WalUnsupportedError):
raise
msg = str(exc).lower()
if not any(marker in msg for marker in _WAL_INCOMPAT_MARKERS):
# Unrelated OperationalError — don't silently swallow.
raise
# ``disk i/o error`` is ambiguous: on ZFS / APFS-CoW it is a
# deterministic WAL-incompatibility (SHM corruption under concurrent
# connection bursts — #55305, #71498), but it can also be a one-shot
# transient EIO (page-cache pressure, brief lock contention).
# Treating a transient EIO as a permanent downgrade signal produced
# the mixed-journal-mode corruption pattern fixed in 5c49cd0ed0
# (process A downgrades to DELETE while sibling processes set WAL).
# Disambiguate by retrying the pragma a couple of times: transient
# EIO clears and we return "wal"; the deterministic filesystem cases
# keep failing and fall through to the guarded DELETE fallback.
if "disk i/o error" in msg:
for _ in range(2):
time.sleep(0.05)
try:
row = conn.execute("PRAGMA journal_mode=WAL").fetchone()
except sqlite3.OperationalError as retry_exc:
if "disk i/o error" not in str(retry_exc).lower():
raise
exc = retry_exc
continue
mode = (
str(row[0]).strip().lower()
if row and row[0] is not None
else ""
)
if mode == "wal":
_apply_macos_checkpoint_barrier(conn)
_enforce_macos_synchronous_full(conn)
return "wal"
break
# Don't downgrade if another process already set WAL on disk.
existing = _on_disk_journal_mode(conn)
if existing == "wal":
raise
if require_wal:
# Caller mandates WAL — fail loudly instead of degrading to DELETE.
raise WalUnsupportedError(str(exc)) from exc
_log_wal_fallback_once(db_label, exc)
conn.execute("PRAGMA journal_mode=DELETE")
return "delete"
def _apply_delete_for_wal_reset_bug(
conn: sqlite3.Connection,
*,
db_label: str,
require_delete: bool = False,
) -> str:
"""Avoid enabling WAL when the linked SQLite has the WAL-reset bug.
- Already-WAL on disk: leave WAL alone (no live downgrade) and warn.
- Otherwise: set DELETE and warn.
- For an explicit operator request, verify SQLite accepted DELETE.
"""
current = ""
try:
row = conn.execute("PRAGMA journal_mode").fetchone()
if row and row[0] is not None:
current = str(row[0]).strip().lower()
except sqlite3.OperationalError:
current = ""
if current == "wal":
# Do not TRUNCATE / journal_mode=DELETE while other processes may
# still hold this WAL DB open — same safety rule as the NFS path.
_log_wal_reset_bug_once(db_label, kept_wal=True)
_apply_macos_checkpoint_barrier(conn)
_enforce_macos_synchronous_full(conn)
return "wal"
actual = ""
try:
row = conn.execute("PRAGMA journal_mode=DELETE").fetchone()
if row and row[0] is not None:
actual = str(row[0]).strip().lower()
except sqlite3.OperationalError:
if require_delete:
raise
# Best-effort for the automatic vulnerable-runtime fallback: DELETE is
# normally already the default for new file-backed databases.
if require_delete and actual != "delete":
raise sqlite3.OperationalError(
"could not set configured journal_mode=delete "
f"(got {actual or 'no result'})"
)
_log_wal_reset_bug_once(db_label, kept_wal=False)
return "delete"
def _log_wal_reset_bug_once(
db_label: str,
*,
kept_wal: bool,
) -> None:
"""Log once per (process, db_label) about the WAL-reset vulnerability path."""
with _wal_reset_bug_warned_lock:
if db_label in _wal_reset_bug_warned_paths:
return
_wal_reset_bug_warned_paths.add(db_label)
action = (
"is already in WAL mode — leaving WAL in place (no live "
"downgrade under concurrent openers)"
if kept_wal
else "using journal_mode=DELETE instead of enabling WAL"
)
logger.warning(
"%s: linked SQLite %s is vulnerable to the WAL-reset corruption "
"bug (https://sqlite.org/wal.html#walresetbug) — %s. "
"Upgrade to SQLite 3.51.3+ (or backports 3.50.7 / 3.44.6); "
"Hermes-managed installs can repair the embedded runtime with "
"`hermes update`. See `hermes doctor`. This warning fires once per "
"process per database.",
db_label,
sqlite3.sqlite_version,
action,
)
def _log_wal_fallback_once(db_label: str, exc: Exception) -> None:
"""Log a single ERROR per (process, db_label) about WAL fallback.
ERROR (not WARNING): a DB silently dropped to DELETE means a real loss of
concurrency — under the kanban dispatcher + workers a write blocks readers,
surfacing as SQLITE_BUSY/lock contention — so it must be loud, not cosmetic.
Without this dedup, NFS users running kanban (which opens a fresh
connection on every operation — see hermes_cli/kanban_db.py) would
fill errors.log with hundreds of identical errors per hour.
"""
with _wal_fallback_warned_lock:
if db_label in _wal_fallback_warned_paths:
return
_wal_fallback_warned_paths.add(db_label)
logger.error(
"%s: WAL journal_mode unsupported on this filesystem (%s) — "
"falling back to journal_mode=DELETE (slower rollback-journal "
"mode; reduces concurrency but works on NFS/SMB/FUSE/ZFS). See "
"https://www.sqlite.org/wal.html for details. This message "
"fires once per process per database.",
db_label,
exc,
)
# ---------------------------------------------------------------------------
# Config-driven database pragmas
# ---------------------------------------------------------------------------
def apply_database_pragmas(
conn: sqlite3.Connection,
*,
db_label: str = "state.db",
) -> None:
"""Apply optional WAL-sizing PRAGMAs from ``config.yaml``.
Reads the ``database:`` section and applies ``wal_autocheckpoint``
and ``journal_size_limit`` when set to integer values. The journal
mode itself is NOT handled here — ``database.journal_mode`` is owned
by :func:`resolve_journal_mode` inside :func:`apply_wal_with_fallback`,
which layers the operator setting under all the safety guards
(never live-downgrading an on-disk WAL DB, filesystem fallback,
WAL-reset-bug gating). Keeping a single owner prevents a second,
unguarded journal-mode switch path.
Best-effort: config load or pragma failures are ignored so DB init
never breaks on a malformed ``database:`` section.
"""
try:
# Local import avoids a circular import with hermes_cli.config.
from hermes_cli.config import cfg_get, load_config_readonly
cfg = load_config_readonly()
except Exception:
return
for pragma_name in ("wal_autocheckpoint", "journal_size_limit"):
raw_value = cfg_get(cfg, "database", pragma_name, default=None)
if raw_value is None:
continue
try:
value = int(str(raw_value).strip())
except (TypeError, ValueError):
logger.warning(
"%s: ignoring non-integer database.%s=%r",
db_label,
pragma_name,
raw_value,
)
continue
try:
conn.execute(f"PRAGMA {pragma_name}={value}")
except sqlite3.OperationalError:
pass
# ---------------------------------------------------------------------------
# Malformed-schema recovery
# ---------------------------------------------------------------------------
# A distinct, nastier failure class than a malformed FTS *inverted index*:
# the ``sqlite_master`` schema table itself becomes inconsistent — most
# commonly a DUPLICATE object definition, e.g. two ``CREATE VIRTUAL TABLE
# messages_fts`` rows. SQLite parses the entire schema while preparing the
# FIRST statement on a connection, so on this class *every* statement raises
# before it runs — including ``PRAGMA journal_mode`` (which is why this trips
# in ``apply_wal_with_fallback`` during ``SessionDB.__init__``, long before
# ``_init_schema`` is reached) and even ``PRAGMA integrity_check`` and a plain
# ``DROP TABLE``. The only operations that still work are
# ``PRAGMA writable_schema=ON`` plus direct ``sqlite_master`` surgery.
#
# Symptom users hit (Desktop/Dashboard show "no sessions" while 200+ JSON
# files sit on disk):
# sqlite3.DatabaseError: malformed database schema (messages_fts) -
# table messages_fts already exists
#
# The canonical ``sessions`` / ``messages`` data is intact in these cases —
# only the derived schema is broken — so recovery preserves all transcripts
# and merely rebuilds the FTS layer.
_MALFORMED_SCHEMA_MARKERS = (
"malformed database schema",
"database disk image is malformed",
)
# Process-global guard so auto-repair is attempted at most once per DB path
# per process (prevents repair loops and serialises concurrent web_server /
# gateway opens against the same malformed file).
_repair_attempted_paths: set[str] = set()
_repair_attempt_lock = threading.Lock()
def is_malformed_db_error(exc: BaseException) -> bool:
"""True if *exc* is a SQLite 'malformed schema / disk image' error.
These are the corruption classes where the schema fails to parse, so
targeted ``sqlite_master`` surgery (not an ordinary FTS rebuild) is the
only recovery path.
"""
if not isinstance(exc, sqlite3.DatabaseError):
return False
return any(marker in str(exc).lower() for marker in _MALFORMED_SCHEMA_MARKERS)
def _claim_repair_attempt(db_path: Path) -> bool:
"""Claim the one-shot repair attempt for *db_path* in this process.
Returns True for the first caller, False afterwards. Keeps a malformed
DB from triggering an unbounded repair/reopen loop and stops concurrent
callers from racing surgery on the same file.
"""
key = str(db_path)
with _repair_attempt_lock:
if key in _repair_attempted_paths:
return False
_repair_attempted_paths.add(key)
return True
def _backup_db_file(db_path: Path) -> Optional[Path]:
"""Copy a (possibly malformed) DB file to a timestamped backup beside it.
Raw file copy on purpose: the DB won't open cleanly, so we preserve the
bytes exactly for forensics / manual restore. WAL and SHM sidecars are
copied too when present. Returns the backup path, or None on failure.
Refuses when a connection to this database is still live in the process:
reading the file would ``close()`` a descriptor for it and cancel that
connection's POSIX advisory locks (see ``hermes_cli.sqlite_safe_read``).
The repair path can be entered by one SessionDB while the gateway holds
others, so this is a real possibility rather than a theoretical one.
"""
import datetime
import shutil
try:
from hermes_cli.sqlite_safe_read import has_live_connection
except ImportError:
has_live_connection = None # type: ignore[assignment]
if has_live_connection is not None and has_live_connection(db_path):
logger.error(
"Refusing to raw-copy %s for backup: a connection to it is still "
"open in this process and the copy would cancel that connection's "
"POSIX locks. Close all SessionDB handles first.",
db_path,
)
return None
stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
backup_path = db_path.with_name(f"{db_path.name}.malformed-backup-{stamp}")
try:
shutil.copy2(db_path, backup_path)
for suffix in ("-wal", "-shm"):
sidecar = db_path.with_name(db_path.name + suffix)
if sidecar.exists():
shutil.copy2(sidecar, backup_path.with_name(backup_path.name + suffix))
return backup_path
except Exception as exc: # pragma: no cover - best effort
logger.warning("Could not back up malformed DB %s: %s", db_path, exc)
return None
def preflight_db_writability(
db_path: Path,
*,
db_label: str = "state.db",
) -> None:
"""Refuse-or-repair read-only DB files BEFORE the first connection opens.
Port of Kilo-Org/kilocode#12508's startup preflight. A stray read-only
``state.db`` / ``-wal`` / ``-shm`` (sudo run, restored backup, copied
dotfiles) previously surfaced as an opaque
``sqlite3.OperationalError: attempt to write a readonly database`` raised
from deep inside ``_init_schema`` — naming no file and no fix — and the
obvious wrong "fix" (deleting the ``-wal``) silently loses committed
transactions. This preflight:
- **Repairs** permissions with ``chmod u+rw`` when the file lives inside
the Hermes home tree (``get_hermes_home()``) — the safe repair scope:
Hermes owns those files, and the OS makes ``chmod`` fail on files the
user doesn't own, which bounds the repair exactly.
- **Fails fast with an actionable error** naming the exact file and the
exact ``chmod`` command for anything else (root-owned files, read-only
mounts, custom paths outside the home tree).
- Never deletes or truncates a WAL sidecar — once writable, the normal
open path checkpoints its committed frames into the DB as intended.
``:memory:`` and ``file:`` URI paths are skipped (no plain on-disk files
to check). Shared by :class:`SessionDB` and ``hermes_cli.kanban_db``.
"""
raw = str(db_path)
if raw == ":memory:" or raw.startswith("file:"):
return
try:
home: Optional[Path] = Path(get_hermes_home()).resolve()
except Exception: # pragma: no cover - defensive
home = None
def _in_repair_scope(p: Path) -> bool:
if home is None:
return False
try:
return p.resolve().is_relative_to(home)
except (OSError, ValueError):
return False
def _ensure_writable(p: Path, *, is_dir: bool = False) -> None:
import stat as _stat
if os.access(p, os.R_OK | os.W_OK):
return
if _in_repair_scope(p):
try:
add = _stat.S_IRUSR | _stat.S_IWUSR | (_stat.S_IXUSR if is_dir else 0)
os.chmod(p, p.stat().st_mode | add)
except OSError:
pass
if os.access(p, os.R_OK | os.W_OK):
logger.info(
"%s preflight: repaired read-only %s (chmod u+rw%s)",
db_label,
p,
"x" if is_dir else "",
)
return
kind = "directory" if is_dir else "file"
wal_note = (
" Do NOT delete the -wal file — it contains committed data that "
"will be merged into the database once it is writable."
if p.name.endswith("-wal")
else ""
)
raise sqlite3.OperationalError(
f"{db_label} is not writable: {kind} {p} is read-only for this "
f"user. Hermes needs read-write access to open the database. "
f"Fix with: chmod u+rw{'x' if is_dir else ''} '{p}'"
f" (files owned by another user may need sudo/chown).{wal_note}"
)
parent = db_path.parent
if parent.is_dir():
# SQLite needs a writable directory in every journal mode (WAL and
# SHM sidecars in WAL mode; the rollback journal in DELETE mode).
_ensure_writable(parent, is_dir=True)
for suffix in ("", "-wal", "-shm"):
p = db_path.with_name(db_path.name + suffix) if suffix else db_path
if p.is_file():
_ensure_writable(p)
def _db_opens_cleanly(db_path: Path) -> Optional[str]:
"""Probe a DB on a fresh connection. Returns None if healthy, else a reason.
Runs the same first-statement (``PRAGMA journal_mode``) that trips the
malformed-schema parse, then ``PRAGMA integrity_check`` and a canonical
``sessions`` read, and finally a rolled-back ``messages`` write so that
FTS5 index corruption — which leaves base-table reads and
``integrity_check`` passing while every ``INSERT INTO messages`` fails
through the FTS triggers — is reported as unhealthy rather than slipping
past as a false "ok" (#50502).
"""
conn = sqlite3.connect(str(db_path), isolation_level=None)
try:
# Best-effort tokenizer load: a DB carrying the messages_fts_cjk
# index needs the cjk_unicode61 extension before any statement can
# touch that table — including the trigger-driven write probe below.
# Without it, this probe sees the DB exactly as a tokenizer-less
# SessionDB open would (which drops the cjk triggers to keep writes
# working), so tokenizer absence must never classify as corruption.
load_fts5_cjk_extension(conn)
conn.execute("PRAGMA journal_mode").fetchone()
rows = conn.execute("PRAGMA integrity_check").fetchall()
problems = [str(r[0]) for r in rows if r and str(r[0]).lower() != "ok"]
if problems:
return "; ".join(problems[:3])
conn.execute("SELECT COUNT(*) FROM sessions").fetchone()
# FTS5 read probe: run a representative MATCH query against the
# messages_fts* virtual tables. The FTS *write* probe below catches
# the corruption class where base tables read fine but writes fail
# through the triggers (#50502). It does NOT catch partial FTS5
# index corruption — bad shadow-table segments where reads still
# parse but MATCH / snippet / rank queries error out with
# "database disk image is malformed" (a `sqlite3.DatabaseError`,
# not `OperationalError`). session_search, /resume title resolution,
# and any feature relying on FTS5 discovery then break silently
# because the official repair tool's check-only path reports the
# DB as healthy. #66724.
# Catch the full sqlite3 exception hierarchy (not just
# OperationalError) so the malformed-shadow-table class is reported
# rather than letting it crash the caller.
for fts_table in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk"):
try:
# No-op queries against the actual FTS5 APIs the search
# tools use. The trigram table is included because it backs
# the title-resolution path; either corruption mode would
# break session recall without this probe. MATCH '""' is
# the empty phrase-token probe — FTS5 rejects MATCH ''
# outright ("fts5: syntax error"), but a quoted empty
# phrase parses, scans zero rows, and exercises the same
# shadow-table read path the search tools use.
conn.execute(
f"SELECT 1 FROM {fts_table} WHERE {fts_table} MATCH '\"\"' LIMIT 1"
).fetchone()
except sqlite3.OperationalError as exc:
# Use the canonical capability classifier instead of a
# hand-rolled substring check. On SQLite builds without the
# fts5 module, the legacy messages_fts table may exist on
# disk (from a prior build that had FTS5) and MATCH queries
# against it raise OperationalError("no such module: fts5");
# the substring check below would misclassify that as
# corruption and send the DB into the repair path, whose
# final fallback deletes the messages_fts% schema
# (hermes_state.py:645-723). The supported degraded-runtime
# path (SessionDB._is_fts5_unavailable_error + the
# regression suite in tests/test_hermes_state.py:600-632)
# treats both "no such module: fts5" and
# "no such tokenizer: trigram" as the capability error.
if SessionDB._is_fts5_unavailable_error(exc):
# Degraded runtime — not the corruption class we probe.
continue
msg = str(exc).lower()
if "no such table" in msg or "no such column" in msg:
# FTS5 not built yet (brand new file mid-init) — not the
# corruption class we probe.
continue
return f"fts5 read probe failed on {fts_table}: {exc}"
except sqlite3.DatabaseError as exc:
# This is the corruption class #66724 actually wants caught:
# partial shadow-table damage where MATCH / snippet / rank
# queries raise DatabaseError("database disk image is malformed")
# while reads of the FTS5 table itself parse fine.
return f"fts5 read probe failed on {fts_table}: {exc}"
# FTS write probe: drive a row through the messages_fts* triggers in a
# transaction that is always rolled back, so a corrupt FTS index that
# rejects writes is caught even though reads look healthy. The probe is
# best-effort — if the messages/sessions tables don't exist yet (brand
# new file mid-init) the OperationalError is treated as "not yet a
# populated DB", not corruption.
probe_session_id = f"_hermes_fts_health_probe_{time.time_ns()}"
try:
conn.execute("BEGIN IMMEDIATE")
conn.execute(
"INSERT INTO sessions (id, source, started_at) VALUES (?, ?, ?)",
(probe_session_id, "_health_probe", time.time()),
)
conn.execute(
"INSERT INTO messages (session_id, role, content, timestamp) "
"VALUES (?, ?, ?, ?)",
(probe_session_id, "user", "_fts_health_probe", time.time()),
)
conn.execute("ROLLBACK")
except sqlite3.OperationalError as exc:
# Missing tables / FTS disabled — not the corruption class we probe.
try:
conn.execute("ROLLBACK")
except sqlite3.Error:
pass
msg = str(exc).lower()
if "no such table" in msg or "no such column" in msg:
return None
if "no such tokenizer: cjk_unicode61" in msg:
# This probe process couldn't load the cjk extension while
# the DB carries the cjk index — capability gap, not
# corruption. A tokenizer-capable SessionDB serves it fine;
# a tokenizer-less one self-heals by dropping the triggers.
return None
return str(exc)
return None
except sqlite3.DatabaseError as exc:
return str(exc)
finally:
conn.close()
def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, Any]:
"""Repair a state.db whose ``sqlite_master`` schema is malformed or whose
FTS indexes reject writes.
Handles two corruption classes: the "duplicate object definition" /
malformed-schema class where even ``PRAGMA`` statements fail, and the FTS
write-corruption class (#50502) where base tables read fine and
``integrity_check`` passes but writes fail through the ``messages_fts*``
triggers. Tries least-destructive recovery first and escalates:
1. **Rebuild FTS indexes in place** via the FTS5 ``'rebuild'`` command,
which rewrites the internal b-tree segments from the canonical
``messages`` rows without dropping or recreating anything. Fixes the
FTS write-corruption class while preserving the schema intact.
2. **De-duplicate** ``sqlite_master`` (keep the lowest rowid per
``type``/``name``). Fixes the canonical "table X already exists"
case and PRESERVES the existing FTS index intact.
3. **Drop the FTS schema** (every ``messages_fts*`` object) + ``VACUUM``.
The next ``SessionDB()`` open rebuilds the FTS indexes from the
canonical ``messages`` table.
Canonical ``sessions`` / ``messages`` rows are never modified. A
timestamped raw backup is taken first unless ``backup=False``.
Returns a report dict: ``{repaired: bool, strategy: str|None,
backup_path: str|None, error: str|None}``.
"""
report: Dict[str, Any] = {
"repaired": False,
"strategy": None,
"backup_path": None,
"error": None,
}
db_path = Path(db_path)
if not db_path.exists():
report["error"] = f"{db_path} does not exist"
return report
if _db_opens_cleanly(db_path) is None:
report["repaired"] = True
report["strategy"] = "already_healthy"
return report
if backup:
bpath = _backup_db_file(db_path)
report["backup_path"] = str(bpath) if bpath else None
# ── Strategy 0: rebuild FTS indexes in place (FTS write-corruption) ──
# The FTS5 'rebuild' command rewrites the internal index from the canonical
# content table. This is the recommended, least-destructive recovery for a
# corrupt FTS index that rejects message writes while reads still succeed.
try:
conn = sqlite3.connect(str(db_path), isolation_level=None)
try:
# The cjk index can only be rebuilt with its tokenizer loaded;
# best-effort (a tokenizer-less host skips it at the probe below).
load_fts5_cjk_extension(conn)
for table_name in (
"messages_fts", "messages_fts_trigram", "messages_fts_cjk"
):
try:
conn.execute(
f"INSERT INTO {table_name}({table_name}) VALUES('rebuild')"
)
except sqlite3.OperationalError:
# Table absent (FTS disabled / trigram off / cjk not
# present or tokenizer unavailable) — skip it.
continue
finally:
conn.close()
if _db_opens_cleanly(db_path) is None:
report["repaired"] = True
report["strategy"] = "rebuild_fts"
logger.warning(
"state.db FTS indexes rebuilt in place (schema preserved): %s",
db_path,
)
return report
except sqlite3.DatabaseError as exc:
logger.warning("state.db FTS in-place rebuild pass failed: %s", exc)
# ── Strategy 0.5: rebuild stale B-tree indexes (#63386) ──
# PRAGMA integrity_check can report "wrong # of entries in index" when a
# B-tree index (e.g. idx_sessions_handoff_state) falls out of sync with its
# base table. REINDEX rewrites the index b-tree from the canonical table
# rows using the existing index definition, fixing the mismatch without
# touching data or FTS schema.
try:
conn = sqlite3.connect(str(db_path), isolation_level=None)
try:
conn.execute("REINDEX")
conn.commit()
finally:
conn.close()
if _db_opens_cleanly(db_path) is None:
report["repaired"] = True
report["strategy"] = "reindex_btree"
logger.warning(
"state.db B-tree indexes rebuilt via REINDEX: %s", db_path
)
return report
except sqlite3.DatabaseError as exc:
logger.warning("state.db REINDEX pass failed: %s", exc)
# ── Strategy 1: de-duplicate sqlite_master (keeps FTS index) ──
try:
conn = sqlite3.connect(str(db_path), isolation_level=None)
try:
conn.execute("PRAGMA writable_schema=ON")
dupes = conn.execute(
"SELECT type, name, COUNT(*) AS c, MIN(rowid) AS keep "
"FROM sqlite_master GROUP BY type, name HAVING c > 1"
).fetchall()
for type_, name, _count, keep in dupes:
conn.execute(
"DELETE FROM sqlite_master "
"WHERE type IS ? AND name IS ? AND rowid <> ?",
(type_, name, keep),
)
conn.execute("PRAGMA writable_schema=OFF")
conn.commit()
finally:
conn.close()
if _db_opens_cleanly(db_path) is None:
report["repaired"] = True
report["strategy"] = "dedup_schema"
logger.warning(
"state.db schema repaired by de-duplicating sqlite_master "
"(FTS index preserved): %s", db_path
)
return report
except sqlite3.DatabaseError as exc:
logger.warning("state.db dedup repair pass failed: %s", exc)
# ── Strategy 2: drop all FTS schema, VACUUM, rebuild on next open ──
try:
conn = sqlite3.connect(str(db_path), isolation_level=None)
try:
conn.execute("PRAGMA writable_schema=ON")
conn.execute("DELETE FROM sqlite_master WHERE name LIKE 'messages_fts%'")
conn.execute("PRAGMA writable_schema=OFF")
conn.commit()
conn.execute("VACUUM")
finally:
conn.close()
reason = _db_opens_cleanly(db_path)
if reason is None:
report["repaired"] = True
report["strategy"] = "drop_fts_rebuild"
logger.warning(
"state.db schema repaired by dropping FTS schema; indexes "
"will rebuild from messages on next open: %s", db_path
)
return report
report["error"] = reason
except sqlite3.DatabaseError as exc:
report["error"] = str(exc)
if not report["repaired"]:
logger.error(
"state.db schema repair could not recover %s automatically "
"(backup: %s); manual restore from backup may be required.",
db_path, report["backup_path"],
)
return report
# ── CJK-bigram FTS index (replaces the trigram index when available) ────
#
# The trigram tokenizer needs >=3 chars per query term, so 1-2 char CJK
# terms (ubiquitous in Korean/Chinese: 일본, 구글, 项目, ...) fall through
# to a LIKE full-table scan — measured 3-6s CPU per query on multi-GB
# installs and the dominant base cost of session_search on CJK workloads.
#
# ``cjk_unicode61`` (native/fts5_cjk/, a ~250-line loadable FTS5 tokenizer
# with no dependencies) wraps unicode61: maximal CJK runs are re-emitted as
# overlapping character bigrams (Lucene CJKAnalyzer semantics), everything
# else passes through unchanged. FTS5 phrase semantics turn a query term's
# consecutive bigrams into exact substring matching down to 2 chars at
# index speed. Contributed by Soju06 (PR #65544).
#
# Same v23 storage discipline as the trigram table it replaces:
# external-content over a tool-row-excluding view (zero inline text
# copies; tool rows stay searchable via ``messages_fts``), triggers gated
# on a DEDICATED marker pair (``fts_cjk_rebuild_high_water`` /
# ``fts_cjk_rebuild_progress``) so a cjk-only backfill — e.g. the
# trigram→cjk upgrade on an already-optimized DB — never gates the
# complete ``messages_fts`` index's triggers.
#
# The table exists ONLY when the loadable tokenizer is available
# (``~/.hermes/lib/libfts5_cjk.so``, built by ``native/fts5_cjk/build.sh``).
# A process that cannot load it self-heals by dropping the cjk triggers
# (message writes keep working; the index goes stale and is rebuilt by the
# next ``hermes sessions optimize-storage`` on a capable host).
#
# Split DDL: the table/view part is safe to ensure any time; the triggers
# are created ONLY while the index is complete-or-marker-gated. A stale
# index (trigger gap of unknown extent) must keep its triggers DROPPED —
# an external-content 'delete' op for a rowid the index never held is the
# canonical FTS5 index-corruption hazard the v23 marker gating exists to
# prevent.
FTS_CJK_TABLE_SQL = """
CREATE VIEW IF NOT EXISTS messages_fts_cjk_src AS
SELECT id, role, content, tool_name, tool_calls
FROM messages
WHERE role <> 'tool';
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_cjk USING fts5(
content,
tool_name,
tool_calls,
content='messages_fts_cjk_src',
content_rowid='id',
tokenize='cjk_unicode61'
);
"""
FTS_CJK_TRIGGER_SQL = """
CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_insert AFTER INSERT ON messages
WHEN new.role <> 'tool'
AND (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
WHERE key = 'fts_cjk_rebuild_high_water'), -1)
OR new.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
WHERE key = 'fts_cjk_rebuild_progress'), -1))
BEGIN
INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls)
VALUES (new.id, new.content, new.tool_name, new.tool_calls);
END;
CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_delete AFTER DELETE ON messages
WHEN old.role <> 'tool'
AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
WHERE key = 'fts_cjk_rebuild_high_water'), -1)
OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
WHERE key = 'fts_cjk_rebuild_progress'), -1))
BEGIN
INSERT INTO messages_fts_cjk(messages_fts_cjk, rowid, content, tool_name, tool_calls)
VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls);
END;
CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_update AFTER UPDATE ON messages
WHEN (old.content IS NOT new.content
OR old.tool_name IS NOT new.tool_name
OR old.tool_calls IS NOT new.tool_calls
OR old.role IS NOT new.role)
AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
WHERE key = 'fts_cjk_rebuild_high_water'), -1)
OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
WHERE key = 'fts_cjk_rebuild_progress'), -1))
BEGIN
INSERT INTO messages_fts_cjk(messages_fts_cjk, rowid, content, tool_name, tool_calls)
SELECT 'delete', old.id, old.content, old.tool_name, old.tool_calls
WHERE old.role <> 'tool';
INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls)
SELECT new.id, new.content, new.tool_name, new.tool_calls
WHERE new.role <> 'tool';
END;
"""
def fts5_cjk_so_path() -> Path:
"""Location of the cjk_unicode61 loadable extension."""
env = os.getenv("HERMES_FTS5_CJK_SO")
if env:
return Path(env).expanduser()
return get_hermes_home() / "lib" / "libfts5_cjk.so"
def _cjk_fts_config_enabled() -> bool:
"""config.yaml ``sessions.cjk_fts`` (default on), via its env bridge."""
return os.getenv("HERMES_CJK_FTS", "1").strip().lower() not in (
"0", "false", "off", "no",
)
def load_fts5_cjk_extension(conn: sqlite3.Connection) -> bool:
"""Best-effort load of the cjk_unicode61 tokenizer into ``conn``.
Returns False (never raises) when the .so is absent, the feature is
disabled via ``sessions.cjk_fts``, or this Python build has extension
loading compiled out — every caller treats False as "behave exactly as
before the cjk index existed".
"""
if not _cjk_fts_config_enabled():
return False
path = fts5_cjk_so_path()
if not path.exists():
return False
try:
conn.enable_load_extension(True)
try:
conn.load_extension(str(path))
finally:
conn.enable_load_extension(False)
return True
except Exception:
logger.warning("fts5_cjk extension load failed (%s)", path, exc_info=True)
return False
class CompressionSessionClosedError(RuntimeError):
"""A durable write targeted a parent already closed by compression."""
def __init__(self, session_id: str):
self.session_id = session_id
super().__init__(
f"Session {session_id!r} is closed by compression; "
"adopt its live continuation before appending messages"
)
class CompressionSessionBusyError(RuntimeError):
"""A non-owner tried to write while compression owns the session."""
def _connect_tracked_db(path, tracking_path=None, **kwargs):
"""``sqlite3.connect`` that registers the open fd for lock-safety.
While a connection is live, byte-level probes of the same file are
refused: an ``open()``/``close()`` cancels every POSIX advisory lock this
process holds on it -- including a running VACUUM's EXCLUSIVE lock.
Released automatically on ``close()``.
The ONLY tolerated fallback is the helper being absent entirely
(scaffold/embed installs that ship hermes_state without hermes_cli). A
real connection failure must propagate: silently retrying an *untracked*
connect would disable the guard for the lifetime of that connection,
which is precisely the failure mode this module exists to prevent.
"""
try:
from hermes_cli.sqlite_safe_read import connect_tracked
except ImportError:
logger.debug(
"hermes_cli.sqlite_safe_read unavailable; opening %s untracked "
"(byte-probe guard inactive in this install)",
path,
)
return sqlite3.connect(str(path), **kwargs)
# Open through THIS module's sqlite3.connect so callers (and tests) that
# patch hermes_state.sqlite3.connect keep control of connection creation;
# the helper still owns tracking.
return connect_tracked(
path,
tracking_path=tracking_path,
connect_fn=sqlite3.connect,
**kwargs,
)
def is_zeroed_state_db(
path: Path, *, probe_bytes: int = 100, force: bool = False
) -> bool:
"""Detect the #68474 zeroed state.db signature (size>0, NUL header).
Byte-level probe, so it is only safe BEFORE any connection to *path*
exists in this process: ``close()`` cancels every POSIX advisory lock the
process holds on the file, which can pull the EXCLUSIVE lock out from
under a running VACUUM and corrupt the database. The read is routed
through ``read_header_bytes_preopen``, which refuses (returning False
here) once a connection is live. Pass ``force=True`` only for offline
files -- quarantined copies, snapshots, archives.
Prefer ``hermes_cli.backup.is_zeroed_sqlite_file`` when available; this
local copy keeps SessionDB openable without importing the CLI package
in constrained embed paths.
"""
try:
from hermes_cli.backup import is_zeroed_sqlite_file
return is_zeroed_sqlite_file(path, probe_bytes=probe_bytes, force=force)
except Exception:
pass
try:
size = path.stat().st_size
except OSError:
return False
if size <= 0:
return False
from hermes_cli.sqlite_safe_read import read_header_bytes_preopen
head = read_header_bytes_preopen(
path, length=max(16, probe_bytes), force=force
)
if not head or head.startswith(b"SQLite format 3"):
return False
return all(byte == 0 for byte in head)
def quarantine_zeroed_state_db(path: Path) -> Optional[Path]:
"""Move a zeroed state.db aside (preserve bytes) and return quarantine path.
Uses a cross-process lock (``#68805``) so two concurrent startups cannot
race: the first process moves the zeroed file and the second re-checks
under the lock, finding the file already gone (or a fresh DB in its place)
instead of clobbering the quarantine.
"""
import platform
lock_path = path.with_name(path.name + ".quarantine.lock")
lock_path.parent.mkdir(parents=True, exist_ok=True)
handle = lock_path.open("a+b")
acquired = False
try:
deadline = time.monotonic() + 5.0
if platform.system() == "Windows":
import msvcrt
while True:
try:
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
acquired = True
break
except OSError:
if time.monotonic() >= deadline:
break
time.sleep(0.020)
else:
import fcntl
while True:
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
acquired = True
break
except (BlockingIOError, OSError):
if time.monotonic() >= deadline:
break
time.sleep(0.020)
if not acquired:
# Fail closed: do NOT proceed without the lock. A slow or paused
# startup that still owns the lock can overlap this fallback and
# the two processes can act on the same live file (#68805 review).
logger.error(
"quarantine lock for %s not acquired within 5s — refusing to "
"quarantine without the cross-process lock. The zeroed file "
"is left in place. If sessions fail to load, restore from "
"state-snapshots via `hermes snapshot list` / "
"`hermes snapshot restore <id>`.",
path,
)
return None
# Re-check under the lock: another process may have already quarantined
# the file, leaving a fresh DB (or no file at all) in its place.
if not path.exists():
logger.info(
"quarantine_zeroed_state_db: %s already moved by another process",
path,
)
return None
if not is_zeroed_state_db(path):
logger.info(
"quarantine_zeroed_state_db: %s is no longer zeroed (another "
"process quarantined it and a fresh DB was created)",
path,
)
return None
try:
ts = time.strftime("%Y%m%d-%H%M%S")
except Exception:
ts = "unknown"
# Unique destination with PID suffix to avoid collision across
# concurrent startups that somehow both enter the lock.
dest = path.with_name(
f"{path.name}.zeroed-{ts}-{os.getpid()}.bak"
)
# Non-clobbering: if dest somehow exists, append a counter.
n = 0
while dest.exists():
n += 1
dest = path.with_name(
f"{path.name}.zeroed-{ts}-{os.getpid()}-{n}.bak"
)
try:
path.rename(dest)
except OSError as exc:
logger.error("Failed to quarantine zeroed %s: %s", path, exc)
return None
# Also move empty WAL/SHM if present so a fresh open is clean
for suffix in ("-wal", "-shm"):
side = Path(str(path) + suffix)
if side.exists():
try:
side.rename(Path(str(dest) + suffix))
except OSError:
pass
return dest
finally:
try:
if acquired:
if platform.system() == "Windows":
import msvcrt
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
except (OSError, AttributeError):
pass
finally:
handle.close()
class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin):
"""
SQLite-backed session storage with FTS5 search.
Thread-safe for the common gateway pattern (multiple reader threads,
single writer via WAL mode). Each method opens its own cursor.
"""
# ── Write-contention tuning ──
# With multiple hermes processes (gateway + CLI sessions + worktree agents)
# all sharing one state.db, WAL write-lock contention causes visible TUI
# freezes. SQLite's built-in busy handler uses a deterministic sleep
# schedule that causes convoy effects under high concurrency.
#
# Instead, we keep the SQLite timeout short (1s) and handle retries at the
# application level with random jitter, which naturally staggers competing
# writers and avoids the convoy.
#
# Patience is TIME-based, not attempt-based. A shared state.db is
# legitimately held for multi-second stretches by sibling Hermes
# processes: a TRUNCATE checkpoint at close on a large WAL, VACUUM after
# an auto-prune, offline recovery, or an older still-running process
# whose FTS maintenance predates the bounded-merge protocol (every
# `hermes update` leaves mixed-version processes sharing the DB until
# the old ones exit). An attempt-counted budget (~15s incidental worst
# case) silently loses that race and surfaces as
# session_persistence_failed — a destroyed turn — even though the store
# is healthy and merely busy (#74478).
#
# Two budgets: routine writes give up after _WRITE_PATIENCE_S so
# background/UI callers don't stall excessively, while transcript
# writes (append_message / session-row creation — the ones whose
# failure aborts the user's turn) ride out anything shorter than
# _TRANSCRIPT_WRITE_PATIENCE_S. Jitter stays small for the first
# _WRITE_RETRY_SLOW_AFTER_S (fast reclaim on millisecond contention),
# then backs off so a long hold isn't hammered with BEGIN IMMEDIATE
# attempts.
_WRITE_PATIENCE_S = 20.0
_TRANSCRIPT_WRITE_PATIENCE_S = 60.0
_WRITE_RETRY_MIN_S = 0.020 # 20ms
_WRITE_RETRY_MAX_S = 0.150 # 150ms
_WRITE_RETRY_SLOW_AFTER_S = 2.0
_WRITE_RETRY_SLOW_MIN_S = 0.250 # 250ms
_WRITE_RETRY_SLOW_MAX_S = 1.000 # 1s
# Attempt a WAL checkpoint every N successful writes (PASSIVE mode).
_CHECKPOINT_EVERY_N_WRITES = 50
# Retain the existing coarse 1000-write maintenance cadence, but replace
# the unbounded FTS5 ``'optimize'`` (measured holding the write lock for
# 9-18 s per index on a 10 GB production DB — longer than a competing
# writer's full retry patience, surfacing as "database is locked" /
# session_persistence_failed) with bounded ``'merge'`` commands. A
# positive merge rank is an approximate output-page budget, so each
# command holds the write lock for milliseconds; up to
# ``_FTS_MERGE_COMMANDS_PER_PASS`` commands run per index per cadence,
# stopping early on the documented no-progress signal. ``usermerge`` is
# lowered to 2 so positive merges act on any level with >= 2 segments —
# without that, levels below the default threshold of 4 are skipped and
# a fragmented index never converges (SQLite FTS5 §6.8-6.9).
_FTS_MERGE_EVERY_N_WRITES = 1000
_FTS_MERGE_MAX_PAGES_PER_INDEX = 500
_FTS_MERGE_COMMANDS_PER_PASS = 4
# Session imports intentionally use a lower cap than exports: import holds
# one BEGIN IMMEDIATE transaction, so bounded batches avoid starving live
# gateway/CLI writers. The dashboard accepts one exported JSON/JSONL file
# at a time, so these still cover normal history restores.
_IMPORT_MAX_SESSIONS = 500
_IMPORT_MAX_MESSAGES_PER_SESSION = 10_000
_IMPORT_MAX_TOTAL_MESSAGES = 50_000
_IMPORT_MAX_SESSION_BYTES = 5 * 1024 * 1024
_IMPORT_MAX_TOTAL_BYTES = 25 * 1024 * 1024
def __init__(self, db_path: Path = None, read_only: bool = False):
self.db_path = db_path or _default_db_path()
self.read_only = read_only
self._lock = threading.Lock()
# Read-path split (WAL only): recall/browse queries run on per-thread
# read-only connections so they never queue behind writer flushes on
# self._lock. See _read_ctx().
self._read_local = threading.local()
# Strong set of all live read connections across all threads. We
# hold a reference so short-lived reader threads' connections are
# not GC'd without close() — that would leak tracked fds in
# _live_connections. close() drains this set.
self._read_conns: "set[sqlite3.Connection]" = set()
self._read_conns_lock = threading.Lock()
# Set when close() begins. _get_read_conn checks this under the
# lock so a reader that finishes opening after the drain finds the
# shutdown in progress and closes its own connection immediately.
self._read_conns_closed = False
self._wal_active = False
self._write_count = 0
# One-shot guard for the runtime FTS rebuild recovery on the write
# path. A corrupt FTS shadow table makes EVERY message write raise
# the malformed/corrupt error class via the sync triggers; we repair
# in place at most once per SessionDB instance so a genuinely
# unrecoverable database can't put writers into a rebuild loop.
self._fts_runtime_rebuild_attempted = False
# One-shot guard for the usermerge-floor config write on the
# incremental FTS merge cadence (see _merge_fts_incrementally).
self._fts_usermerge_floor_applied = False
self._fts_enabled = False
self._trigram_available = False
# CJK-bigram index (cjk_unicode61 loadable tokenizer). _fts_cjk_loaded:
# extension present on the writer connection; _fts_cjk_available: the
# messages_fts_cjk table is queryable AND not marked stale. Set during
# _init_schema / _probe_fts_cjk.
self._fts_cjk_loaded = False
self._fts_cjk_available = False
self._fts_unavailable_warned = False
self._conn = None
# Async token accounting (see queue_token_counts). The condition
# guards queue + writer state; it is distinct from self._lock so
# enqueue/flush bookkeeping never contends with SQLite writes.
self._token_queue: deque = deque()
self._token_queue_cond = threading.Condition(threading.Lock())
self._token_writer_thread: Optional[threading.Thread] = None
self._token_writer_stop = False
self._token_writer_busy = False
try:
if read_only:
# Read-only attach for cross-profile aggregation: SELECT-only,
# so we skip schema init entirely (no DDL, no FTS probe, no
# column reconcile). Crucially this takes NO write lock, so
# polling another profile's live DB on every sidebar refresh
# never contends with that profile's running backend. The DB
# must already exist + be initialised (callers guard on
# db_path.exists()); a SELECT against an empty file raises and
# the caller degrades per-profile.
self._conn = _connect_tracked_db(
f"file:{self.db_path}?mode=ro",
tracking_path=self.db_path,
uri=True,
check_same_thread=False,
timeout=1.0,
isolation_level=None,
)
self._conn.row_factory = sqlite3.Row
return
self.db_path.parent.mkdir(parents=True, exist_ok=True)
# Read-only file/sidecar preflight (port of kilocode#12508):
# repair-or-refuse BEFORE the first connection so users get an
# actionable message instead of an opaque "attempt to write a
# readonly database" from deep inside _init_schema.
if not read_only:
preflight_db_writability(self.db_path, db_label="state.db")
# #68474: zeroed state.db (size>0, all-NUL header) used to fail as a
# generic "file is not a database" with no recovery path. Quarantine
# the bytes (do not delete) and continue so a fresh DB can open;
# point the operator at pre-update snapshots.
if (
not read_only
and self.db_path.exists()
and is_zeroed_state_db(self.db_path)
):
try:
zsize = self.db_path.stat().st_size
except OSError:
zsize = -1
qpath = quarantine_zeroed_state_db(self.db_path)
snaps = self.db_path.parent / "state-snapshots"
msg = (
f"state.db looks ZEROED ({zsize} bytes, no SQLite header). "
f"Preserved at {qpath or '(quarantine failed — file left in place)'}. "
f"Restore from {snaps} via `hermes snapshot list` / "
f"`hermes snapshot restore <id>` if available. "
"Opening a fresh empty database so the agent can start."
)
logger.error(msg)
_set_last_init_error(msg)
# If quarantine failed, do not open the zeroed file (would fail
# opaquely or risk further damage). Raise with the clear message.
if qpath is None and self.db_path.exists() and is_zeroed_state_db(self.db_path):
raise sqlite3.DatabaseError(msg)
def _connect_and_init():
self._conn = _connect_tracked_db(
str(self.db_path),
check_same_thread=False,
# Short timeout — application-level retry with random
# jitter handles contention instead of sitting in
# SQLite's internal busy handler for up to 30s.
timeout=1.0,
# auto-starts transactions on DML, which conflicts with
# our explicit BEGIN IMMEDIATE. None = we manage
# transactions ourselves.
isolation_level=None,
)
self._conn.row_factory = sqlite3.Row
self._wal_active = (
apply_wal_with_fallback(self._conn, db_label="state.db") == "wal"
)
apply_database_pragmas(self._conn, db_label="state.db")
self._conn.execute("PRAGMA foreign_keys=ON")
self._fts_cjk_loaded = load_fts5_cjk_extension(self._conn)
self._init_schema()
def _connect_and_init_with_lock_patience():
# Lock contention during open: _init_schema's DDL/reconcile
# statements run on a 1s-timeout connection with no retry, so
# a sibling process holding the write lock (VACUUM, TRUNCATE
# checkpoint at close, a long FTS pass from an older
# still-running install) used to fail the ENTIRE open —
# callers then disable persistence for the whole run
# ("Failed to initialize SessionDB ... database is locked",
# #74478). The store is healthy; wait it out with the same
# jittered patience the write path uses. Non-lock errors
# (including the malformed class) propagate immediately.
deadline = time.monotonic() + self._WRITE_PATIENCE_S
while True:
try:
_connect_and_init()
return
except sqlite3.OperationalError as exc:
err = str(exc).lower()
if "locked" not in err and "busy" not in err:
raise
try:
if self._conn is not None:
self._conn.close()
except Exception:
pass
now = time.monotonic()
if now >= deadline:
raise
time.sleep(
min(
random.uniform(
self._WRITE_RETRY_SLOW_MIN_S,
self._WRITE_RETRY_SLOW_MAX_S,
),
max(deadline - now, 0.001),
)
)
try:
_connect_and_init_with_lock_patience()
except sqlite3.DatabaseError as exc:
# The malformed-schema class (e.g. a duplicate sqlite_master
# row for messages_fts) fails on the very first statement —
# before _init_schema can run — so it can't be caught at the
# FTS-rebuild layer. Recover by repairing sqlite_master in
# place (backup first; canonical sessions/messages preserved),
# then reopen once. This is what lets Desktop/Dashboard
# self-heal instead of silently showing "no sessions".
if not is_malformed_db_error(exc) or not _claim_repair_attempt(self.db_path):
raise
logger.error(
"state.db schema is malformed (%s) — attempting automatic "
"repair (a backup copy is made first).", exc,
)
try:
if self._conn is not None:
self._conn.close()
except Exception:
pass
report = repair_state_db_schema(self.db_path)
if not report.get("repaired"):
raise
_connect_and_init_with_lock_patience()
# NOTE: the v23 FTS optimization is OPT-IN (`hermes db optimize`),
# never auto-started on open. Legacy installs keep their working
# v22 inline FTS untouched here; only the explicit foreground
# command demotes + rebuilds. This avoids a background worker
# racing session lifecycle and the surprise disk/latency cost on
# an unattended open. (An interrupted optimize resumes when the
# user re-runs the command.)
except Exception as exc:
# Capture the cause so /resume and friends can surface WHY the
# session DB is unavailable instead of a bare "Session database
# not available." Callers that catch this exception keep their
# existing ``self._session_db = None`` degradation path.
#
# Note: we deliberately do NOT clear _last_init_error on the
# success path (no else branch). In multi-threaded callers
# (gateway, web_server per-request SessionDB()), a concurrent
# successful open racing past this failure would erase the
# cause that another thread's /resume is about to format.
# Tests that need to reset the state can call
# ``hermes_state._set_last_init_error(None)`` explicitly.
_set_last_init_error(f"{type(exc).__name__}: {exc}")
raise
# ── Read-path split ──
def _get_read_conn(self) -> Optional[sqlite3.Connection]:
"""Per-thread read-only connection, or None when unavailable.
Only used under WAL: WAL readers see a consistent snapshot and never
block on (or get blocked by) the writer, so recall/browse queries can
skip self._lock entirely. Under DELETE journal mode (NFS fallback) a
reader can hit SQLITE_BUSY storms during writes, so we keep the
legacy locked single-connection path there.
Fresh read transactions begin per statement (autocommit), so each
query observes everything committed so far — read-your-writes holds
for the flush-then-search patterns in a turn.
"""
if not self._wal_active or self.read_only:
return None
conn = getattr(self._read_local, "conn", None)
if conn is not None:
return conn
if getattr(self._read_local, "failed", False):
return None
try:
conn = _connect_tracked_db(
f"file:{self.db_path}?mode=ro",
tracking_path=self.db_path,
uri=True,
timeout=5.0,
isolation_level=None,
)
conn.row_factory = sqlite3.Row
# Load the CJK tokenizer extension on this connection so
# messages_fts_cjk queries work on the read path. The .so
# registers the tokenizer in the connection's in-memory
# registry, not the database file, so mode=ro is fine.
if self._fts_cjk_loaded:
load_fts5_cjk_extension(conn)
with self._read_conns_lock:
if self._read_conns_closed:
# close() already drained — don't register; close
# immediately so no tracked fd leaks.
conn.close()
self._read_local.failed = True
return None
self._read_conns.add(conn)
except sqlite3.Error:
# Mark this thread failed so we don't retry the open on every
# query; the locked writer connection still serves reads.
self._read_local.failed = True
logger.debug("read-only connection open failed for %s", self.db_path, exc_info=True)
return None
self._read_local.conn = conn
return conn
@contextmanager
def _read_ctx(self):
"""Yield a connection for read-only statements.
WAL: a per-thread read-only connection with NO lock — recall queries
never convoy behind writer flushes (the gateway shares one SessionDB
across every agent, so this lock was a global choke point).
Non-WAL or read-conn failure: the shared writer connection under
self._lock, byte-for-byte the legacy behavior.
"""
conn = self._get_read_conn()
if conn is not None:
yield conn
return
with self._lock:
yield self._conn
# ── Core write helper ──
@staticmethod
def _is_fts5_unavailable_error(exc: sqlite3.OperationalError) -> bool:
err = str(exc).lower()
if "no such module" in err and "fts5" in err:
return True
# SQLite builds that have FTS5 but lack the optional trigram tokenizer
# raise "no such tokenizer: trigram" instead of "no such module".
# Scope to trigram specifically to avoid masking unrelated tokenizer errors.
if "no such tokenizer: trigram" in err:
return True
# The cjk_unicode61 tokenizer is a loadable extension — a process
# that couldn't load it sees the same capability-error shape.
if "no such tokenizer: cjk_unicode61" in err:
return True
return False
@staticmethod
def _is_trigram_unavailable_error(exc: sqlite3.OperationalError) -> bool:
"""True when only an optional tokenizer is missing (FTS5 itself works).
Covers the built-in trigram tokenizer (needs SQLite >= 3.34) and the
loadable cjk_unicode61 tokenizer — both mean "this one index can't be
served here", never "disable FTS".
"""
err = str(exc).lower()
return (
"no such tokenizer: trigram" in err
or "no such tokenizer: cjk_unicode61" in err
)
@staticmethod
def _db_has_legacy_inline_fts(cursor: sqlite3.Cursor) -> bool:
"""True when messages_fts exists in ANY pre-v23 shape.
v23's messages_fts is external-content over THREE real columns
(content, tool_name, tool_calls). Every pre-v23 shape lacks the
tool_name/tool_calls columns — whether the old inline single-column
form (v11..v22) or the even older external-content single-column form
(v10-era, pre-#16751). We therefore detect "needs optimize" as "the
stored CREATE lacks the tool_name column", which is the precise v23
marker and correctly catches BOTH legacy variants.
Returns False when messages_fts doesn't exist yet (fresh DB mid-init):
the post-migration FTS setup block will create it in the v23 shape.
"""
row = cursor.execute(
"SELECT sql FROM sqlite_master "
"WHERE type = 'table' AND name = 'messages_fts'"
).fetchone()
if row is None:
return False
sql = (row[0] if not isinstance(row, sqlite3.Row) else row["sql"]) or ""
# The v23 table declares tool_name/tool_calls columns. Their absence
# means a legacy shape that doesn't index tool metadata → optimize.
return "tool_name" not in sql
def _warn_trigram_unavailable(self, exc: sqlite3.OperationalError) -> None:
"""Log once that the trigram tokenizer is missing; base FTS5 stays enabled."""
if getattr(self, "_trigram_unavailable_warned", False):
return
self._trigram_unavailable_warned = True
logger.info(
"SQLite trigram tokenizer unavailable for %s "
"(requires SQLite >= 3.34, this build is %s); "
"CJK/substring search will fall back to LIKE: %s",
self.db_path,
sqlite3.sqlite_version,
exc,
)
def _warn_fts5_unavailable(self, exc: sqlite3.OperationalError) -> None:
self._fts_enabled = False
if self._fts_unavailable_warned:
return
self._fts_unavailable_warned = True
logger.warning(
"SQLite FTS5 unavailable for %s; full-text session search "
"disabled. Run `hermes update` to rebuild the venv with a "
"current Python (managed uv guarantees FTS5). "
"(underlying error: %s)",
self.db_path,
exc,
)
def _ensure_fts_cjk_schema(self, cursor) -> None:
"""Create / repair / self-heal the CJK-bigram index surface.
``cursor`` may be a Cursor or a Connection (both expose execute /
executescript). Called only for v23-shape DBs with the base FTS
surface healthy. Sets ``self._fts_cjk_available``. Never raises;
every failure mode degrades to "no cjk index" (trigram/LIKE routing
keeps working).
Cases:
tokenizer loaded, table absent → create. Empty DB: index is
complete by construction (triggers cover everything). Populated
DB: set the cjk backfill markers so the id-gated triggers stay
correct and `optimize-storage` can backfill; the index is NOT
served until the backfill completes.
tokenizer loaded, table present → ensure triggers (recreates any
dropped by a tokenizer-less process), honour the stale
breadcrumb (serve only when absent and no backfill pending).
tokenizer NOT loaded, table present with live triggers → drop the
cjk triggers so message INSERTs don't fail at trigger time,
and leave the stale breadcrumb (#self-heal). The table itself
stays for a later capable open to rebuild.
"""
cjk_present = bool(cursor.execute(
"SELECT 1 FROM sqlite_master WHERE type = 'table' "
"AND name = 'messages_fts_cjk'"
).fetchone())
if not self._fts_cjk_loaded:
if cjk_present:
live = [
r[0] for r in cursor.execute(
"SELECT name FROM sqlite_master WHERE type = 'trigger' "
f"AND name IN ({','.join('?' for _ in _FTS_CJK_TRIGGERS)})",
_FTS_CJK_TRIGGERS,
).fetchall()
]
if live:
# Self-heal: this process cannot tokenize, so every
# message INSERT would die inside the cjk trigger.
# Breadcrumb FIRST (crash between the two statements is
# merely conservative), then drop.
logger.warning(
"messages_fts_cjk triggers present but the "
"cjk_unicode61 tokenizer is unavailable (%s) — "
"dropping the cjk triggers so message writes keep "
"working. CJK search falls back to trigram/LIKE; "
"run `hermes sessions optimize-storage` on a host "
"with the extension to rebuild.",
fts5_cjk_so_path(),
)
cursor.execute(
"INSERT INTO state_meta (key, value) VALUES (?, '1') "
"ON CONFLICT(key) DO UPDATE SET value = '1'",
(FTS_CJK_STALE_KEY,),
)
for trig in live:
cursor.execute(f"DROP TRIGGER IF EXISTS {trig}")
self._fts_cjk_available = False
return
try:
cursor.executescript(FTS_CJK_TABLE_SQL)
if not cjk_present:
# Freshly created. An empty DB's index is complete by
# construction (triggers will cover every future row); a
# populated DB (e.g. a v23 install predating the cjk index)
# gets the dedicated marker pair so the id-gated triggers
# keep NEW rows indexed while old rows await the
# `optimize-storage` backfill. Either way any old stale
# breadcrumb refers to a table that no longer exists.
cursor.execute(
"DELETE FROM state_meta WHERE key = ?",
(FTS_CJK_STALE_KEY,),
)
n_msgs = cursor.execute(
"SELECT COUNT(*) FROM messages WHERE role <> 'tool'"
).fetchone()[0]
if n_msgs > 0:
hw = cursor.execute(
"SELECT COALESCE(MAX(id), 0) FROM messages"
).fetchone()[0]
for k, v in (
("fts_cjk_rebuild_high_water", str(hw)),
("fts_cjk_rebuild_progress", "0"),
):
cursor.execute(
"INSERT INTO state_meta (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
(k, v),
)
stale = cursor.execute(
"SELECT 1 FROM state_meta WHERE key = ?",
(FTS_CJK_STALE_KEY,),
).fetchone()
if stale:
# A tokenizer-less process dropped the triggers at some
# unknown point — the index has a gap of unknown extent.
# Do NOT reinstall triggers (an external-content 'delete'
# for an unindexed rowid corrupts the index); the next
# `optimize-storage` run rebuilds from scratch.
self._fts_cjk_available = False
return
cursor.executescript(FTS_CJK_TRIGGER_SQL)
backfill_pending = cursor.execute(
"SELECT 1 FROM state_meta "
"WHERE key = 'fts_cjk_rebuild_high_water' LIMIT 1"
).fetchone()
self._fts_cjk_available = not backfill_pending
except sqlite3.OperationalError:
# Includes "no such tokenizer: cjk_unicode61" if the extension
# loaded but registration failed — degrade to trigram/LIKE.
logger.warning(
"messages_fts_cjk ensure failed; CJK search stays on "
"trigram/LIKE", exc_info=True,
)
self._fts_cjk_available = False
@staticmethod
def _drop_fts_triggers(cursor: sqlite3.Cursor) -> None:
for trigger in _FTS_TRIGGERS:
try:
cursor.execute(f"DROP TRIGGER IF EXISTS {trigger}")
except sqlite3.OperationalError:
pass
def _ensure_fts_schema(
self,
cursor: sqlite3.Cursor,
table_name: str,
ddl: str,
) -> bool:
status = self._fts_table_probe(cursor, table_name)
if status is None:
return False
try:
# Run even when the virtual table exists so any dropped or missing
# triggers are recreated after a previous no-FTS5 runtime disabled
# them to keep message writes working.
cursor.executescript(ddl)
return True
except sqlite3.OperationalError as exc:
if not self._is_fts5_unavailable_error(exc):
raise
# Only disable FTS entirely when the whole FTS5 module is missing.
# A missing specific tokenizer (e.g. trigram) means only that
# particular table cannot be created — the base FTS5 table is fine.
if self._is_trigram_unavailable_error(exc):
self._warn_trigram_unavailable(exc)
else:
self._warn_fts5_unavailable(exc)
return False
def _execute_write(
self,
fn: Callable[[sqlite3.Connection], T],
patience_s: Optional[float] = None,
) -> T:
"""Execute a write transaction with BEGIN IMMEDIATE and jitter retry.
*fn* receives the connection and should perform INSERT/UPDATE/DELETE
statements. The caller must NOT call ``commit()`` — that's handled
here after *fn* returns.
BEGIN IMMEDIATE acquires the WAL write lock at transaction start
(not at commit time), so lock contention surfaces immediately.
On ``database is locked``, we release the Python lock, sleep a
random jitter, and retry — breaking the convoy pattern that
SQLite's built-in deterministic backoff creates.
*patience_s* is the total time budget for lock retries (default
``_WRITE_PATIENCE_S``). Transcript-critical writes pass
``_TRANSCRIPT_WRITE_PATIENCE_S`` so a sibling process holding the
lock for a legitimate long operation (VACUUM, TRUNCATE checkpoint,
pre-bounded-merge FTS optimize from an older still-running
install) exhausts routine writers' patience without destroying a
user turn. Jitter starts small (20-150ms) for fast reclaim on
millisecond contention and backs off to 250ms-1s once the lock has
been held longer than ``_WRITE_RETRY_SLOW_AFTER_S``.
Returns whatever *fn* returns.
"""
if patience_s is None:
patience_s = self._WRITE_PATIENCE_S
deadline = time.monotonic() + patience_s
while True:
try:
with self._lock:
self._conn.execute("BEGIN IMMEDIATE")
try:
result = fn(self._conn)
self._conn.commit()
except BaseException:
try:
self._conn.rollback()
except Exception:
pass
raise
# Success — periodic best-effort checkpoint + FTS merge.
self._write_count += 1
if self._write_count % self._CHECKPOINT_EVERY_N_WRITES == 0:
self._try_wal_checkpoint()
if self._write_count % self._FTS_MERGE_EVERY_N_WRITES == 0:
self._try_incremental_merge_fts()
return result
except sqlite3.OperationalError as exc:
err_msg = str(exc).lower()
if "locked" in err_msg or "busy" in err_msg:
now = time.monotonic()
if now < deadline:
elapsed = now - (deadline - patience_s)
if elapsed >= self._WRITE_RETRY_SLOW_AFTER_S:
jitter = random.uniform(
self._WRITE_RETRY_SLOW_MIN_S,
self._WRITE_RETRY_SLOW_MAX_S,
)
else:
jitter = random.uniform(
self._WRITE_RETRY_MIN_S,
self._WRITE_RETRY_MAX_S,
)
# Never overshoot the deadline by a full slow-jitter.
time.sleep(min(jitter, max(deadline - now, 0.001)))
continue
# Patience exhausted — say what actually happened so the
# surfaced error doesn't read as disk/permission damage.
raise sqlite3.OperationalError(
f"database is locked (another Hermes process held the "
f"state.db write lock for over {patience_s:.0f}s — "
"likely a long maintenance operation such as VACUUM, "
"a large WAL checkpoint, or an older pre-update "
"process; the database itself is healthy)"
) from exc
# Non-lock error — propagate.
raise
except sqlite3.DatabaseError as exc:
# Corrupt FTS shadow tables make every write raise the
# malformed/corrupt error class through the FTS sync triggers
# while the canonical messages table is intact. The gateway
# session store has its own retry queue for transcript
# appends (#65637 salvage), but cron and CLI writers call
# SessionDB directly — without this, their writes hard-fail
# until the next process restart triggers the offline repair.
# Rebuild the FTS index in place (once per instance) via
# rebuild_fts() and retry the failed write immediately.
if not self._try_runtime_fts_rebuild(exc):
raise
continue
@staticmethod
def _is_fts_write_corruption_error(exc: sqlite3.DatabaseError) -> bool:
"""True for the error class a corrupt FTS index raises on writes.
The message varies by SQLite version: older builds raise the generic
``database disk image is malformed`` (covered by
``is_malformed_db_error``); newer builds (e.g. ubuntu-latest CI)
raise the FTS5-specific ``fts5: corrupt structure record for table
"messages_fts"``. Both mean the same thing for the write path: the
canonical rows are fine, the FTS shadow tables are not.
"""
if is_malformed_db_error(exc):
return True
msg = str(exc).lower()
return "fts5" in msg and "corrupt" in msg
def _try_runtime_fts_rebuild(self, exc: sqlite3.DatabaseError) -> bool:
"""One-shot in-place FTS rebuild after a corrupt-index write failure.
Returns True when a rebuild was performed and the failed write should
be retried; False when the error isn't the FTS-corruption class, FTS
is disabled, or a rebuild was already attempted for this instance.
Delegates to :meth:`rebuild_fts` (the FTS5 ``'rebuild'`` command —
index rewritten from the canonical messages table, zero message-row
mutation). Safe to call from ``_execute_write``'s except path: the
failed transaction was rolled back and ``self._lock`` released before
the exception propagated, and ``rebuild_fts`` re-acquires it.
E2E-verified: a corrupted ``messages_fts_data`` shadow table rejects
every append; after the in-place rebuild the same append succeeds and
search works again.
"""
if self._fts_runtime_rebuild_attempted:
return False
if not self._fts_enabled:
return False
if not self._is_fts_write_corruption_error(exc):
return False
self._fts_runtime_rebuild_attempted = True
logger.warning(
"state.db write failed with an FTS-corruption error (%s) — "
"attempting one-shot in-place FTS rebuild; canonical message "
"rows are preserved.", exc,
)
try:
rebuilt = self.rebuild_fts()
except Exception as rebuild_exc:
logger.error(
"In-place FTS rebuild failed (%s); the database needs the "
"full offline repair path (repair_state_db_schema).",
rebuild_exc,
)
return False
if not rebuilt:
logger.error(
"In-place FTS rebuild made no progress; the database needs "
"the full offline repair path (repair_state_db_schema)."
)
return False
logger.warning(
"state.db FTS indexes rebuilt in place (%d); retrying the failed write.",
rebuilt,
)
return True
def _try_wal_checkpoint(self) -> None:
"""Best-effort PASSIVE WAL checkpoint. Never raises.
Flushes committed WAL frames back into the main DB file without
requiring an exclusive lock. PASSIVE is safe for frequent
periodic use because it does not block concurrent writers and
cannot corrupt B-tree pages under I/O pressure.
PASSIVE does not truncate the WAL file — it stays at its
high-water mark. WAL truncation happens in :meth:`close`
(TRUNCATE) and pre-VACUUM checkpoints, which run infrequently
under controlled conditions.
Previous TRUNCATE strategy caused B-tree corruption on large
databases (65K+ pages) due to the exclusive-lock I/O pressure
from checkpointing thousands of frames at once (issue #45383).
"""
try:
with self._lock:
result = self._conn.execute(
"PRAGMA wal_checkpoint(PASSIVE)"
).fetchone()
if result and result[1] > 0:
logger.debug(
"WAL checkpoint: %d/%d pages checkpointed",
result[2], result[1],
)
except Exception as exc:
logger.warning("WAL checkpoint (PASSIVE) failed: %s", exc)
def close(self):
"""Close the database connection.
Drains queued token deltas first (the background writer needs the
connection), then attempts a TRUNCATE WAL checkpoint so that
exiting processes help shrink the WAL file.
"""
self._stop_token_writer()
# The atexit hook holds a strong reference to this instance (bound
# method); without unregistering, every closed SessionDB stays
# reachable until interpreter exit. Bound methods compare equal by
# (instance, function), so this removes exactly our registration;
# no-op when the writer never started.
atexit.unregister(self._drain_token_queue_at_exit)
# Close all read-only connections across all threads. Per-thread
# connections live in threading.local() and would otherwise be GC'd
# without calling close(), leaking tracked fds in _live_connections.
# The strong set holds references so short-lived reader threads'
# connections survive until close() drains them. Setting the closed
# flag under the lock prevents a reader from registering a new
# connection after the drain.
with self._read_conns_lock:
self._read_conns_closed = True
read_conns = list(self._read_conns)
self._read_conns.clear()
for conn in read_conns:
try:
conn.close()
except Exception:
pass
self._read_local.conn = None
with self._lock:
if self._conn:
try:
self._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
except Exception as exc:
logger.debug("WAL checkpoint (TRUNCATE) at close failed: %s", exc)
self._conn.close()
self._conn = None
# ── Chunked FTS rebuild engine (v23 opt-in optimize) ──
#
# `optimize_fts_storage()` (the `hermes sessions optimize-storage`
# command) drops the legacy inline FTS indexes and backfills the new
# external-content ones. A single blocking rebuild measured ~16 minutes
# of held write lock on a real 25 GB DB, so the backfill runs in small
# chunks, each in its own short write transaction:
# - concurrent readers/writers are never starved (WAL stays small,
# each chunk checkpoints via the normal _execute_write cadence);
# - an interrupted run (Ctrl-C, crash) resumes from
# fts_rebuild_progress when the command is re-run;
# - multiple processes sharing the DB don't double-run it — each chunk
# claims work by compare-and-swap on fts_rebuild_progress, so even a
# concurrent second runner just interleaves chunks safely.
#
# THROTTLING (the part that keeps a live gateway sharing the DB
# responsive): a greedy chunk loop re-acquires BEGIN IMMEDIATE nearly
# back-to-back and can starve another process's writer into exhausting
# its lock retries (an early 5000-row/50ms version owned the write lock
# ~85% of the time and visibly froze concurrent CLI sessions on a large
# install). Two layers prevent that:
# 1. Small chunks (500 rows) — a foreground write queues behind a
# chunk for at most ~tens of ms.
# 2. Inter-chunk pause — the loop sleeps max(_FTS_REBUILD_MIN_PAUSE,
# chunk cost x _FTS_REBUILD_DUTY_FACTOR) between chunks, capping
# this process's share of DB bandwidth so concurrent writers always
# find open windows. This works cross-process (unlike any
# same-process activity stamp) because it bounds our own duty
# cycle unconditionally.
_FTS_REBUILD_CHUNK_ROWS = 500
_FTS_REBUILD_DUTY_FACTOR = 4.0 # sleep >= 4x chunk cost (≤20% duty)
_FTS_REBUILD_MIN_PAUSE = 0.2 # seconds — floor between chunks
# Demoted v22 FTS shadow tables awaiting teardown (see the v23 migration:
# DROP of a multi-GB FTS vtable blocks for minutes, so the migration
# demotes the vtable definitions out of sqlite_master and renames the
# orphaned shadow tables — now plain tables — to fts_v22_trash_*; the
# worker empties them in bounded chunks, then drops them cheaply).
_FTS_TRASH_PREFIX = "fts_v22_trash_"
# ── CJK-bigram index backfill (dedicated marker pair) ──
#
# Same chunk engine as the main deferred rebuild, but on the
# ``fts_cjk_rebuild_*`` markers so a cjk-only backfill (the common case:
# an already-optimized v23 DB gaining the cjk index) never gates the
# complete ``messages_fts`` / trigram triggers.
# ── Opt-in v23 FTS storage optimization (`hermes sessions optimize-storage`) ──
#
# This is the ONLY path that migrates an existing legacy (v22 inline) DB
# to the v23 external-content schema. It is deliberately foreground and
# user-invoked, never automatic, because it is disk-heavy and long. It
# runs the throttled/resumable chunk engine above to completion
# synchronously — demote → new schema → chunked backfill → chunked
# teardown — with progress callbacks, a disk preflight in the CLI
# wrapper, a VACUUM at the end, and a defensive schema_version bump.
def _has_fts_trash(self, conn) -> bool:
"""True when demoted v22 shadow tables are still awaiting teardown.
Caller must hold ``self._lock`` (or pass a migration-time cursor)."""
return bool(conn.execute(
"SELECT 1 FROM sqlite_master WHERE type = 'table' "
"AND name LIKE ? ESCAPE '\\' LIMIT 1",
(self._FTS_TRASH_PREFIX.replace("_", "\\_") + "%",),
).fetchone())
# =========================================================================
# Session lifecycle
# =========================================================================
def _insert_session_row(
self,
session_id: str,
source: str,
model: str = None,
model_config: Dict[str, Any] = None,
system_prompt: str = None,
user_id: str = None,
session_key: str = None,
chat_id: str = None,
chat_type: str = None,
thread_id: str = None,
parent_session_id: str = None,
cwd: str = None,
profile_name: str = None,
git_repo_root: str = None,
) -> None:
"""Insert a session row, enriching NULL metadata on conflict.
The gateway's ``get_or_create_session`` creates a bare row (source +
user_id) *before* the agent exists; the agent's later
``create_session`` then carries the real ``model`` / ``model_config`` /
``system_prompt``. A plain ``INSERT OR IGNORE`` silently dropped that
enrichment, leaving gateway sessions with NULL model/billing metadata.
The ``ON CONFLICT`` upsert backfills those fields via ``COALESCE`` —
only filling columns that are still NULL, never overwriting values an
earlier writer already set (so a later bare call with source="unknown"
can't clobber a real source/model).
``chat_id``/``thread_id`` record the messaging origin (the chat/room and
thread the session was started in) so that gateway ``/resume`` can prove
a persisted, now-inactive row belongs to the caller's chat/thread before
switching to it (IDOR scoping — without them the ``sessions`` table has
no chat/thread to compare).
When ``parent_session_id`` is set (compression fork, delegate/subagent
spawn, branch continuation) and this row's own ``cwd``/``git_repo_root``/
``git_branch``/``profile_name`` are still NULL after the insert, they are
backfilled from the parent row. Callers of ``create_session`` for a child
session historically didn't propagate these fields themselves (e.g. the
compression-fork path), so a lineage could silently lose its working
directory and drop out of the project sidebar every time it forked
(#64709), or lose its owning profile and be aggregated as "default" every
time it rotated or branched (the cross-profile session-jump bug). This
only fills NULLs — an explicit value on the child is never overwritten.
For compression forks specifically
(parent ended with ``end_reason='compression'``), the gateway origin
columns (``user_id``/``session_key``/``chat_id``/``chat_type``/
``thread_id``/``display_name``/``origin_json``) are inherited too, so a
crash before the gateway re-records the peer can't strand the child
without a recoverable routing mapping (#59527).
"""
def _do(conn):
conn.execute(
"""INSERT INTO sessions (
id, source, user_id, session_key, chat_id, chat_type, thread_id,
model, model_config, system_prompt, parent_session_id, cwd,
profile_name, git_repo_root, started_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
model = COALESCE(sessions.model, excluded.model),
model_config = COALESCE(sessions.model_config, excluded.model_config),
system_prompt = COALESCE(sessions.system_prompt, excluded.system_prompt),
session_key = COALESCE(sessions.session_key, excluded.session_key),
chat_id = COALESCE(sessions.chat_id, excluded.chat_id),
chat_type = COALESCE(sessions.chat_type, excluded.chat_type),
thread_id = COALESCE(sessions.thread_id, excluded.thread_id),
parent_session_id = COALESCE(sessions.parent_session_id, excluded.parent_session_id),
cwd = COALESCE(sessions.cwd, excluded.cwd),
profile_name = COALESCE(sessions.profile_name, excluded.profile_name),
git_repo_root = COALESCE(sessions.git_repo_root, excluded.git_repo_root)""",
(
session_id,
source,
user_id,
session_key,
chat_id,
chat_type,
thread_id,
model,
json.dumps(model_config) if model_config else None,
system_prompt,
parent_session_id,
cwd,
profile_name,
git_repo_root,
time.time(),
),
)
if parent_session_id:
conn.execute(
"""UPDATE sessions
SET cwd = COALESCE(sessions.cwd,
(SELECT p.cwd FROM sessions p
WHERE p.id = sessions.parent_session_id)),
git_repo_root = COALESCE(sessions.git_repo_root,
(SELECT p.git_repo_root FROM sessions p
WHERE p.id = sessions.parent_session_id)),
git_branch = COALESCE(sessions.git_branch,
(SELECT p.git_branch FROM sessions p
WHERE p.id = sessions.parent_session_id)),
profile_name = COALESCE(sessions.profile_name,
(SELECT p.profile_name FROM sessions p
WHERE p.id = sessions.parent_session_id))
WHERE id = ? AND parent_session_id IS NOT NULL""",
(session_id,),
)
# Belt-and-suspenders for gateway routing metadata (#59527):
# the gateway re-records the peer on the child after rotation
# (d5b4879d4), but a hard crash between child creation and that
# write leaves the child row without origin columns, so
# ``find_latest_gateway_session_for_peer`` can't recover the
# mapping on restart. Inherit them from the parent at creation
# time — but ONLY for compression forks (parent already ended
# with end_reason='compression'). Delegate/subagent children
# are spawned while the parent is still live and must NOT
# inherit routing keys, or peer recovery could repoint gateway
# traffic into a subagent's session.
conn.execute(
"""UPDATE sessions
SET user_id = COALESCE(sessions.user_id,
(SELECT p.user_id FROM sessions p
WHERE p.id = sessions.parent_session_id)),
session_key = COALESCE(sessions.session_key,
(SELECT p.session_key FROM sessions p
WHERE p.id = sessions.parent_session_id)),
chat_id = COALESCE(sessions.chat_id,
(SELECT p.chat_id FROM sessions p
WHERE p.id = sessions.parent_session_id)),
chat_type = COALESCE(sessions.chat_type,
(SELECT p.chat_type FROM sessions p
WHERE p.id = sessions.parent_session_id)),
thread_id = COALESCE(sessions.thread_id,
(SELECT p.thread_id FROM sessions p
WHERE p.id = sessions.parent_session_id)),
display_name = COALESCE(sessions.display_name,
(SELECT p.display_name FROM sessions p
WHERE p.id = sessions.parent_session_id)),
origin_json = COALESCE(sessions.origin_json,
(SELECT p.origin_json FROM sessions p
WHERE p.id = sessions.parent_session_id))
WHERE id = ? AND parent_session_id IS NOT NULL
AND EXISTS (
SELECT 1 FROM sessions p
WHERE p.id = sessions.parent_session_id
AND p.end_reason = 'compression'
)""",
(session_id,),
)
# Session-row creation is transcript-critical: if it fails, the
# first flush of a new session fails and the turn is aborted as
# session_persistence_failed. Ride out long sibling holds.
self._execute_write(_do, patience_s=self._TRANSCRIPT_WRITE_PATIENCE_S)
def create_session(self, session_id: str, source: str, **kwargs) -> str:
"""Create a new session record. Returns the session_id."""
self._insert_session_row(session_id, source, **kwargs)
return session_id
def record_gateway_session_peer(
self,
session_id: str,
*,
source: str,
user_id: str = None,
session_key: str = None,
chat_id: str = None,
chat_type: str = None,
thread_id: str = None,
display_name: str = None,
origin_json: str = None,
) -> None:
"""Persist the gateway routing peer for an existing session row.
``display_name`` / ``origin_json`` carry the gateway's presentation
and full origin metadata (#9006) so consumers (mcp_serve, mirror,
channel directory) can read routing data from state.db instead of
sessions.json. They are COALESCE'd only in the sense that ``None``
leaves the existing value untouched.
"""
if not session_id or not session_key:
return
def _do(conn):
conn.execute(
"""UPDATE sessions
SET session_key = ?, source = ?, user_id = ?, chat_id = ?,
chat_type = ?, thread_id = ?,
display_name = COALESCE(?, display_name),
origin_json = COALESCE(?, origin_json)
WHERE id = ?""",
(
session_key,
source,
user_id,
chat_id,
chat_type,
thread_id,
display_name,
origin_json,
session_id,
),
)
self._execute_write(_do)
def set_expiry_finalized(self, session_id: str, finalized: bool = True) -> None:
"""Mark a gateway session's expiry-finalization flag in state.db.
Mirrors ``SessionEntry.expiry_finalized`` (sessions.json) so the flag
survives even if the JSON index is pruned or lost (#9006).
"""
if not session_id:
return
def _do(conn):
conn.execute(
"UPDATE sessions SET expiry_finalized = ? WHERE id = ?",
(1 if finalized else 0, session_id),
)
self._execute_write(_do)
# ── Gateway routing index (replaces sessions.json, #9006 follow-up) ────
def save_gateway_routing_entry(
self, session_key: str, entry_json: str, *, scope: str = ""
) -> None:
"""Upsert one gateway routing entry (session_key -> SessionEntry JSON).
The gateway_routing table is the durable replacement for
sessions.json: one row per routing key, holding the full serialized
``SessionEntry`` so the gateway can rehydrate exactly what it wrote.
``scope`` namespaces the index the way separate sessions.json files
did (one per sessions_dir) — callers pass their sessions_dir path so
two stores with different directories never share routing state.
"""
if not session_key or not entry_json:
return
def _do(conn):
conn.execute(
"""INSERT INTO gateway_routing (scope, session_key, entry_json, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(scope, session_key) DO UPDATE SET
entry_json = excluded.entry_json,
updated_at = excluded.updated_at""",
(scope, session_key, entry_json, time.time()),
)
self._execute_write(_do)
def replace_gateway_routing_entries(
self, entries: Dict[str, str], *, scope: str = ""
) -> None:
"""Atomically replace the routing index for *scope* with *entries*.
Mirrors the sessions.json full-rewrite semantics: keys absent from
*entries* are removed (pruned/reset sessions disappear from the
index). Runs as a single write transaction. Other scopes are
untouched.
"""
now = time.time()
def _do(conn):
conn.execute("DELETE FROM gateway_routing WHERE scope = ?", (scope,))
if entries:
conn.executemany(
"INSERT INTO gateway_routing (scope, session_key, entry_json, updated_at) "
"VALUES (?, ?, ?, ?)",
[(scope, k, v, now) for k, v in entries.items() if k and v],
)
self._execute_write(_do)
def load_gateway_routing_entries(self, *, scope: str = "") -> Dict[str, str]:
"""Load routing entries for *scope* as {session_key: entry_json}."""
with self._lock:
rows = self._conn.execute(
"SELECT session_key, entry_json FROM gateway_routing WHERE scope = ?",
(scope,),
).fetchall()
return {r["session_key"]: r["entry_json"] for r in rows}
def delete_gateway_routing_entries(
self, session_keys: List[str], *, scope: str = ""
) -> None:
"""Remove routing entries for the given session keys in *scope*."""
if not session_keys:
return
def _do(conn):
conn.executemany(
"DELETE FROM gateway_routing WHERE scope = ? AND session_key = ?",
[(scope, k) for k in session_keys],
)
self._execute_write(_do)
def list_gateway_sessions(
self,
*,
platform: Optional[str] = None,
active_only: bool = True,
) -> List[Dict[str, Any]]:
"""List gateway sessions (rows with a session_key) from state.db.
Returns the newest row per session_key — the same shape consumers got
from sessions.json: one live mapping per routing key. ``platform``
filters on ``source``; ``active_only`` restricts to sessions that
have not ended.
"""
# Full rows carry token/cost totals (MCP listings, /status) — drain
# queued async accounting deltas so consumers see exact counters.
self.flush_token_counts()
query = """
SELECT sessions.*,
COALESCE(
(SELECT MAX(m.timestamp) FROM messages m
WHERE m.session_id = sessions.id),
sessions.started_at
) AS last_active
FROM sessions
WHERE session_key IS NOT NULL
AND started_at = (
SELECT MAX(s2.started_at) FROM sessions s2
WHERE s2.session_key = sessions.session_key
)
"""
params: list = []
if platform:
query += " AND LOWER(source) = LOWER(?)"
params.append(platform)
if active_only:
query += " AND ended_at IS NULL"
query += " ORDER BY last_active DESC"
with self._lock:
rows = self._conn.execute(query, params).fetchall()
return [dict(r) for r in rows]
def find_session_by_origin(
self,
*,
platform: str,
chat_id: str,
thread_id: Optional[str] = None,
user_id: Optional[str] = None,
) -> Optional[str]:
"""Find the most recent live session_id for a platform + chat origin.
Equivalent of gateway/mirror's sessions.json scan: matches on
source + chat_id (+ thread_id when provided). When ``user_id`` is
provided, exact sender matches are preferred; if multiple distinct
users share the chat and none matches, returns None rather than
contaminating another participant's session.
"""
if not platform or chat_id in (None, ""):
return None
query = """
SELECT id, user_id, started_at FROM sessions
WHERE LOWER(source) = LOWER(?)
AND session_key IS NOT NULL
AND chat_id = ?
AND ended_at IS NULL
"""
params: list = [platform, str(chat_id)]
if thread_id is not None:
query += " AND COALESCE(thread_id, '') = ?"
params.append(str(thread_id))
query += " ORDER BY started_at DESC"
with self._lock:
rows = [dict(r) for r in self._conn.execute(query, params).fetchall()]
if not rows:
return None
if user_id:
exact = [r for r in rows if str(r.get("user_id") or "") == str(user_id)]
if exact:
return str(exact[0]["id"])
if len(rows) > 1:
return None
elif len(rows) > 1:
distinct_users = {
str(r.get("user_id") or "").strip()
for r in rows
if str(r.get("user_id") or "").strip()
}
if len(distinct_users) > 1:
return None
return str(rows[0]["id"])
def find_latest_gateway_session_for_peer(
self,
*,
source: str,
user_id: Optional[str] = None,
session_key: Optional[str] = None,
chat_id: Optional[str] = None,
chat_type: Optional[str] = None,
thread_id: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
"""Find the latest recoverable gateway session for a routing peer.
``sessions.json`` is the fast routing index, but it can be missing or
pruned after process-level restart bugs. New gateway sessions persist
the deterministic ``session_key`` on the durable session row so the
mapping can be rebuilt exactly. Rows ended only by older gateway
cleanup's ``agent_close`` bug or a mistaken TUI ``ws_orphan_reap``
(dashboard viewer disconnect before #60609) are treated as recoverable;
explicit conversation boundaries such as /new, /resume switches, and
compression splits are not.
"""
if not session_key:
return None
with self._lock:
row = self._conn.execute(
"""
SELECT * FROM sessions
WHERE session_key = ?
AND source = ?
AND (ended_at IS NULL OR end_reason IN ('agent_close', 'ws_orphan_reap'))
AND (COALESCE(message_count, 0) > 0 OR EXISTS (
SELECT 1 FROM messages WHERE messages.session_id = sessions.id LIMIT 1
))
ORDER BY started_at DESC
LIMIT 1
""",
(session_key, source),
).fetchone()
if row is not None:
return dict(row)
# Conservative fallback for rows created by current code but with a
# temporarily-missing exact key: still require the complete peer
# tuple so we never cross chats/threads/users.
if chat_id is None or chat_type is None:
return None
row = self._conn.execute(
"""
SELECT * FROM sessions
WHERE source = ?
AND COALESCE(user_id, '') = COALESCE(?, '')
AND COALESCE(chat_id, '') = COALESCE(?, '')
AND COALESCE(chat_type, '') = COALESCE(?, '')
AND COALESCE(thread_id, '') = COALESCE(?, '')
AND (ended_at IS NULL OR end_reason IN ('agent_close', 'ws_orphan_reap'))
AND (COALESCE(message_count, 0) > 0 OR EXISTS (
SELECT 1 FROM messages WHERE messages.session_id = sessions.id LIMIT 1
))
ORDER BY started_at DESC
LIMIT 1
""",
(source, user_id, chat_id, chat_type, thread_id),
).fetchone()
return dict(row) if row else None
def find_live_compression_child(
self, parent_session_id: str
) -> Optional[Dict[str, Any]]:
"""Return the unique live direct child of a compression-ended session.
A stale agent may observe that another compression path already rotated
its parent. Recovery is safe only when the durable lineage identifies
exactly one live direct continuation. Multiple children are treated as
ambiguous and fail closed rather than guessing which transcript owns
subsequent messages.
"""
if not parent_session_id:
return None
with self._lock:
parent = self._conn.execute(
"SELECT ended_at, end_reason FROM sessions WHERE id = ?",
(parent_session_id,),
).fetchone()
if (
parent is None
or parent["ended_at"] is None
or parent["end_reason"] != "compression"
):
return None
rows = self._conn.execute(
"""
SELECT * FROM sessions
WHERE parent_session_id = ?
AND ended_at IS NULL
AND json_extract(COALESCE(model_config, '{}'), '$._branched_from') IS NULL
AND json_extract(COALESCE(model_config, '{}'), '$._delegate_from') IS NULL
AND COALESCE(source, '') != 'tool'
ORDER BY started_at ASC
LIMIT 2
""",
(parent_session_id,),
).fetchall()
return dict(rows[0]) if len(rows) == 1 else None
def publish_compression_child(
self,
*,
parent_session_id: str,
child_session_id: str,
source: str,
messages: List[Dict[str, Any]],
model: str = None,
model_config: Dict[str, Any] = None,
system_prompt: str = None,
cwd: str = None,
profile_name: str = None,
compression_lock_holder: str = None,
require_compression_lease: bool = True,
) -> None:
"""Atomically close a parent and publish its durable compression child.
The parent closure, child row, and compacted handoff become visible in
one transaction. Readers can therefore observe either the live parent or
a complete child, never an ended parent with a missing/empty child.
"""
def _do(conn):
lock_row = conn.execute(
"SELECT holder, expires_at FROM compression_locks WHERE session_id = ?",
(parent_session_id,),
).fetchone()
if require_compression_lease and (
lock_row is None
or not compression_lock_holder
or lock_row["holder"] != compression_lock_holder
or float(lock_row["expires_at"]) <= time.time()
):
raise CompressionSessionBusyError(
f"Compression lease lost before publication: {parent_session_id}"
)
parent = conn.execute(
"""SELECT ended_at, cwd, git_branch, git_repo_root,
user_id, session_key, chat_id, chat_type,
thread_id, display_name, origin_json, profile_name
FROM sessions WHERE id = ?""",
(parent_session_id,),
).fetchone()
if parent is None:
raise RuntimeError(f"Compression parent not found: {parent_session_id}")
if parent["ended_at"] is not None:
raise RuntimeError(f"Compression parent already ended: {parent_session_id}")
if not messages:
raise RuntimeError("Compression child handoff must not be empty")
conn.execute(
"""INSERT INTO sessions (
id, source, model, model_config, system_prompt,
parent_session_id, cwd, git_branch, git_repo_root,
profile_name, user_id, session_key, chat_id, chat_type,
thread_id, display_name, origin_json, started_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(
child_session_id,
source,
model,
json.dumps(model_config) if model_config else None,
system_prompt,
parent_session_id,
cwd or parent["cwd"],
parent["git_branch"],
parent["git_repo_root"],
# Same inheritance contract as _insert_session_row's
# compression-fork backfill (#59527 / cross-profile jump
# fix): the child stays on the parent's profile and keeps
# the gateway routing/origin columns so peer recovery
# still works after a crash at the boundary.
profile_name or parent["profile_name"],
parent["user_id"],
parent["session_key"],
parent["chat_id"],
parent["chat_type"],
parent["thread_id"],
parent["display_name"],
parent["origin_json"],
time.time(),
),
)
total_messages, total_tool_calls = self._insert_message_rows(
conn, child_session_id, messages
)
conn.execute(
"UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?",
(total_messages, total_tool_calls, child_session_id),
)
updated = conn.execute(
"UPDATE sessions SET ended_at = ?, end_reason = 'compression' "
"WHERE id = ? AND ended_at IS NULL",
(time.time(), parent_session_id),
)
if updated.rowcount != 1:
raise RuntimeError(
f"Compression parent changed during publication: {parent_session_id}"
)
self._execute_write(_do)
def end_session(self, session_id: str, end_reason: str) -> None:
"""Mark a session as ended.
No-ops when the session is already ended. The first end_reason wins:
compression-split sessions must keep their ``end_reason = 'compression'``
record even if a later stale ``end_session()`` call (e.g. from a
desynced CLI session_id after ``/resume`` or ``/branch``) targets them
with a different reason. Use ``reopen_session()`` first if you
intentionally need to re-end a closed session with a new reason.
"""
def _do(conn):
conn.execute(
"UPDATE sessions SET ended_at = ?, end_reason = ? "
"WHERE id = ? AND ended_at IS NULL",
(time.time(), end_reason, session_id),
)
self._execute_write(_do)
def reopen_session(self, session_id: str) -> None:
"""Clear ended_at/end_reason so a session can be resumed."""
def _do(conn):
conn.execute(
"UPDATE sessions SET ended_at = NULL, end_reason = NULL WHERE id = ?",
(session_id,),
)
self._execute_write(_do)
def promote_to_session_reset(
self, session_id: str, reason: str = "session_reset"
) -> bool:
"""Durably mark a session as ended by an intentional reset boundary.
Promotes *only* live rows (``ended_at IS NULL``) or rows carrying an
accidental end_reason that the recovery query
(``find_latest_gateway_session_for_peer``) treats as recoverable:
``agent_close`` (older gateway cleanup bug) and ``ws_orphan_reap``
(mistaken TUI reaper). Explicit conversation boundaries such as
``compression``, ``session_reset``, ``session_switch``, etc. are
preserved — the first writer wins for those, and a later expiry
finalization must not silently overwrite them.
Plain ``end_session()`` is NOT sufficient for reset boundaries: it
no-ops on an already-ended row, so a row that agent cleanup already
closed as ``agent_close`` would stay recoverable and stale-route
recovery would resurrect the reset session with its full history
(#61220, #61993, #63539).
Keep this promotion set in sync with the recoverable set in
``find_latest_gateway_session_for_peer`` — any reason recovery would
reopen must be promotable here.
``reason`` lets reset paths keep their auditable specific reasons
(``idle``, ``daily``, ``suspended``, ``resume_pending_expired``).
Returns ``True`` when the row was promoted, ``False`` when skipped
(already has a different explicit end_reason, or row not found).
"""
if not session_id:
return False
now = time.time()
def _do(conn):
cursor = conn.execute(
"UPDATE sessions SET ended_at = ?, end_reason = ? "
"WHERE id = ? AND (ended_at IS NULL "
"OR end_reason IN ('agent_close', 'ws_orphan_reap'))",
(now, reason, session_id),
)
return cursor.rowcount
try:
rows = self._execute_write(_do)
return bool(rows)
except Exception:
return False
def update_session_cwd(
self, session_id: str, cwd: str, git_branch: str = None, git_repo_root: str = None
) -> None:
"""Persist the session working directory when a frontend knows it.
``git_branch`` records the git branch checked out in ``cwd`` at the time
the session started/resumed. The sidebar groups main-checkout sessions
by this so feature-branch work doesn't pile under a single "main" row
(the main checkout's *current* branch is transient and would
misattribute past sessions).
``git_repo_root`` records the git repo this cwd belongs to — the
authoritative project key. Resolving it here, at the lowest level, means
every surface reads the same membership instead of re-probing git in the
GUI over a partial page. Each field is only written when non-empty so a
probe failure never clobbers a previously-captured value.
"""
if not session_id or not cwd:
return
branch = (git_branch or "").strip()
repo_root = (git_repo_root or "").strip()
sets = ["cwd = ?"]
params: List[Any] = [cwd]
if branch:
sets.append("git_branch = ?")
params.append(branch)
if repo_root:
sets.append("git_repo_root = ?")
params.append(repo_root)
params.append(session_id)
def _do(conn):
conn.execute(f"UPDATE sessions SET {', '.join(sets)} WHERE id = ?", params)
self._execute_write(_do)
def backfill_repo_roots(self, cwd_to_root: Dict[str, str]) -> None:
"""Persist resolved git repo roots for cwds that don't have one yet.
Backfills history so projects light up for sessions created before the
column existed, without clobbering an already-recorded root. Only
non-empty roots are written (a non-git cwd stays NULL).
"""
pairs = [(root, cwd) for cwd, root in cwd_to_root.items() if root and cwd]
if not pairs:
return
def _do(conn):
for root, cwd in pairs:
conn.execute(
"UPDATE sessions SET git_repo_root = ? "
"WHERE cwd = ? AND COALESCE(git_repo_root, '') = ''",
(root, cwd),
)
self._execute_write(_do)
def record_compression_failure_cooldown(
self,
session_id: str,
cooldown_until: float,
error: Optional[str] = None,
) -> None:
"""Persist the active compression-failure cooldown for a session."""
if not session_id:
return
def _do(conn):
conn.execute(
"UPDATE sessions SET compression_failure_cooldown_until = ?, "
"compression_failure_error = ? WHERE id = ?",
(cooldown_until, error, session_id),
)
try:
self._execute_write(_do)
except sqlite3.Error as exc:
logger.warning(
"record_compression_failure_cooldown(%s) failed: %s",
session_id, exc,
)
def get_compression_failure_cooldown(
self,
session_id: str,
) -> Optional[Dict[str, Any]]:
"""Return the active compression-failure cooldown for ``session_id``."""
if not session_id:
return None
now = time.time()
with self._lock:
row = self._conn.execute(
"SELECT compression_failure_cooldown_until, compression_failure_error "
"FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if row is None:
return None
cooldown_until = (
row["compression_failure_cooldown_until"]
if isinstance(row, sqlite3.Row)
else row[0]
)
if cooldown_until is None:
return None
cooldown_until = float(cooldown_until)
if cooldown_until <= now:
return None
error = (
row["compression_failure_error"]
if isinstance(row, sqlite3.Row)
else row[1]
)
return {
"cooldown_until": cooldown_until,
"remaining_seconds": cooldown_until - now,
"error": error,
}
def clear_compression_failure_cooldown(self, session_id: str) -> None:
"""Clear any persisted compression-failure cooldown for a session."""
if not session_id:
return
def _do(conn):
conn.execute(
"UPDATE sessions SET compression_failure_cooldown_until = NULL, "
"compression_failure_error = NULL WHERE id = ?",
(session_id,),
)
try:
self._execute_write(_do)
except sqlite3.Error as exc:
logger.warning(
"clear_compression_failure_cooldown(%s) failed: %s",
session_id, exc,
)
def get_compression_fallback_streak(self, session_id: str) -> int:
"""Return the persisted deterministic-fallback streak."""
if not session_id:
return 0
with self._lock:
conn = self._conn
if conn is None:
return 0
row = conn.execute(
"SELECT compression_fallback_streak FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if row is None:
return 0
value = (
row["compression_fallback_streak"]
if isinstance(row, sqlite3.Row)
else row[0]
)
try:
return max(0, int(value or 0))
except (TypeError, ValueError):
return 0
def set_compression_fallback_streak(self, session_id: str, streak: int) -> None:
"""Persist the deterministic-fallback streak for one session."""
if not session_id:
return
normalized = max(0, int(streak))
def _do(conn):
conn.execute(
"UPDATE sessions SET compression_fallback_streak = ? WHERE id = ?",
(normalized, session_id),
)
self._execute_write(_do)
def get_compression_ineffective_count(self, session_id: str) -> int:
"""Return the persisted ineffective-compaction strike count.
Mirrors ``get_compression_fallback_streak``: this is the durable half
of the anti-thrash guard (``_ineffective_compression_count`` on the
built-in compressor), persisted so that a fresh compressor bound to a
resumed session inherits an armed/tripped guard instead of starting
from zero across process restarts (#54923).
"""
if not session_id:
return 0
with self._lock:
conn = self._conn
if conn is None:
return 0
row = conn.execute(
"SELECT compression_ineffective_count FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if row is None:
return 0
value = (
row["compression_ineffective_count"]
if isinstance(row, sqlite3.Row)
else row[0]
)
try:
return max(0, int(value or 0))
except (TypeError, ValueError):
return 0
def set_compression_ineffective_count(self, session_id: str, count: int) -> None:
"""Persist the ineffective-compaction strike count for one session."""
if not session_id:
return
normalized = max(0, int(count))
def _do(conn):
conn.execute(
"UPDATE sessions SET compression_ineffective_count = ? WHERE id = ?",
(normalized, session_id),
)
self._execute_write(_do)
# ──────────────────────────────────────────────────────────────────────
# Compression locks
# ──────────────────────────────────────────────────────────────────────
# Atomic per-session locks that prevent two compression paths from
# racing on the same session_id and producing orphan child sessions.
#
# The race: ``conversation_compression.py`` rotates ``agent.session_id``
# as a side effect of a successful compression (end old session, create
# new). That mutation is local to the AIAgent instance — but ``state.db``
# is shared across all instances. Two AIAgents that share the same
# ``session_id`` at the moment they both decide to compress (most
# commonly the parent turn's agent + a background-review fork started
# right after the turn ended) each end the parent and create their own
# NEW session, parented to the same old id. The gateway SessionEntry
# only catches one rotation; the other child silently accumulates
# writes — Damien's "parent → two orphan children" repro shape.
#
# The lock is keyed by ``session_id`` and is held for the duration of
# the compress() call plus the rotation. ``holder`` identifies the
# current owner (pid:tid:nonce) for diagnostics; the lock is recovered
# via ``expires_at`` if the holder process crashed without releasing.
def refresh_compression_lock(
self,
session_id: str,
holder: str,
ttl_seconds: float = 300.0,
) -> bool:
"""Extend the compression lock lease if ``holder`` still owns it.
Ownership is decided by the ``holder`` column alone, deliberately NOT
by ``expires_at``: a live owner whose refresher thread was starved
(GC pause, loaded CI runner, a slow write escaping ``_execute_write``'s
retry budget) past its own TTL must be able to revive its still-unclaimed
row on the next tick. Requiring ``expires_at >= now`` here made such a
stall permanent — every later refresh matched 0 rows, so the owner kept
compressing and rotating with no lease at all, which is exactly the
unprotected window a competing path can fork the session lineage in.
This does not resurrect a lock somebody else already took: SQLite
serialises writes, so a reclaim (DELETE-expired + INSERT-or-IGNORE in
:meth:`try_acquire_compression_lock`) and this UPDATE never interleave.
Reclaim-first replaces ``holder``, so this UPDATE matches nothing and
returns False; refresh-first pushes ``expires_at`` into the future, so
the reclaimer's DELETE-expired matches nothing and its acquire fails.
"""
if not session_id or not holder:
return False
now = time.time()
expires_at = now + ttl_seconds
def _do(conn):
cur = conn.execute(
"UPDATE compression_locks SET expires_at = ? "
"WHERE session_id = ? AND holder = ?",
(expires_at, session_id, holder),
)
return cur.rowcount > 0
try:
return bool(self._execute_write(_do))
except sqlite3.Error as exc:
logger.warning(
"refresh_compression_lock(%s) failed: %s",
session_id, exc,
)
return False
def try_acquire_compression_lock(
self,
session_id: str,
holder: str,
ttl_seconds: float = 300.0,
) -> bool:
"""Try to atomically acquire the compression lock for ``session_id``.
Returns ``True`` on success (caller now owns the lock and must
release via :meth:`release_compression_lock`). Returns ``False``
if another holder already owns a non-expired lock — the caller
MUST NOT proceed with compression in that case (its rotation would
race against the holder's, splitting the session lineage).
Expired locks (``expires_at < now``) are reclaimed transparently.
Structured holders whose local ``pid=`` no longer exists are reclaimed
immediately, so a gateway killed during compression does not stall the
replacement process for the full lease TTL.
Implementation: single-transaction DELETE-expired + INSERT-or-IGNORE,
followed by a SELECT to confirm we got the row. SQLite serialises
writes, so the whole sequence is atomic against other writers.
"""
if not session_id:
return False
now = time.time()
expires_at = now + ttl_seconds
def _do(conn):
reclaimed_holder = None
row = conn.execute(
"SELECT holder, expires_at FROM compression_locks "
"WHERE session_id = ?",
(session_id,),
).fetchone()
if row is not None:
current_holder = (
row["holder"] if isinstance(row, sqlite3.Row) else row[0]
)
current_expires_at = (
row["expires_at"] if isinstance(row, sqlite3.Row) else row[1]
)
if (
current_expires_at < now
or _compression_lock_holder_process_is_dead(current_holder)
):
conn.execute(
"DELETE FROM compression_locks "
"WHERE session_id = ? AND holder = ?",
(session_id, current_holder),
)
reclaimed_holder = current_holder
# Then: try to insert. INSERT OR IGNORE returns no rowcount
# difference — verify ownership via SELECT.
conn.execute(
"INSERT OR IGNORE INTO compression_locks "
"(session_id, holder, acquired_at, expires_at) "
"VALUES (?, ?, ?, ?)",
(session_id, holder, now, expires_at),
)
row = conn.execute(
"SELECT holder FROM compression_locks WHERE session_id = ?",
(session_id,),
).fetchone()
acquired = row is not None and (
row["holder"] if isinstance(row, sqlite3.Row) else row[0]
) == holder
return acquired, reclaimed_holder
try:
acquired, reclaimed_holder = self._execute_write(_do)
if reclaimed_holder:
logger.warning(
"Reclaimed stale compression lock for session=%s "
"(holder=%s)",
session_id,
reclaimed_holder,
)
return bool(acquired)
except sqlite3.Error as exc:
logger.warning(
"try_acquire_compression_lock(%s) failed: %s",
session_id, exc,
)
# Fail open: returning False makes the caller skip compression,
# which is the safe behaviour when the lock subsystem is broken.
return False
def release_compression_lock(self, session_id: str, holder: str) -> None:
"""Release the compression lock for ``session_id`` iff we own it.
Idempotent: no-op when the lock has already expired and been
reclaimed by a different holder, or when no lock exists. The
``holder`` check prevents a late-returning compressor from
clobbering a fresh lock held by someone else.
"""
if not session_id:
return
def _do(conn):
conn.execute(
"DELETE FROM compression_locks "
"WHERE session_id = ? AND holder = ?",
(session_id, holder),
)
try:
self._execute_write(_do)
except sqlite3.Error as exc:
logger.warning(
"release_compression_lock(%s) failed: %s",
session_id, exc,
)
def get_compression_lock_holder(self, session_id: str) -> Optional[str]:
"""Return the current (non-expired) holder for ``session_id``, or None.
Diagnostic helper — not used by the locking protocol itself.
"""
if not session_id:
return None
now = time.time()
row = self._conn.execute(
"SELECT holder FROM compression_locks "
"WHERE session_id = ? AND expires_at >= ?",
(session_id, now),
).fetchone()
if row is None:
return None
return row["holder"] if isinstance(row, sqlite3.Row) else row[0]
def update_session_meta(
self,
session_id: str,
model_config_json: str,
model: Optional[str] = None,
) -> None:
"""Update model_config and optionally model for an existing session.
Uses COALESCE so that passing model=None leaves the stored model
column unchanged. Routes through _execute_write for the standard
BEGIN IMMEDIATE + jitter-retry + lock guarantee.
"""
# Barrier against queued token deltas — see update_session_model.
self.flush_token_counts()
def _do(conn):
conn.execute(
"UPDATE sessions SET model_config = ?, model = COALESCE(?, model) WHERE id = ?",
(model_config_json, model, session_id),
)
self._execute_write(_do)
def update_system_prompt(self, session_id: str, system_prompt: str) -> None:
"""Store the full assembled system prompt snapshot."""
def _do(conn):
conn.execute(
"UPDATE sessions SET system_prompt = ? WHERE id = ?",
(system_prompt, session_id),
)
self._execute_write(_do)
def update_session_model(self, session_id: str, model: str) -> None:
"""Update the model for a session after a mid-session switch.
Unlike ``update_token_counts`` which uses ``COALESCE(model, ?)``
(only filling in NULL), this unconditionally sets the model column
so that the dashboard reflects the user's latest /model choice.
Also nulls ``system_prompt`` so stale ``Model:`` / ``Provider:``
footer metadata is rebuilt on the next turn. A successful /model
switch explicitly replaces any confirmed Browser runtime lock while
preserving unrelated lineage markers in ``model_config``.
"""
# This write bypasses the token queue, so deltas enqueued before the
# switch must land first: a still-queued first delta carries the
# pre-switch route, and applying it after this UPDATE would trip the
# first_accounted_route overwrite in update_token_counts (row sees
# api_call_count == 0 + a route mismatch) and resurrect the old
# model/provider. Flushing here restores the pre-queue ordering.
self.flush_token_counts()
def _do(conn):
conn.execute(
"""UPDATE sessions SET
model = ?,
model_config = CASE
WHEN model_config IS NULL THEN NULL
WHEN json_valid(model_config)
THEN json_remove(model_config, '$.browser_model_lock')
ELSE model_config
END,
system_prompt = NULL
WHERE id = ?""",
(model, session_id),
)
self._execute_write(_do)
def update_session_runtime_lock(
self,
session_id: str,
*,
model: Optional[str] = None,
provider: Optional[str] = None,
model_options: Optional[Dict[str, Any]] = None,
route_source: Optional[str] = None,
confirmed: bool = False,
) -> None:
"""Persist a Browser / API client runtime lock without clobbering lineage markers.
Merges ``browser_model_lock`` into the existing ``model_config`` JSON so
``_branched_from`` / ``_delegate_from`` survive. Nulls ``system_prompt``
so cached ``Model:`` / ``Provider:`` footers cannot lie after a switch.
"""
lock = {
"provider": provider or "",
"model": model or "",
"model_options": model_options or {},
"route_source": route_source or "",
"confirmed": bool(confirmed),
"updated_at": time.time(),
}
def _do(conn):
row = conn.execute(
"SELECT model_config FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if row is None:
return
raw = row["model_config"] if isinstance(row, sqlite3.Row) else row[0]
config: Dict[str, Any] = {}
if isinstance(raw, str) and raw.strip():
try:
parsed = json.loads(raw)
if isinstance(parsed, dict):
config = parsed
except Exception:
config = {}
elif isinstance(raw, dict):
config = dict(raw)
config["browser_model_lock"] = lock
conn.execute(
"""UPDATE sessions SET
model_config = ?,
model = COALESCE(?, model),
system_prompt = NULL
WHERE id = ?""",
(json.dumps(config), model, session_id),
)
self._execute_write(_do)
def update_session_billing_route(
self,
session_id: str,
*,
provider: str,
base_url: str,
billing_mode: Optional[str] = None,
) -> None:
"""Unconditionally update the billing provider/base_url for a session.
Unlike ``update_token_counts`` which uses ``COALESCE(billing_provider, ?)``
(only filling in NULL), this unconditionally sets the billing fields so
that the dashboard reflects the user's latest /model switch.
Also nulls ``system_prompt`` so the cached snapshot (which embeds a
stale ``Model:`` / ``Provider:`` header) is rebuilt — matching the
behavior of ``update_session_model`` (see #48173, #48248).
"""
# Barrier against queued token deltas — see update_session_model.
self.flush_token_counts()
def _do(conn):
conn.execute(
"""UPDATE sessions SET
billing_provider = ?,
billing_base_url = ?,
billing_mode = COALESCE(?, billing_mode),
system_prompt = NULL
WHERE id = ?""",
(provider, base_url, billing_mode, session_id),
)
self._execute_write(_do)
# ── Async token accounting ──
# update_token_counts() runs a sessions UPDATE (plus a per-model usage
# upsert) inside BEGIN IMMEDIATE; against a cold multi-GB state.db one
# call can stall the turn thread for tens to hundreds of ms, and the
# tool loop pays it after EVERY API call (measured p50 3.3ms / p95 70ms
# per call in production). queue_token_counts() reduces the critical
# path to a deque append: a dedicated single-writer thread applies
# deltas in enqueue order, coalescing consecutive same-route deltas
# into one UPDATE when a backlog forms. Readers that need exact
# mid-turn totals (get_session and friends) call flush_token_counts()
# first — a plain attribute check when nothing is queued.
# Delta fields summed when coalescing. Route fields must be equal for
# two deltas to merge: model/billing_* feed COALESCE backfill and the
# per-model usage attribution key, and cost_status/cost_source are
# last-non-None-wins — equality makes the merged UPDATE byte-for-byte
# equivalent to applying the deltas sequentially.
_TOKEN_DELTA_SUM_FIELDS = (
"input_tokens", "output_tokens", "cache_read_tokens",
"cache_write_tokens", "reasoning_tokens", "api_call_count",
)
_TOKEN_DELTA_COST_FIELDS = ("estimated_cost_usd", "actual_cost_usd")
_TOKEN_DELTA_ROUTE_FIELDS = (
"model", "cost_status", "cost_source", "pricing_version",
"billing_provider", "billing_base_url", "billing_mode",
)
def queue_token_counts(self, session_id: str, **kwargs) -> None:
"""Enqueue a token/cost delta for the background writer.
Accepts the same keyword arguments as :meth:`update_token_counts`
and applies them asynchronously with identical semantics. Cheap
(append + notify) — safe to call on the turn thread after every
API call. After close() has stopped the writer, falls back to the
synchronous path and may raise like :meth:`update_token_counts`.
"""
with self._token_queue_cond:
thread = self._token_writer_thread
writer_stopped = self._token_writer_stop and (
thread is None or not thread.is_alive()
)
if not writer_stopped:
self._token_queue.append((session_id, kwargs))
if thread is None or not thread.is_alive():
# Daemon so process exit never hangs on accounting; the
# atexit hook drains anything still queued at interpreter
# shutdown (registered once per instance, on first use).
# ``not is_alive()`` (rather than ``is None`` only)
# respawns the writer if it ever died from an unexpected
# escape — otherwise a dead thread object would block
# respawn forever and deltas would pile up on the deque
# until a reader's flush drained them synchronously.
thread = threading.Thread(
target=self._token_writer_loop,
name="session-db-token-writer",
daemon=True,
)
self._token_writer_thread = thread
thread.start()
atexit.register(self._drain_token_queue_at_exit)
self._token_queue_cond.notify_all()
if writer_stopped:
# Writer permanently stopped (close() ran; a stop-flagged but
# still-live writer keeps accepting — its loop drains before
# exiting). Enqueueing now would drop the delta silently: no
# writer will run and close() already unregistered the atexit
# hook. Apply inline instead so a closed-connection failure
# raises at the call site, exactly like the old synchronous
# update_token_counts path these call sites still guard for.
self.update_token_counts(session_id, **kwargs)
def flush_token_counts(self, timeout: float = 5.0) -> bool:
"""Block until every queued token delta has been applied.
Returns True when the queue is fully drained, False on timeout
(callers then read totals that are stale by the still-queued
deltas — no worse than reading before the flush existed).
Never raises: apply failures are logged by the writer.
"""
# Fast path — nothing queued, nothing in flight.
if not self._token_queue and not self._token_writer_busy:
return True
batch = None
with self._token_queue_cond:
deadline = time.monotonic() + timeout
while self._token_queue or self._token_writer_busy:
# A live writer is authoritative even when stop-flagged
# (close() in progress): its loop drains the queue before
# exiting, and draining here instead would race its
# in-flight batch — newer deltas committing before older
# ones breaks the last-non-None-wins / first-accounted-
# route / COALESCE-backfill fields. Only when the writer is
# dead (or never started for these deltas) does the caller
# take the leftovers. Re-checked each wakeup: the writer
# can exit mid-wait with deltas enqueued after its final
# empty-queue check. busy is claimed while draining (same
# protocol as the writer) so a concurrent flush cannot
# report drained — or pop a newer delta — while this batch
# is still unapplied; a claimed busy therefore also means
# "wait", never "drain alongside".
thread = self._token_writer_thread
if (
(thread is None or not thread.is_alive())
and not self._token_writer_busy
):
self._token_writer_busy = True
batch = list(self._token_queue)
self._token_queue.clear()
break
remaining = deadline - time.monotonic()
if remaining <= 0:
return False
self._token_queue_cond.wait(remaining)
if batch:
try:
self._apply_token_batch(batch)
finally:
with self._token_queue_cond:
self._token_writer_busy = False
self._token_queue_cond.notify_all()
return True
def _token_writer_loop(self) -> None:
while True:
with self._token_queue_cond:
while not self._token_queue and not self._token_writer_stop:
self._token_queue_cond.wait()
if not self._token_queue:
return # stop requested and fully drained
# busy is set BEFORE the queue is cleared: the lock-free
# fast path in flush_token_counts() reads queue-then-busy,
# so this order guarantees it can never observe an empty
# queue while the popped batch is still unapplied.
self._token_writer_busy = True
batch = list(self._token_queue)
self._token_queue.clear()
try:
self._apply_token_batch(batch)
finally:
with self._token_queue_cond:
self._token_writer_busy = False
self._token_queue_cond.notify_all()
def _apply_token_batch(self, batch: List[Tuple[str, Dict[str, Any]]]) -> None:
"""Apply queued deltas in order, coalescing where safe. Never raises."""
try:
coalesced = self._coalesce_token_deltas(batch)
except Exception as exc:
# Coalescing must never kill the writer thread (a dead writer
# can't be observed by callers). Fall back to applying the raw
# batch delta-by-delta — the merge is an optimization only.
logger.warning(
"async token accounting: coalesce failed, applying raw "
"batch: %s", exc,
)
coalesced = batch
for session_id, kwargs in coalesced:
try:
self.update_token_counts(session_id, **kwargs)
except Exception as exc:
# Same contract as the old inline call sites: accounting
# loss is logged, never raised into a turn.
logger.warning(
"async token accounting: apply failed (session=%s): %s",
session_id, exc,
)
def _coalesce_token_deltas(
self, batch: List[Tuple[str, Dict[str, Any]]]
) -> List[Tuple[str, Dict[str, Any]]]:
"""Merge consecutive incremental deltas with an identical route.
Only adjacent deltas merge, so ordering across sessions and across
a mid-session /model switch is preserved exactly. absolute=True
deltas (cumulative overwrites) never merge.
"""
groups: List[Tuple[Optional[tuple], str, Dict[str, Any]]] = []
for session_id, kwargs in batch:
key = None
if not kwargs.get("absolute"):
key = (session_id,) + tuple(
kwargs.get(f) for f in self._TOKEN_DELTA_ROUTE_FIELDS
)
if groups and key is not None and groups[-1][0] == key:
merged = groups[-1][2]
for f in self._TOKEN_DELTA_SUM_FIELDS:
merged[f] = merged.get(f, 0) + kwargs.get(f, 0)
for f in self._TOKEN_DELTA_COST_FIELDS:
value = kwargs.get(f)
if value is not None:
# None-preserving sum: an all-None run must stay
# None so COALESCE keeps the stored value untouched.
merged[f] = (merged.get(f) or 0.0) + value
else:
groups.append((key, session_id, dict(kwargs)))
return [(sid, kw) for _, sid, kw in groups]
def _stop_token_writer(self, join_timeout: float = 10.0) -> None:
"""Stop the writer thread and drain remaining deltas. Never raises."""
with self._token_queue_cond:
self._token_writer_stop = True
self._token_queue_cond.notify_all()
thread = self._token_writer_thread
if thread is not None and thread.is_alive():
thread.join(timeout=join_timeout)
if thread.is_alive():
# Writer stuck mid-apply (pathological lock contention).
# Leave any queued deltas unapplied rather than racing the
# stuck apply and misordering/double-counting.
logger.warning(
"async token accounting: writer did not stop within %.0fs; "
"%d queued delta(s) not persisted",
join_timeout, len(self._token_queue),
)
return
# Writer exited (or never started) — apply leftovers synchronously.
# Claim busy like the writer/flush drains do, so a concurrent
# flush_token_counts cannot fast-path True while this batch is
# still being applied; conversely, wait out a flush caller-drain
# that already claimed busy — close() nulls the connection right
# after this returns, and must not yank it mid-batch.
with self._token_queue_cond:
deadline = time.monotonic() + join_timeout
while self._token_writer_busy:
remaining = deadline - time.monotonic()
if remaining <= 0:
logger.warning(
"async token accounting: concurrent drain did not "
"finish within %.0fs; %d queued delta(s) not persisted",
join_timeout, len(self._token_queue),
)
return
self._token_queue_cond.wait(remaining)
# busy is claimed BEFORE the queue is cleared — same ordering
# as the writer loop and the flush caller-drain. The lock-free
# fast path in flush_token_counts() reads queue-then-busy
# without the cond, so clearing first would let a concurrent
# flush observe "empty and idle" and return True while this
# popped batch is still unapplied.
batch = list(self._token_queue)
if batch:
self._token_writer_busy = True
self._token_queue.clear()
if batch:
try:
self._apply_token_batch(batch)
finally:
with self._token_queue_cond:
self._token_writer_busy = False
self._token_queue_cond.notify_all()
def _drain_token_queue_at_exit(self) -> None:
try:
self._stop_token_writer()
except Exception:
pass # Best effort — never fatal at interpreter shutdown.
def update_token_counts(
self,
session_id: str,
input_tokens: int = 0,
output_tokens: int = 0,
model: str = None,
cache_read_tokens: int = 0,
cache_write_tokens: int = 0,
reasoning_tokens: int = 0,
estimated_cost_usd: Optional[float] = None,
actual_cost_usd: Optional[float] = None,
cost_status: Optional[str] = None,
cost_source: Optional[str] = None,
pricing_version: Optional[str] = None,
billing_provider: Optional[str] = None,
billing_base_url: Optional[str] = None,
billing_mode: Optional[str] = None,
api_call_count: int = 0,
absolute: bool = False,
) -> None:
"""Update token counters and backfill model if not already set.
When *absolute* is False (default), values are **incremented** — use
this for per-API-call deltas (CLI path).
When *absolute* is True, values are **set directly** — use this when
the caller already holds cumulative totals (gateway path, where the
cached agent accumulates across messages).
"""
# Ensure the session row exists so the UPDATE doesn't silently affect
# 0 rows. Under concurrent load (cron + kanban + delegate_task) the
# initial create_session() may have failed due to SQLite locking.
# INSERT OR IGNORE is cheap and idempotent.
self._insert_session_row(session_id, "unknown", model=model)
if absolute:
sql = """UPDATE sessions SET
input_tokens = ?,
output_tokens = ?,
cache_read_tokens = ?,
cache_write_tokens = ?,
reasoning_tokens = ?,
estimated_cost_usd = COALESCE(?, 0),
actual_cost_usd = CASE
WHEN ? IS NULL THEN actual_cost_usd
ELSE ?
END,
cost_status = COALESCE(?, cost_status),
cost_source = COALESCE(?, cost_source),
pricing_version = COALESCE(?, pricing_version),
billing_provider = COALESCE(billing_provider, ?),
billing_base_url = COALESCE(billing_base_url, ?),
billing_mode = COALESCE(billing_mode, ?),
model = COALESCE(model, ?),
api_call_count = ?
WHERE id = ?"""
else:
sql = """UPDATE sessions SET
input_tokens = input_tokens + ?,
output_tokens = output_tokens + ?,
cache_read_tokens = cache_read_tokens + ?,
cache_write_tokens = cache_write_tokens + ?,
reasoning_tokens = reasoning_tokens + ?,
estimated_cost_usd = COALESCE(estimated_cost_usd, 0) + COALESCE(?, 0),
actual_cost_usd = CASE
WHEN ? IS NULL THEN actual_cost_usd
ELSE COALESCE(actual_cost_usd, 0) + ?
END,
cost_status = COALESCE(?, cost_status),
cost_source = COALESCE(?, cost_source),
pricing_version = COALESCE(?, pricing_version),
billing_provider = COALESCE(billing_provider, ?),
billing_base_url = COALESCE(billing_base_url, ?),
billing_mode = COALESCE(billing_mode, ?),
model = COALESCE(model, ?),
api_call_count = COALESCE(api_call_count, 0) + ?
WHERE id = ?"""
has_accounted_usage = bool(
input_tokens or output_tokens or cache_read_tokens
or cache_write_tokens or reasoning_tokens or api_call_count
or estimated_cost_usd or actual_cost_usd
)
params = (
input_tokens,
output_tokens,
cache_read_tokens,
cache_write_tokens,
reasoning_tokens,
estimated_cost_usd,
actual_cost_usd,
actual_cost_usd,
cost_status,
cost_source,
pricing_version,
billing_provider if has_accounted_usage else None,
billing_base_url if has_accounted_usage else None,
billing_mode if has_accounted_usage else None,
model if has_accounted_usage else None,
api_call_count,
session_id,
)
# Per-model usage attribution. ``update_token_counts`` is the single
# chokepoint every per-API-call delta flows through (CLI, gateway, cron,
# delegated runs — see conversation_loop / codex_runtime), and each call
# carries the model/provider *active at the time of that call*. The
# ``sessions`` row only keeps one (model, billing_provider) pair, so a
# mid-session ``/model`` switch otherwise attributes every token to the
# initial model (issue #51607). Recording the per-call delta into
# session_model_usage keyed by the live model preserves an accurate
# per-model breakdown regardless of how many times the user switches.
#
# Only the incremental path records here. Absolute cumulative updates
# cannot be split back into routes; Insights reconciles any positive
# residual against the aggregate session row instead.
record_model_usage = (not absolute) and (
input_tokens or output_tokens or cache_read_tokens
or cache_write_tokens or reasoning_tokens or api_call_count
or estimated_cost_usd
)
def _do(conn):
row = conn.execute(
"SELECT model, billing_provider, api_call_count FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
existing_model = row["model"] if row is not None else None
existing_provider = row["billing_provider"] if row is not None else None
existing_api_calls = int((row["api_call_count"] if row is not None else 0) or 0)
# Session creation records the requested primary route before any API
# call. If it fails and fallback succeeds, the first accounted usage
# event is the first authoritative route. After that, preserve the
# legacy row: one row cannot represent mixed-provider usage.
first_accounted_route = (
existing_api_calls == 0
and has_accounted_usage
and bool(model)
and bool(billing_provider)
and (existing_model != model or existing_provider != billing_provider)
)
if first_accounted_route:
conn.execute(
"""UPDATE sessions
SET model = ?, billing_provider = ?,
billing_base_url = ?, billing_mode = ?
WHERE id = ?""",
(model, billing_provider, billing_base_url, billing_mode, session_id),
)
conn.execute(sql, params)
if record_model_usage:
self._record_model_usage(
conn,
session_id,
model=model,
billing_provider=billing_provider,
billing_base_url=billing_base_url,
billing_mode=billing_mode,
input_tokens=input_tokens,
output_tokens=output_tokens,
cache_read_tokens=cache_read_tokens,
cache_write_tokens=cache_write_tokens,
reasoning_tokens=reasoning_tokens,
estimated_cost_usd=estimated_cost_usd,
actual_cost_usd=actual_cost_usd,
cost_status=cost_status,
cost_source=cost_source,
api_call_count=api_call_count,
)
self._execute_write(_do)
def _record_model_usage(
self,
conn,
session_id: str,
*,
model: Optional[str],
billing_provider: Optional[str],
billing_base_url: Optional[str],
billing_mode: Optional[str],
input_tokens: int,
output_tokens: int,
cache_read_tokens: int,
cache_write_tokens: int,
reasoning_tokens: int,
estimated_cost_usd: Optional[float],
actual_cost_usd: Optional[float],
cost_status: Optional[str],
cost_source: Optional[str],
api_call_count: int,
task: str = "",
) -> None:
"""Accumulate a per-API-call usage delta into session_model_usage.
Runs inside the caller's write transaction (after the ``sessions``
UPDATE) so the per-model rows stay consistent with the summary row.
When the caller omits the model/provider (some paths only pass token
deltas), fall back to the values already recorded on the session row —
the same COALESCE-from-session behaviour the summary update uses.
``task`` distinguishes what kind of work consumed the tokens:
``''`` (empty) is the main agent loop; auxiliary calls record their
task name (``vision``, ``compression``, ``title_generation``, ...)
via :meth:`record_auxiliary_usage` (issue #23270).
"""
row = conn.execute(
"SELECT model, billing_provider, billing_base_url, billing_mode "
"FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
sess_model = row["model"] if row is not None else None
sess_provider = row["billing_provider"] if row is not None else None
sess_base_url = row["billing_base_url"] if row is not None else None
sess_billing_mode = row["billing_mode"] if row is not None else None
# Aux-task rows (task != '') must NOT inherit the session's main-loop
# route: an aux call may use a completely different provider/model
# (vision on gemini while the main loop runs anthropic). Missing info
# stays 'unknown'/empty rather than borrowing a misleading route.
if task:
eff_model = model or "unknown"
eff_provider = billing_provider or ""
eff_base_url = billing_base_url or ""
eff_billing_mode = billing_mode or ""
else:
eff_model = model or sess_model or "unknown"
eff_provider = billing_provider or sess_provider or ""
eff_base_url = billing_base_url or sess_base_url or ""
eff_billing_mode = billing_mode or sess_billing_mode or ""
now = time.time()
conn.execute(
"""INSERT INTO session_model_usage (
session_id, model, billing_provider, billing_base_url, billing_mode,
task, api_call_count, input_tokens, output_tokens,
cache_read_tokens, cache_write_tokens, reasoning_tokens,
estimated_cost_usd, actual_cost_usd, cost_status, cost_source,
first_seen, last_seen
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(session_id, model, billing_provider, billing_base_url, billing_mode, task)
DO UPDATE SET
api_call_count = api_call_count + excluded.api_call_count,
input_tokens = input_tokens + excluded.input_tokens,
output_tokens = output_tokens + excluded.output_tokens,
cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens,
cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens,
reasoning_tokens = reasoning_tokens + excluded.reasoning_tokens,
estimated_cost_usd = estimated_cost_usd + excluded.estimated_cost_usd,
actual_cost_usd = actual_cost_usd + excluded.actual_cost_usd,
cost_status = COALESCE(excluded.cost_status, cost_status),
cost_source = COALESCE(excluded.cost_source, cost_source),
last_seen = excluded.last_seen""",
(
session_id,
eff_model,
eff_provider,
eff_base_url,
eff_billing_mode,
task or "",
api_call_count or 0,
input_tokens or 0,
output_tokens or 0,
cache_read_tokens or 0,
cache_write_tokens or 0,
reasoning_tokens or 0,
float(estimated_cost_usd or 0.0),
float(actual_cost_usd or 0.0),
cost_status,
cost_source,
now,
now,
),
)
def ensure_session(
self,
session_id: str,
source: str = "unknown",
model: str = None,
**kwargs,
) -> str:
"""Ensure a session row exists (INSERT OR IGNORE). Accepts optional kwargs."""
self._insert_session_row(session_id, source, model=model, **kwargs)
return session_id
def record_auxiliary_usage(
self,
session_id: str,
task: str,
*,
model: Optional[str] = None,
billing_provider: Optional[str] = None,
billing_base_url: Optional[str] = None,
input_tokens: int = 0,
output_tokens: int = 0,
cache_read_tokens: int = 0,
cache_write_tokens: int = 0,
reasoning_tokens: int = 0,
estimated_cost_usd: Optional[float] = None,
) -> None:
"""Record an auxiliary LLM call's usage against *session_id* (issue #23270).
Auxiliary calls (vision, compression, title_generation, web_extract,
session_search, ...) historically discarded their usage, leaving the
dashboard's per-model analytics blind to aux model spend. This writes
a per-(model, provider, task) delta into ``session_model_usage`` —
the same table the main loop's ``update_token_counts`` feeds — WITHOUT
touching the ``sessions`` summary row. That separation is deliberate:
the gateway overwrites session counters with absolute main-loop totals,
so folding aux tokens into the summary row would either be clobbered
or double-counted. Insights/analytics read the union of both.
Best-effort by contract: callers must never fail an aux call because
accounting failed.
"""
if not session_id or not task:
return
# FK on session_model_usage.session_id → sessions.id: ensure the row
# exists (same INSERT OR IGNORE guard update_token_counts uses — the
# initial create_session() can fail under concurrent SQLite locking).
self._insert_session_row(session_id, "unknown")
def _do(conn):
self._record_model_usage(
conn,
session_id,
model=model,
billing_provider=billing_provider,
billing_base_url=billing_base_url,
billing_mode=None,
input_tokens=input_tokens or 0,
output_tokens=output_tokens or 0,
cache_read_tokens=cache_read_tokens or 0,
cache_write_tokens=cache_write_tokens or 0,
reasoning_tokens=reasoning_tokens or 0,
estimated_cost_usd=estimated_cost_usd,
actual_cost_usd=None,
cost_status=None,
cost_source=None,
api_call_count=1,
task=task,
)
self._execute_write(_do)
def prune_empty_ghost_sessions(self, sessions_dir: "Optional[Path]" = None) -> int:
"""Remove empty TUI ghost sessions (no messages, no title, >24hr old)."""
cutoff = time.time() - 86400 # Only sessions older than 24 hours
def _do(conn):
rows = conn.execute("""
SELECT id FROM sessions
WHERE source = 'tui'
AND title IS NULL
AND ended_at IS NOT NULL
AND started_at < ?
AND NOT EXISTS (
SELECT 1 FROM messages WHERE messages.session_id = sessions.id
)
""", (cutoff,)).fetchall()
ids = [r[0] if isinstance(r, (tuple, list)) else r["id"] for r in rows]
if ids:
placeholders = ",".join("?" * len(ids))
conn.execute(
f"DELETE FROM sessions WHERE id IN ({placeholders})", ids
)
return ids
removed_ids = self._execute_write(_do) or []
# Clean up any on-disk session files (belt-and-suspenders)
if sessions_dir and removed_ids:
for sid in removed_ids:
self._remove_session_files(sessions_dir, sid)
return len(removed_ids)
def finalize_orphaned_compression_sessions(self) -> int:
"""Mark orphaned compression continuation sessions as ended.
Targets child sessions that were never finalized: parent is ended
with reason='compression', child has messages but no end_reason/ended_at
and api_call_count=0. Non-destructive: preserves all messages and sets
end_reason='orphaned_compression'. Fix for #20001.
"""
cutoff = time.time() - 604800 # 7 days
def _do(conn):
now = time.time()
result = conn.execute(
"""
UPDATE sessions
SET ended_at = ?,
end_reason = 'orphaned_compression'
WHERE api_call_count = 0
AND end_reason IS NULL
AND ended_at IS NULL
AND started_at < ?
AND parent_session_id IS NOT NULL
AND EXISTS (
SELECT 1 FROM sessions p
WHERE p.id = sessions.parent_session_id
AND p.end_reason = 'compression'
AND p.ended_at IS NOT NULL
)
AND EXISTS (
SELECT 1 FROM messages m
WHERE m.session_id = sessions.id
)
""",
(now, cutoff),
)
return result.rowcount
return self._execute_write(_do) or 0
def get_session(self, session_id: str) -> Optional[Dict[str, Any]]:
"""Get a session by ID."""
# Cost/usage readers (/status, /usage, gateway endpoints) reach the
# row through here; drain queued token deltas so they see exact
# totals. No-op attribute check when nothing is queued.
self.flush_token_counts()
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT * FROM sessions WHERE id = ?", (session_id,)
)
row = cursor.fetchone()
return dict(row) if row else None
def resolve_session_id(self, session_id_or_prefix: str) -> Optional[str]:
"""Resolve an exact or uniquely prefixed session ID to the full ID.
Returns the exact ID when it exists. Otherwise treats the input as a
prefix and returns the single matching session ID if the prefix is
unambiguous. Returns None for no matches or ambiguous prefixes.
"""
exact = self.get_session(session_id_or_prefix)
if exact:
return exact["id"]
escaped = (
session_id_or_prefix
.replace("\\", "\\\\")
.replace("%", "\\%")
.replace("_", "\\_")
)
with self._lock:
cursor = self._conn.execute(
"SELECT id FROM sessions WHERE id LIKE ? ESCAPE '\\' ORDER BY started_at DESC LIMIT 2",
(f"{escaped}%",),
)
matches = [row["id"] for row in cursor.fetchall()]
if len(matches) == 1:
return matches[0]
return None
# Maximum length for session titles
MAX_TITLE_LENGTH = 100
@staticmethod
def sanitize_title(title: Optional[str]) -> Optional[str]:
"""Validate and sanitize a session title.
- Strips leading/trailing whitespace
- Removes ASCII control characters (0x00-0x1F, 0x7F) and problematic
Unicode control chars (zero-width, RTL/LTR overrides, etc.)
- Collapses internal whitespace runs to single spaces
- Normalizes empty/whitespace-only strings to None
- Enforces MAX_TITLE_LENGTH
Returns the cleaned title string or None.
Raises ValueError if the title exceeds MAX_TITLE_LENGTH after cleaning.
"""
if not title:
return None
# Lone surrogates cannot be bound by sqlite3 (UnicodeEncodeError at
# UTF-8 encode time) — scrub them like every other write path here.
title = _sanitize_surrogates(title)
# Remove ASCII control characters (0x00-0x1F, 0x7F) but keep
# whitespace chars (\t=0x09, \n=0x0A, \r=0x0D) so they can be
# normalized to spaces by the whitespace collapsing step below
cleaned = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', title)
# Remove problematic Unicode control characters:
# - Zero-width chars (U+200B-U+200F, U+FEFF)
# - Directional overrides (U+202A-U+202E, U+2066-U+2069)
# - Object replacement (U+FFFC), interlinear annotation (U+FFF9-U+FFFB)
cleaned = re.sub(
r'[\u200b-\u200f\u2028-\u202e\u2060-\u2069\ufeff\ufffc\ufff9-\ufffb]',
'', cleaned,
)
# Collapse internal whitespace runs and strip
cleaned = re.sub(r'\s+', ' ', cleaned).strip()
if not cleaned:
return None
if len(cleaned) > SessionDB.MAX_TITLE_LENGTH:
raise ValueError(
f"Title too long ({len(cleaned)} chars, max {SessionDB.MAX_TITLE_LENGTH})"
)
return cleaned
def _is_compression_ancestor(
self, conn, *, ancestor_id: str, descendant_id: str
) -> bool:
"""Return True if *ancestor_id* is a compression predecessor of
*descendant_id* (walking parent links up the continuation chain).
The continuation edge is the canonical one shared with
:func:`_ephemeral_child_sql` / :meth:`set_session_archived`
(``_COMPRESSION_CHILD_SQL``): a parent → child edge counts only when the
parent ended with ``end_reason = 'compression'`` and the child started
at or after the parent's ``ended_at``, which distinguishes continuations
from delegate subagents / branch children that also carry a
``parent_session_id``. Expressed as a single recursive CTE rather than a
per-hop Python walk so the edge definition lives in exactly one place.
"""
if not ancestor_id or not descendant_id or ancestor_id == descendant_id:
return False
# Walk parent links up from the descendant, following only compression
# continuation edges, and check whether ancestor_id is reached.
edge = _COMPRESSION_CHILD_SQL.format(a="child")
row = conn.execute(
f"""
WITH RECURSIVE ancestors(id) AS (
SELECT ?
UNION
SELECT parent.id
FROM ancestors a
JOIN sessions child ON child.id = a.id
JOIN sessions parent ON parent.id = child.parent_session_id
WHERE {edge}
)
SELECT 1 FROM ancestors WHERE id = ? AND id != ? LIMIT 1
""",
(descendant_id, ancestor_id, descendant_id),
).fetchone()
return row is not None
def _set_session_title(
self,
session_id: str,
title: str,
*,
only_if_empty: bool,
) -> bool:
title = self.sanitize_title(title)
def _do(conn):
if only_if_empty:
current = conn.execute(
"SELECT title FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if current is None or current["title"] is not None:
return 0
if title:
# Check uniqueness (allow the same session to keep its own title)
cursor = conn.execute(
"SELECT id FROM sessions WHERE title = ? AND id != ?",
(title, session_id),
)
conflict = cursor.fetchone()
if conflict:
conflict_id = conflict["id"]
# A compression continuation is the live, projected-forward
# head of its conversation; its compressed predecessors are
# ended and hidden from the session list (list_sessions_rich
# projects roots → tip). When the title that "conflicts" is
# held by such a hidden ancestor, the user has no way to free
# it — renaming the visible tip back to the base name would
# dead-end with "already in use by <session they can't see>".
# Treat this as a transfer: move the title off the ancestor
# onto the continuation. Uniqueness is preserved (still only
# one session carries the exact title) and the parent-link
# lineage is untouched.
if self._is_compression_ancestor(
conn, ancestor_id=conflict_id, descendant_id=session_id
):
conn.execute(
"UPDATE sessions SET title = NULL WHERE id = ?",
(conflict_id,),
)
else:
raise ValueError(
f"Title '{title}' is already in use by session {conflict_id}"
)
predicate = " AND title IS NULL" if only_if_empty else ""
cursor = conn.execute(
f"UPDATE sessions SET title = ? WHERE id = ?{predicate}",
(title, session_id),
)
return cursor.rowcount
rowcount = self._execute_write(_do)
return rowcount > 0
def set_session_title(self, session_id: str, title: str) -> bool:
"""Set or update a session's title.
Returns True if session was found and title was set.
Raises ValueError if title is already in use by another session,
or if the title fails validation (too long, invalid characters).
Empty/whitespace-only strings are normalized to None (clearing the title).
"""
return self._set_session_title(session_id, title, only_if_empty=False)
def set_auto_title_if_empty(self, session_id: str, title: str) -> bool:
"""Set an auto-generated title only when the current title is NULL.
The predicate and write run in one transaction so a concurrent manual
rename cannot be overwritten. Validation and uniqueness behavior match
:meth:`set_session_title`.
"""
return self._set_session_title(session_id, title, only_if_empty=True)
def get_session_title(self, session_id: str) -> Optional[str]:
"""Get the title for a session, or None."""
with self._lock:
cursor = self._conn.execute(
"SELECT title FROM sessions WHERE id = ?", (session_id,)
)
row = cursor.fetchone()
return row["title"] if row else None
def set_session_archived(self, session_id: str, archived: bool) -> bool:
"""Archive or unarchive a session.
Archived sessions are hidden from the default session list but keep all
their messages — this is a soft hide, not a delete. For compression
chains, archive the whole logical conversation. Desktop lists compression
roots projected forward to their latest continuation; updating only the
displayed tip lets the still-unarchived root resurrect it on refresh.
Returns True when at least one row was updated.
"""
def _do(conn):
cursor = conn.execute(
"""
WITH RECURSIVE
ancestors(id) AS (
SELECT ?
UNION
SELECT parent.id
FROM ancestors a
JOIN sessions child ON child.id = a.id
JOIN sessions parent ON parent.id = child.parent_session_id
WHERE parent.end_reason = 'compression'
),
descendants(id) AS (
SELECT ?
UNION
SELECT child.id
FROM descendants d
JOIN sessions parent ON parent.id = d.id
JOIN sessions child ON child.parent_session_id = parent.id
WHERE parent.end_reason = 'compression'
),
lineage(id) AS (
SELECT id FROM ancestors
UNION
SELECT id FROM descendants
)
UPDATE sessions
SET archived = ?
WHERE id IN (SELECT id FROM lineage)
""",
(session_id, session_id, 1 if archived else 0),
)
rowcount = cursor.rowcount
if rowcount is None or rowcount < 0:
rowcount = conn.execute("SELECT changes()").fetchone()[0]
return rowcount
rowcount = self._execute_write(_do)
return rowcount > 0
def set_session_pinned(self, session_id: str, pinned: bool) -> bool:
"""Pin or unpin a session (and its whole compression lineage).
``pinned`` is a durable "keep" flag: pinned sessions are exempt from
the ``sessions.auto_archive`` stale sweep (see
:meth:`archive_stale_sessions`). Desktop is the current writer — its
sidebar pins mirror here so a backend/other-surface sweep honours
them. Like :meth:`set_session_archived` the whole compression chain is
flipped as a unit, so pinning the surfaced tip protects the root (and
vice-versa) no matter which id the caller holds. Returns True when at
least one row changed.
"""
def _do(conn):
cursor = conn.execute(
"""
WITH RECURSIVE
ancestors(id) AS (
SELECT ?
UNION
SELECT parent.id
FROM ancestors a
JOIN sessions child ON child.id = a.id
JOIN sessions parent ON parent.id = child.parent_session_id
WHERE parent.end_reason = 'compression'
),
descendants(id) AS (
SELECT ?
UNION
SELECT child.id
FROM descendants d
JOIN sessions parent ON parent.id = d.id
JOIN sessions child ON child.parent_session_id = parent.id
WHERE parent.end_reason = 'compression'
),
lineage(id) AS (
SELECT id FROM ancestors
UNION
SELECT id FROM descendants
)
UPDATE sessions
SET pinned = ?
WHERE id IN (SELECT id FROM lineage)
""",
(session_id, session_id, 1 if pinned else 0),
)
rowcount = cursor.rowcount
if rowcount is None or rowcount < 0:
rowcount = conn.execute("SELECT changes()").fetchone()[0]
return rowcount
rowcount = self._execute_write(_do)
return rowcount > 0
def get_session_by_title(self, title: str) -> Optional[Dict[str, Any]]:
"""Look up a session by exact title. Returns session dict or None."""
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT * FROM sessions WHERE title = ?", (title,)
)
row = cursor.fetchone()
return dict(row) if row else None
def resolve_session_by_title(self, title: str) -> Optional[str]:
"""Resolve a title to a session ID, preferring the latest in a lineage.
If the exact title exists, returns that session's ID.
If not, searches for "title #N" variants and returns the latest one.
If the exact title exists AND numbered variants exist, returns the
latest numbered variant (the most recent continuation).
"""
# First try exact match
exact = self.get_session_by_title(title)
# Also search for numbered variants: "title #2", "title #3", etc.
# Escape SQL LIKE wildcards (%, _) in the title to prevent false matches
escaped = title.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
with self._read_ctx() as conn:
cursor = conn.execute(
"SELECT id, title, started_at FROM sessions "
"WHERE title LIKE ? ESCAPE '\\' ORDER BY started_at DESC",
(f"{escaped} #%",),
)
numbered = cursor.fetchall()
if numbered:
# Return the most recent numbered variant
return numbered[0]["id"]
elif exact:
return exact["id"]
return None
def get_next_title_in_lineage(self, base_title: str) -> str:
"""Generate the next title in a lineage (e.g., "my session" → "my session #2").
Strips any existing " #N" suffix to find the base name, then finds
the highest existing number and increments.
"""
# Strip existing #N suffix to find the true base
match = re.match(r'^(.*?) #(\d+)$', base_title)
if match:
base = match.group(1)
else:
base = base_title
# Find all existing numbered variants
# Escape SQL LIKE wildcards (%, _) in the base to prevent false matches
escaped = base.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
with self._lock:
cursor = self._conn.execute(
"SELECT title FROM sessions WHERE title = ? OR title LIKE ? ESCAPE '\\'",
(base, f"{escaped} #%"),
)
existing = [row["title"] for row in cursor.fetchall()]
if not existing:
return base # No conflict, use the base name as-is
# Find the highest number
max_num = 1 # The unnumbered original counts as #1
for t in existing:
m = re.match(r'^.* #(\d+)$', t)
if m:
max_num = max(max_num, int(m.group(1)))
return f"{base} #{max_num + 1}"
def get_compression_tip(self, session_id: str) -> Optional[str]:
"""Walk the compression-continuation chain forward and return the tip.
A compression continuation is a child of a session whose
``end_reason = 'compression'``. Older builds tried to distinguish
continuations from branches/subagents by requiring
``child.started_at >= parent.ended_at``. That ordering is too brittle:
gateway + compression races can insert the real continuation row before
the parent row's ``ended_at`` is written, while a stale websocket later
creates/reuses a sibling that *does* satisfy the timestamp test. The
visible symptom is brutal: desktop resume follows the stale sibling and
the user's latest messages look "lost" even though they are persisted in
the real continuation chain.
Instead, only follow children of compression-ended parents, exclude
explicit branch/delegate/tool children, and prefer children that are
themselves continuing the compression chain (``end_reason='compression'``)
or still live over stale closed siblings such as ``ws_orphan_reap``.
Returns the latest continuation tip, or the input id when no
continuation exists.
"""
current = session_id
seen = {current} if current else set()
# Bound the walk defensively — compression chains this deep are
# pathological and shouldn't happen in practice. 100 = plenty.
for _ in range(100):
with self._lock:
cursor = self._conn.execute(
"""
SELECT child.id
FROM sessions parent
JOIN sessions child ON child.parent_session_id = parent.id
WHERE parent.id = ?
AND parent.end_reason = 'compression'
AND json_extract(COALESCE(child.model_config, '{}'), '$._branched_from') IS NULL
AND json_extract(COALESCE(child.model_config, '{}'), '$._delegate_from') IS NULL
AND COALESCE(child.source, '') != 'tool'
ORDER BY
CASE
WHEN child.end_reason = 'compression' THEN 0
WHEN child.ended_at IS NULL THEN 1
ELSE 2
END,
COALESCE(
(SELECT MAX(m.timestamp) FROM messages m WHERE m.session_id = child.id),
child.started_at
) DESC,
child.started_at DESC,
child.id DESC
LIMIT 1
""",
(current,),
)
row = cursor.fetchone()
if row is None:
return current
child_id = row["id"]
if not child_id or child_id in seen:
return current
seen.add(child_id)
current = child_id
return current
# Columns excluded from compact_rows projections: only the payload-heavy
# blob no list consumer renders. Everything else — including gateway
# routing fields and desktop sidebar fields like git_branch — stays, and
# the projection is derived from SCHEMA_SQL so columns added later via
# declarative reconciliation are included automatically instead of
# silently dropping out of list rows.
_SESSION_COMPACT_EXCLUDED = frozenset({"system_prompt"})
_session_compact_cols_sql: Optional[str] = None
def list_sessions_rich(
self,
source: str = None,
sources: List[str] = None,
exclude_sources: List[str] = None,
cwd_prefix: str = None,
limit: int = 20,
offset: int = 0,
include_children: bool = False,
min_message_count: int = 0,
project_compression_tips: bool = True,
order_by_last_active: bool = False,
include_archived: bool = False,
archived_only: bool = False,
id_query: str = None,
search_query: str = None,
compact_rows: bool = False,
include_pinned: bool = False,
) -> List[Dict[str, Any]]:
"""List sessions with preview (first user message) and last active timestamp.
Returns dicts with keys: id, source, model, title, started_at, ended_at,
message_count, preview (first 60 chars of first user message),
last_active (timestamp of last message).
Uses a single query with correlated subqueries instead of N+2 queries.
By default, child sessions (subagent runs, compression continuations)
are excluded. Pass ``include_children=True`` to include them.
With ``project_compression_tips=True`` (default), sessions that are
roots of compression chains are projected forward to their latest
continuation — one logical conversation = one list entry, showing the
live continuation's id/message_count/title/last_active. This prevents
compressed continuations from being invisible to users while keeping
delegate subagents and branches hidden. Pass ``False`` to return the
raw root rows (useful for admin/debug UIs).
Pass ``order_by_last_active=True`` to sort by most-recent activity
instead of original conversation start time. For compression chains,
the "most-recent activity" is taken from the live tip (not the root),
so an old conversation that was compressed and continued recently
surfaces in the correct slot. Ordering is computed at SQL level via
a recursive CTE that walks compression-continuation edges, so LIMIT
and OFFSET still apply efficiently.
``search_query`` matches case-insensitive substrings against each
surfaced row's title and id (and, like ``id_query``, every title/id in
its forward compression chain). A punctuation-stripped variant is also
matched so e.g. ``an94`` finds ``AN-94``. Only honored in the
``order_by_last_active`` path.
Pass ``compact_rows=True`` for dashboard and picker callers that only
need lightweight metadata. This omits the ``system_prompt`` blob from
the SELECT so SQLite never copies it out of the B-tree page — a
significant I/O saving on large databases where the blob routinely
runs to tens of kilobytes per row.
Pass ``include_pinned=True`` to back-fill any conversation carrying the
durable ``pinned`` flag that the LIMIT/OFFSET window left out. A pin is
a "this must always be reachable" statement, so a pinned conversation
aging past the requested page is a bug, not a paging outcome — the
desktop sidebar would render an empty Pinned section. Back-filled rows
obey the same filters (source, archived, min_message_count) as the
page: an archived or filtered-out conversation stays out.
"""
# Rows carry token/cost totals — drain queued deltas first so
# listings (sidebar, /resume, dashboards) show exact counters.
self.flush_token_counts()
where_clauses = []
params = []
if not include_children:
# Show root sessions and branch sessions, while still hiding
# sub-agent runs and compression continuations (which also carry a
# parent_session_id but were spawned while the parent was still
# live — i.e., started_at < parent.ended_at).
#
# Branch sessions are identified two ways, OR'd for robustness:
# 1. A stable ``_branched_from`` marker in model_config, written
# by /branch at creation time. This survives the parent being
# reopened and re-ended with a different end_reason (e.g.
# tui_shutdown overwriting 'branched'), which otherwise hides
# the branch — see issue #20856.
# 2. The legacy heuristic (parent ended with 'branched' before the
# child started), covering branch sessions created before the
# marker existed.
where_clauses.append(_LISTABLE_CHILD_SQL)
where_clauses.append(f"{_delegate_from_json('s.model_config')} IS NULL")
include_sources = [source] if source else list(sources or [])
if include_sources:
placeholders = ",".join("?" for _ in include_sources)
where_clauses.append(f"s.source IN ({placeholders})")
params.extend(include_sources)
if exclude_sources:
placeholders = ",".join("?" for _ in exclude_sources)
where_clauses.append(f"s.source NOT IN ({placeholders})")
params.extend(exclude_sources)
if cwd_prefix:
clause, clause_params = _cwd_prefix_clause(cwd_prefix)
where_clauses.append(clause)
params.extend(clause_params)
if min_message_count > 0:
where_clauses.append("s.message_count >= ?")
params.append(min_message_count)
if archived_only:
where_clauses.append("s.archived = 1")
elif not include_archived:
where_clauses.append("s.archived = 0")
where_sql = f"WHERE {' AND '.join(where_clauses)}" if where_clauses else ""
# Snapshot the filter params before the query builders below extend
# them with LIMIT/OFFSET — the pinned back-fill reuses the same WHERE.
base_where_params = list(params)
# Optional session-id filter, pushed into SQL so callers (Desktop
# session-id search) don't have to fetch every row and filter in
# Python. ``id_query`` is matched as a case-insensitive substring
# against each surfaced row's id AND every id in its forward
# compression chain — so searching a compression *root* id or a *tip*
# id both resolve to the same projected conversation. Only used in the
# order_by_last_active path (which builds the chain CTE); other callers
# pass id_query=None.
id_needle = (id_query or "").strip().lower()
search_needle = (search_query or "").strip().lower()
if order_by_last_active:
# Compute effective_last_active by walking each surfaced session's
# compression-continuation chain forward in SQL and taking the MAX
# timestamp across the chain. This lets us ORDER BY + LIMIT at SQL
# level instead of fetching every row and sorting in Python, while
# still surfacing old compression roots whose live tip is fresh.
#
# The CTE seeds from rows the outer WHERE admits (roots + branch
# children), then recursively joins forward through robust
# compression-continuation edges. Do NOT require
# child.started_at >= parent.ended_at here: real desktop/gateway
# races can insert the continuation row before the parent's
# ended_at is written, while stale websocket siblings may satisfy
# the timestamp test and hijack resume/list projection.
outer_where = where_sql
id_params: List[Any] = []
filter_clauses: List[str] = []
def _like_pattern(needle: str) -> str:
escaped = (
needle.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
)
return f"%{escaped}%"
if id_needle:
# Admit a surfaced row if its own id or any id in its forward
# compression chain matches the needle. LIKE with a leading
# wildcard can't use an index, but the chain membership and
# the small result set keep this bounded — far cheaper than
# fetching every session and scanning in Python.
filter_clauses.append(
"EXISTS (SELECT 1 FROM chain cq"
" WHERE cq.root_id = s.id"
" AND LOWER(cq.cur_id) LIKE ? ESCAPE '\\')"
)
id_params.append(_like_pattern(id_needle))
if search_needle:
# Same chain-membership trick as id_query, but matching either
# the title or the id of any session in the chain. The compact
# (punctuation-stripped) variant lets `an94` match `AN-94`.
compact_needle = re.sub(r"[\W_]+", "", search_needle)
compact_sql = (
"REPLACE(REPLACE(REPLACE(REPLACE(LOWER(COALESCE({0}, '')),"
" '-', ''), '_', ''), '.', ''), ' ', '')"
)
search_clause = (
"EXISTS (SELECT 1 FROM chain cq"
" JOIN sessions cs ON cs.id = cq.cur_id"
" WHERE cq.root_id = s.id"
" AND (LOWER(COALESCE(cs.title, '')) LIKE ? ESCAPE '\\'"
" OR LOWER(cq.cur_id) LIKE ? ESCAPE '\\'"
)
id_params.extend([_like_pattern(search_needle)] * 2)
if compact_needle:
search_clause += (
f" OR {compact_sql.format('cs.title')} LIKE ? ESCAPE '\\'"
)
id_params.append(_like_pattern(compact_needle))
filter_clauses.append(search_clause + "))")
if filter_clauses:
combined = " AND ".join(filter_clauses)
outer_where = (
f"{where_sql} AND {combined}" if where_sql else f"WHERE {combined}"
)
_sel = self._compact_session_cols() if compact_rows else "s.*"
query = f"""
WITH RECURSIVE chain(root_id, cur_id) AS (
SELECT s.id, s.id FROM sessions s {where_sql}
UNION ALL
SELECT c.root_id, child.id
FROM chain c
JOIN sessions parent ON parent.id = c.cur_id
JOIN sessions child ON child.parent_session_id = c.cur_id
WHERE parent.end_reason = 'compression'
AND json_extract(COALESCE(child.model_config, '{{}}'), '$._branched_from') IS NULL
AND json_extract(COALESCE(child.model_config, '{{}}'), '$._delegate_from') IS NULL
AND COALESCE(child.source, '') != 'tool'
),
chain_max AS (
SELECT
root_id,
MAX(COALESCE(
(SELECT MAX(m.timestamp) FROM messages m WHERE m.session_id = cur_id),
(SELECT started_at FROM sessions ss WHERE ss.id = cur_id)
)) AS effective_last_active
FROM chain
GROUP BY root_id
)
SELECT {_sel},
COALESCE(
(SELECT {_PREVIEW_RAW_SELECT}
FROM messages m
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
ORDER BY m.timestamp, m.id LIMIT 1),
''
) AS _preview_raw,
COALESCE(
(SELECT MAX(m2.timestamp) FROM messages m2 WHERE m2.session_id = s.id),
s.started_at
) AS last_active,
COALESCE(cm.effective_last_active, s.started_at) AS _effective_last_active
FROM sessions s
LEFT JOIN chain_max cm ON cm.root_id = s.id
{outer_where}
ORDER BY _effective_last_active DESC, s.started_at DESC, s.id DESC
LIMIT ? OFFSET ?
"""
# WHERE params apply twice (CTE seed + outer select); the id filter
# only applies to the outer select.
params = params + params + id_params + [limit, offset]
else:
_sel = self._compact_session_cols() if compact_rows else "s.*"
query = f"""
SELECT {_sel},
COALESCE(
(SELECT {_PREVIEW_RAW_SELECT}
FROM messages m
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
ORDER BY m.timestamp, m.id LIMIT 1),
''
) AS _preview_raw,
COALESCE(
(SELECT MAX(m2.timestamp) FROM messages m2 WHERE m2.session_id = s.id),
s.started_at
) AS last_active
FROM sessions s
{where_sql}
ORDER BY s.started_at DESC
LIMIT ? OFFSET ?
"""
params.extend([limit, offset])
with self._read_ctx() as conn:
cursor = conn.execute(query, params)
rows = cursor.fetchall()
sessions = []
for row in rows:
s = dict(row)
s["preview"] = _shape_preview(s.pop("_preview_raw", ""))
# Drop the internal ordering column so callers see a clean dict.
s.pop("_effective_last_active", None)
sessions.append(s)
# Back-fill pinned conversations the page missed. A pin outlives
# recency, so this runs BEFORE compression projection below — a
# back-filled root then projects to its live tip exactly like a row
# that had made the page on its own. One extra query, bounded by the
# number of pins (a handful), never N+1 per pin.
if include_pinned:
seen_ids = {s["id"] for s in sessions}
pinned_where = (
f"{where_sql} AND s.pinned = 1" if where_sql else "WHERE s.pinned = 1"
)
_sel = self._compact_session_cols() if compact_rows else "s.*"
pinned_query = f"""
SELECT {_sel},
COALESCE(
(SELECT {_PREVIEW_RAW_SELECT}
FROM messages m
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
ORDER BY m.timestamp, m.id LIMIT 1),
''
) AS _preview_raw,
COALESCE(
(SELECT MAX(m2.timestamp) FROM messages m2 WHERE m2.session_id = s.id),
s.started_at
) AS last_active
FROM sessions s
{pinned_where}
ORDER BY s.started_at DESC
"""
with self._read_ctx() as conn:
pinned_cursor = conn.execute(pinned_query, base_where_params)
pinned_rows = pinned_cursor.fetchall()
for row in pinned_rows:
s = dict(row)
if s["id"] in seen_ids:
continue
s["preview"] = _shape_preview(s.pop("_preview_raw", ""))
seen_ids.add(s["id"])
sessions.append(s)
# Project compression roots forward to their tips. Each row whose
# end_reason is 'compression' has a continuation child; replace the
# surfaced fields (id, message_count, title, last_active, ended_at,
# end_reason, preview) with the tip's values so the list entry acts
# as the live conversation. Keep the root's started_at to preserve
# chronological ordering by original conversation start.
if project_compression_tips and not include_children:
projected = []
for s in sessions:
if s.get("end_reason") != "compression":
projected.append(s)
continue
tip_id = self.get_compression_tip(s["id"])
if tip_id == s["id"]:
projected.append(s)
continue
tip_row = self._get_session_rich_row(tip_id, compact_rows=compact_rows)
if not tip_row:
projected.append(s)
continue
# Preserve the root's started_at for stable sort order, but
# surface the tip's identity and activity data.
merged = dict(s)
for key in (
"id", "ended_at", "end_reason", "message_count",
"tool_call_count", "title", "last_active", "preview",
"model", "system_prompt", "cwd", "git_branch", "git_repo_root",
):
if key in tip_row:
merged[key] = tip_row[key]
merged["_lineage_root_id"] = s["id"]
projected.append(merged)
sessions = projected
return sessions
# =========================================================================
# Message storage
# =========================================================================
# Sentinel prefix used to distinguish JSON-encoded structured content
# (multimodal messages: lists of parts like text + image_url) from plain
# string content. The NUL byte is not legal in normal text, so this
# cannot collide with real user content.
_CONTENT_JSON_PREFIX = "\x00json:"
@classmethod
def _encode_content(cls, content: Any) -> Any:
"""Serialize structured (list/dict) message content for sqlite.
sqlite3 can only bind ``str``, ``bytes``, ``int``, ``float``, and ``None``
to query parameters. Multimodal messages have ``content`` as a list of
parts (``[{"type": "text", ...}, {"type": "image_url", ...}]``), which
raises ``ProgrammingError: Error binding parameter N: type 'list' is
not supported`` when bound directly.
Returns the value unchanged when it's already a safe scalar, or a
sentinel-prefixed JSON string for lists/dicts. Paired with
:meth:`_decode_content` on read.
"""
if isinstance(content, str):
# Lone UTF-16 surrogates reach here inside tool results scraped
# from the web/social platforms (the same input that crashed the
# guardrail hasher). The proactive sanitizer upstream only cleans
# the *api_messages* copy, and the recovery sanitizer only runs
# after the API call itself raises — which it no longer does — so
# the canonical history keeps them and this write is where they
# land. Left raw, sqlite3 raises UnicodeEncodeError, the flush is
# abandoned, and the session silently stops persisting for the
# rest of its life. Scrub so persistence never fails.
return _sanitize_surrogates(content)
if content is None or isinstance(content, (bytes, int, float)):
return content
try:
# json.dumps defaults to ensure_ascii=True, which escapes any
# surrogate as \udXXX — already safe to bind.
return cls._CONTENT_JSON_PREFIX + json.dumps(content)
except (TypeError, ValueError):
# Last-resort fallback: stringify so persistence never fails.
return _sanitize_surrogates(str(content))
@classmethod
def _decode_content(cls, content: Any) -> Any:
"""Reverse :meth:`_encode_content`; returns scalars unchanged."""
if isinstance(content, str) and content.startswith(cls._CONTENT_JSON_PREFIX):
try:
return json.loads(content[len(cls._CONTENT_JSON_PREFIX):])
except (json.JSONDecodeError, TypeError):
logger.warning(
"Failed to decode JSON-encoded message content; "
"returning raw string"
)
return content
return content
@staticmethod
def _encode_display_metadata(display_metadata: Any) -> Optional[str]:
"""Serialize ``display_metadata`` for its TEXT column without double-encoding.
Import/replace paths can hand us an already-serialized JSON string (the
same hazard ``tool_calls`` guards against above). ``json.dumps`` on that
string would store a quoted JSON string, and the single ``json.loads``
on read then yields a ``str`` instead of a dict.
"""
if not display_metadata:
return None
if isinstance(display_metadata, str):
try:
parsed = json.loads(display_metadata)
except (json.JSONDecodeError, TypeError):
logger.warning("Ignoring non-JSON display metadata on write")
return None
if not isinstance(parsed, dict):
logger.warning("Ignoring non-object display metadata on write")
return None
return json.dumps(parsed)
if isinstance(display_metadata, dict):
return json.dumps(display_metadata)
logger.warning(
"Ignoring unexpected display metadata type on write: %s",
type(display_metadata).__name__,
)
return None
@staticmethod
def _decode_display_metadata(raw: Any) -> Optional[Dict[str, Any]]:
"""Decode a ``display_metadata`` column into the dict every reader expects.
Every message read path must go through this. Returning the raw TEXT
instead reaches the desktop as a string, where ``'task_count' in meta``
throws and fails the whole resume. Rows written before the encode guard
landed are double-encoded, so unwrap a second layer when we find one.
"""
if raw is None:
return None
try:
meta = json.loads(raw) if isinstance(raw, str) else raw
if isinstance(meta, str):
meta = json.loads(meta)
except (json.JSONDecodeError, TypeError):
logger.warning("Ignoring invalid display metadata on message row")
return None
if not isinstance(meta, dict):
logger.warning("Ignoring non-object display metadata on message row")
return None
return meta
def append_message(
self,
session_id: str,
role: str,
content: str = None,
tool_name: str = None,
tool_calls: Any = None,
tool_call_id: str = None,
token_count: int = None,
finish_reason: str = None,
reasoning: str = None,
reasoning_content: str = None,
reasoning_details: Any = None,
codex_reasoning_items: Any = None,
codex_message_items: Any = None,
platform_message_id: str = None,
observed: bool = False,
effect_disposition: Optional[str] = None,
timestamp: Any = None,
api_content: Optional[str] = None,
display_kind: Optional[str] = None,
display_metadata: Optional[Dict[str, Any]] = None,
compression_lock_holder: Optional[str] = None,
) -> int:
"""
Append a message to a session. Returns the message row ID.
Also increments the session's message_count (and tool_call_count
if role is 'tool' or tool_calls is present).
``platform_message_id`` is the external messaging platform's own
message ID (e.g. Telegram update_id, Yuanbao msg_id). It is
independent of the SQLite autoincrement primary key and is used by
platform-specific flows like yuanbao's recall guard to redact a
message by its platform-side identifier.
``api_content`` is the exact content string sent to the API for this
message when it differs from ``content`` (ephemeral memory/plugin
injections, persist overrides). It is a byte-fidelity sidecar for
prompt-cache-stable replay — stored as sent, except lone surrogates
(which sqlite3 cannot bind and which the conversation loop scrubs
from every outgoing payload anyway, so the scrubbed form IS the
wire bytes).
"""
# Display metadata is presentation-only and never changes the model
# context role/content replayed to providers.
display_metadata_json = self._encode_display_metadata(display_metadata)
# Serialize structured fields to JSON before entering the write txn
reasoning_details_json = (
json.dumps(reasoning_details)
if reasoning_details else None
)
codex_items_json = (
json.dumps(codex_reasoning_items)
if codex_reasoning_items else None
)
codex_message_items_json = (
json.dumps(codex_message_items)
if codex_message_items else None
)
# tool_calls may arrive as a Python list (from the live agent) or
# as a JSON string (from import/export). Parse first to avoid
# double-encoding.
if isinstance(tool_calls, str):
try:
tool_calls = json.loads(tool_calls)
except (json.JSONDecodeError, TypeError):
tool_calls = []
tool_calls_json = json.dumps(tool_calls) if tool_calls else None
# Multimodal content (list of parts) must be JSON-encoded: sqlite3
# cannot bind list/dict parameters directly.
stored_content = self._encode_content(content)
message_timestamp = time.time()
if timestamp is not None:
try:
if hasattr(timestamp, "timestamp"):
message_timestamp = float(timestamp.timestamp())
else:
message_timestamp = float(timestamp)
except (TypeError, ValueError):
logger.debug("Ignoring invalid explicit message timestamp: %r", timestamp)
# Pre-compute tool call count
num_tool_calls = 0
if tool_calls is not None:
num_tool_calls = len(tool_calls) if isinstance(tool_calls, list) else 1
def _do(conn):
active_lock = conn.execute(
"SELECT holder FROM compression_locks "
"WHERE session_id = ? AND expires_at > ?",
(session_id, time.time()),
).fetchone()
if (
active_lock is not None
and active_lock["holder"] != compression_lock_holder
):
raise CompressionSessionBusyError(
f"Session {session_id!r} is being compressed by another writer"
)
session = conn.execute(
"SELECT ended_at, end_reason FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if (
session is not None
and session["ended_at"] is not None
and session["end_reason"] == "compression"
):
raise CompressionSessionClosedError(session_id)
cursor = conn.execute(
"""INSERT INTO messages (session_id, role, content, tool_call_id,
tool_calls, tool_name, effect_disposition, timestamp, token_count, finish_reason,
reasoning, reasoning_content, reasoning_details, codex_reasoning_items,
codex_message_items, platform_message_id, observed, active, api_content, display_kind, display_metadata)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(
session_id,
role,
stored_content,
tool_call_id,
tool_calls_json,
_scrub_surrogates(tool_name),
effect_disposition,
message_timestamp,
token_count,
finish_reason,
_scrub_surrogates(reasoning),
_scrub_surrogates(reasoning_content),
reasoning_details_json,
codex_items_json,
codex_message_items_json,
platform_message_id,
1 if observed else 0,
1,
_scrub_surrogates(api_content) if isinstance(api_content, str) else None,
_scrub_surrogates(display_kind) if isinstance(display_kind, str) else None,
display_metadata_json,
),
)
msg_id = cursor.lastrowid
# Update counters
if num_tool_calls > 0:
conn.execute(
"""UPDATE sessions SET message_count = message_count + 1,
tool_call_count = tool_call_count + ? WHERE id = ?""",
(num_tool_calls, session_id),
)
else:
conn.execute(
"UPDATE sessions SET message_count = message_count + 1 WHERE id = ?",
(session_id,),
)
return msg_id
# Transcript append is THE critical write: its failure aborts the
# user's turn (session_persistence_failed). Use the long patience so
# a sibling process legitimately holding the write lock for seconds
# (VACUUM, TRUNCATE checkpoint at close, an older pre-bounded-merge
# process's FTS optimize) can't destroy a healthy turn (#74478).
return self._execute_write(
_do, patience_s=self._TRANSCRIPT_WRITE_PATIENCE_S
)
def set_latest_matching_message_display_kind(
self, session_id: str, *, role: str, content: str, display_kind: str,
display_metadata: Optional[Dict[str, Any]] = None,
) -> bool:
"""Stamp presentation metadata on this turn's freshly persisted row.
The model still receives ``role`` and ``content`` unchanged. Gateway and
CLI synthetic inputs call this immediately after their serial turn has
flushed, preserving producer provenance without classifying by content
during transcript rendering.
"""
if not session_id or not content or not display_kind:
return False
def _do(conn):
row = conn.execute(
"SELECT id FROM messages WHERE session_id = ? AND role = ? "
"AND content = ? AND active = 1 ORDER BY id DESC LIMIT 1",
(session_id, role, self._encode_content(content)),
).fetchone()
if row is None:
return False
conn.execute(
"UPDATE messages SET display_kind = ?, display_metadata = ? WHERE id = ?",
(
_scrub_surrogates(display_kind),
self._encode_display_metadata(display_metadata),
row[0],
),
)
return True
return bool(self._execute_write(_do))
def _insert_message_rows(self, conn, session_id: str, messages: List[Dict[str, Any]]) -> tuple[int, int]:
"""Insert *messages* as fresh active rows for *session_id*.
Shared by :meth:`replace_messages` (delete-then-insert) and
:meth:`archive_and_compact` (soft-archive-then-insert). Runs inside the
caller's write transaction (takes the live ``conn``). Returns
``(inserted_count, tool_call_count)``. Does NOT touch sessions.* counters
— the caller owns that, since the two flows reconcile counts differently.
"""
now_ts = time.time()
inserted = 0
tool_calls_total = 0
for msg in messages:
role = msg.get("role", "unknown")
tool_calls = msg.get("tool_calls")
message_timestamp = now_ts
if msg.get("timestamp") is not None:
try:
ts_value = msg.get("timestamp")
if hasattr(ts_value, "timestamp"):
message_timestamp = float(ts_value.timestamp())
else:
message_timestamp = float(ts_value)
except (TypeError, ValueError):
logger.debug("Ignoring invalid explicit message timestamp: %r", msg.get("timestamp"))
reasoning_details = msg.get("reasoning_details") if role == "assistant" else None
codex_reasoning_items = (
msg.get("codex_reasoning_items") if role == "assistant" else None
)
codex_message_items = (
msg.get("codex_message_items") if role == "assistant" else None
)
reasoning_details_json = (
json.dumps(reasoning_details) if reasoning_details else None
)
codex_items_json = (
json.dumps(codex_reasoning_items) if codex_reasoning_items else None
)
codex_message_items_json = (
json.dumps(codex_message_items) if codex_message_items else None
)
# tool_calls may arrive as a Python list (from the live agent)
# or as a JSON string (from import_sessions / export_session,
# which store it as TEXT). json.dumps on an already-serialized
# string double-encodes it, so parse first.
if isinstance(tool_calls, str):
try:
tool_calls = json.loads(tool_calls)
except (json.JSONDecodeError, TypeError):
tool_calls = []
tool_calls_json = json.dumps(tool_calls) if tool_calls else None
# Accept either `platform_message_id` (new explicit name) or
# `message_id` (yuanbao's existing convention on message dicts).
platform_msg_id = (
msg.get("platform_message_id") or msg.get("message_id")
)
api_content = msg.get("api_content")
conn.execute(
"""INSERT INTO messages (session_id, role, content, tool_call_id,
tool_calls, tool_name, effect_disposition, timestamp, token_count, finish_reason,
reasoning, reasoning_content, reasoning_details, codex_reasoning_items,
codex_message_items, platform_message_id, observed, active, api_content, display_kind, display_metadata)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(
session_id,
role,
self._encode_content(msg.get("content")),
msg.get("tool_call_id"),
tool_calls_json,
_scrub_surrogates(msg.get("tool_name")),
msg.get("effect_disposition"),
message_timestamp,
msg.get("token_count"),
msg.get("finish_reason"),
_scrub_surrogates(msg.get("reasoning")) if role == "assistant" else None,
_scrub_surrogates(msg.get("reasoning_content")) if role == "assistant" else None,
reasoning_details_json,
codex_items_json,
codex_message_items_json,
platform_msg_id,
1 if msg.get("observed") else 0,
1,
_scrub_surrogates(api_content) if isinstance(api_content, str) else None,
_scrub_surrogates(msg.get("display_kind")) if isinstance(msg.get("display_kind"), str) else None,
self._encode_display_metadata(msg.get("display_metadata")),
),
)
inserted += 1
if tool_calls is not None:
tool_calls_total += (
len(tool_calls) if isinstance(tool_calls, list) else 1
)
now_ts = max(now_ts + 1e-6, message_timestamp + 1e-6)
return inserted, tool_calls_total
def replace_messages(
self,
session_id: str,
messages: List[Dict[str, Any]],
active_only: bool = False,
) -> None:
"""Atomically replace the stored messages for a session.
Used by transcript-rewrite flows such as /retry, /undo, and /compress.
The delete + reinsert sequence must commit as one transaction so a
mid-rewrite failure does not leave SQLite with a partial transcript.
DESTRUCTIVE by default: every row for the session is DELETEd (and drops
out of the FTS index). For compaction that must preserve the
pre-compaction transcript under the same id, use
:meth:`archive_and_compact` instead.
Pass ``active_only=True`` to replace ONLY the live (``active = 1``) rows,
leaving soft-archived rows (``active = 0`` — e.g. the ``compacted = 1``
turns that :meth:`archive_and_compact` keeps on disk for #38763
durability, or rewind/undo rows) untouched. Callers that share a session
id with an agent already running in-place compaction must use this so a
full-history rewrite doesn't wipe the rows the agent deliberately
archived. ``message_count``/``tool_call_count`` then track the live set,
matching :meth:`archive_and_compact`.
"""
active_clause = " AND active = 1" if active_only else ""
def _do(conn):
session = conn.execute(
"SELECT ended_at, end_reason FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
if (
session is not None
and session["ended_at"] is not None
and session["end_reason"] == "compression"
):
raise CompressionSessionClosedError(session_id)
conn.execute(
f"DELETE FROM messages WHERE session_id = ?{active_clause}",
(session_id,),
)
conn.execute(
"UPDATE sessions SET message_count = 0, tool_call_count = 0 WHERE id = ?",
(session_id,),
)
total_messages, total_tool_calls = self._insert_message_rows(
conn, session_id, messages
)
conn.execute(
"UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?",
(total_messages, total_tool_calls, session_id),
)
self._execute_write(_do)
def has_archived_messages(self, session_id: str) -> bool:
"""Return True if the session has any soft-archived (``active = 0``) rows.
Used by callers (e.g. the ACP adapter's ``_persist``) that must decide
whether a full-history :meth:`replace_messages` would destroy durable
compaction-archived turns. Cheap existence probe — does not load rows.
"""
with self._lock:
cursor = self._conn.execute(
"SELECT 1 FROM messages WHERE session_id = ? AND active = 0 LIMIT 1",
(session_id,),
)
return cursor.fetchone() is not None
def archive_and_compact(
self, session_id: str, compacted_messages: List[Dict[str, Any]]
) -> int:
"""Non-destructive in-place compaction for a single durable session id.
Soft-archives every currently-active message (``active = 0``) and
inserts *compacted_messages* as fresh active rows — atomically, in one
write transaction. The conversation keeps ONE session id for life
(#38763) WITHOUT destroying history:
- The live-context load (:meth:`get_messages_as_conversation`,
:meth:`get_messages`) filters ``active = 1`` by default, so the model
reloads ONLY the compacted set.
- The archived pre-compaction turns stay on disk (active=0) and stay
DISCOVERABLE: they are marked compacted=1, and search_messages()
includes compacted=1 rows by default — so session_search still finds
them, unlike rewind/undo rows (active=0, compacted=0) which stay
hidden. They remain in the FTS index (the messages_fts* triggers
index on INSERT / drop on DELETE and don't key on active/compacted;
flipping to active=0 is a content-preserving UPDATE) and are
recoverable via get_messages(..., include_inactive=True).
This is the durability-preserving alternative to :meth:`replace_messages`
for compaction. ``message_count`` is set to the ACTIVE (compacted) count,
matching what the live load returns. Returns the new active count.
"""
def _do(conn):
# Soft-archive the live turns: active=0 hides them from the live
# context load, compacted=1 marks them as "summarized away" (vs
# rewind/undo's active=0+compacted=0, which means "user took it
# back"). search_messages includes compacted=1 rows by default so
# the pre-compaction transcript stays discoverable; live-context
# loads (active=1 only) still exclude them.
conn.execute(
"UPDATE messages SET active = 0, compacted = 1 "
"WHERE session_id = ? AND active = 1",
(session_id,),
)
inserted, tool_calls_total = self._insert_message_rows(
conn, session_id, compacted_messages
)
# message_count / tool_call_count reflect the LIVE (active) set —
# the archived rows are still on disk but not part of the live count.
conn.execute(
"UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?",
(inserted, tool_calls_total, session_id),
)
return inserted
return self._execute_write(_do)
def set_latest_user_api_content(
self, session_id: str, content: Any, api_content: str
) -> int:
"""Backfill the ``api_content`` sidecar onto the newest ACTIVE user row.
In-place preflight compaction (:meth:`archive_and_compact`) inserts the
current turn's user row BEFORE the turn prologue composes the
prefetch/plugin sidecar, and the subsequent crash persist identity-skips
every compacted dict — without this backfill the stamped sidecar would
never land in the DB and any reload would replay clean content,
re-introducing the prompt-cache divergence the sidecar exists to close.
The ``content`` match is a defensive guard: if the newest active user
row is not the message the caller stamped (racing rewrite, unexpected
tail shape), nothing is written. Returns the number of rows updated
(0 or 1).
"""
encoded = self._encode_content(content)
def _do(conn):
cursor = conn.execute(
"UPDATE messages SET api_content = ? WHERE id = ("
"SELECT id FROM messages "
"WHERE session_id = ? AND role = 'user' AND active = 1 "
"ORDER BY id DESC LIMIT 1"
") AND content IS ?",
(_scrub_surrogates(api_content), session_id, encoded),
)
return cursor.rowcount
return self._execute_write(_do)
def get_messages(
self,
session_id: str,
include_inactive: bool = False,
limit: Optional[int] = None,
offset: int = 0,
) -> List[Dict[str, Any]]:
"""Load messages for a session in insertion order.
By default only active messages are returned. Pass
``include_inactive=True`` to load soft-deleted rows (e.g. for
audit / debug views of rewound history). See
:meth:`rewind_to_message` for the soft-delete mechanic.
Ordered by AUTOINCREMENT id (true insertion order) rather than
timestamp — see c03acca50 for the WSL2 clock-regression rationale.
When ``limit`` is provided, returns at most ``limit`` messages
starting from ``offset`` (0-based, in insertion order). Enables
pagination for the API endpoint to avoid loading entire transcripts.
``offset`` alone (without ``limit``) also pages — SQLite requires a
LIMIT clause for OFFSET, so it's emitted as ``LIMIT -1`` (unbounded).
"""
active_clause = "" if include_inactive else " AND active = 1"
sql = (
"SELECT * FROM messages WHERE session_id = ?"
f"{active_clause} ORDER BY id"
)
params: list = [session_id]
if limit is not None or offset:
# SQLite's OFFSET requires LIMIT; -1 means "no limit".
sql += " LIMIT ? OFFSET ?"
params.extend([-1 if limit is None else limit, offset])
with self._read_ctx() as conn:
cursor = conn.execute(sql, params)
rows = cursor.fetchall()
result = []
for row in rows:
msg = dict(row)
if "content" in msg:
msg["content"] = self._decode_content(msg["content"])
if msg.get("tool_calls"):
try:
msg["tool_calls"] = json.loads(msg["tool_calls"])
except (json.JSONDecodeError, TypeError):
logger.warning("Failed to deserialize tool_calls in get_messages, falling back to []")
msg["tool_calls"] = []
if msg.get("display_metadata") is not None:
msg["display_metadata"] = self._decode_display_metadata(msg["display_metadata"])
result.append(msg)
return result
def get_messages_around(
self,
session_id: str,
around_message_id: int,
window: int = 5,
) -> Dict[str, Any]:
"""Load a window of messages anchored on a specific message id.
Returns a dict with:
- ``window``: up to ``window`` messages before the anchor, the anchor
itself, and up to ``window`` messages after, ordered by id ascending.
- ``messages_before``: count of messages strictly before the anchor
still in the session (== window unless we hit the start).
- ``messages_after``: count of messages strictly after the anchor
still in the session (== window unless we hit the end).
Used by ``session_search`` for both the discovery shape (anchored on the
FTS5 match) and the scroll shape (anchored on any message id). The
``messages_before`` / ``messages_after`` counts let the caller detect
session boundaries: when either is less than ``window``, the agent has
reached one end of the session.
Returns an empty window when ``around_message_id`` is not a real id in
``session_id`` — callers decide how to surface that.
"""
if window < 0:
window = 0
with self._read_ctx() as conn:
# Confirm the anchor exists in this session.
anchor_exists = conn.execute(
"SELECT 1 FROM messages WHERE id = ? AND session_id = ? LIMIT 1",
(around_message_id, session_id),
).fetchone()
if not anchor_exists:
return {"window": [], "messages_before": 0, "messages_after": 0}
# Two queries: anchor + before (DESC, take window+1), and after
# (ASC, take window). Final order is id ASC.
before_rows = conn.execute(
"SELECT * FROM messages "
"WHERE session_id = ? AND id <= ? "
"ORDER BY id DESC LIMIT ?",
(session_id, around_message_id, window + 1),
).fetchall()
after_rows = conn.execute(
"SELECT * FROM messages "
"WHERE session_id = ? AND id > ? "
"ORDER BY id ASC LIMIT ?",
(session_id, around_message_id, window),
).fetchall()
# before_rows is DESC; reverse so it's ASC, then concatenate after_rows.
rows = list(reversed(before_rows)) + list(after_rows)
result = []
for row in rows:
msg = dict(row)
if "content" in msg:
msg["content"] = self._decode_content(msg["content"])
if msg.get("tool_calls"):
try:
msg["tool_calls"] = json.loads(msg["tool_calls"])
except (json.JSONDecodeError, TypeError):
logger.warning(
"Failed to deserialize tool_calls in get_messages_around, falling back to []"
)
msg["tool_calls"] = []
if msg.get("display_metadata") is not None:
msg["display_metadata"] = self._decode_display_metadata(msg["display_metadata"])
result.append(msg)
# before_rows includes the anchor itself; subtract 1 for the count of
# messages strictly before the anchor in the returned slice.
messages_before = max(0, len(before_rows) - 1)
messages_after = len(after_rows)
return {
"window": result,
"messages_before": messages_before,
"messages_after": messages_after,
}
def resolve_resume_session_id(self, session_id: str) -> str:
"""Redirect a resume target to the descendant session that holds the messages.
Context compression ends the current session and forks a new child session
(linked via ``parent_session_id``). The flush cursor is reset, so the
child is where new messages actually land — the parent ends up with
``message_count = 0`` rows unless messages had already been flushed to
it before compression. See #15000.
This helper walks ``parent_session_id`` forward from ``session_id`` and
returns the descendant in the chain that has the **most recent** messages.
Unlike the original logic, it does NOT short-circuit when the starting
session already has messages — a descendant that was created by
compression may hold the continuation content and should be preferred
by the WebUI and gateway for ``--resume`` and session loading.
If no descendant (including the starting session) has any messages,
the original ``session_id`` is returned unchanged.
The chain is always walked via the child whose ``started_at`` is
latest; that matches the single-chain shape that compression creates.
A depth cap (32) guards against accidental loops in malformed data.
"""
if not session_id:
return session_id
# Follow the compression-continuation chain forward to the live tip
# FIRST. Auto-compression ends the current session and forks a
# continuation child, but a long-lived parent keeps its own flushed
# message rows — so the empty-head walk below never redirects it, and
# resuming the parent id reloads the pre-compression transcript while
# the turns generated *after* compression (and their responses) sit in
# the continuation. ``get_compression_tip`` is lineage-aware: it only
# follows children whose parent ended with ``end_reason='compression'``
# (created after the parent was ended), so delegation / branch children
# never hijack the resume. This is the fix for the desktop "I came back
# and the reply isn't there" report on large sessions.
try:
tip = self.get_compression_tip(session_id)
except Exception:
tip = session_id
if tip and tip != session_id:
session_id = tip
with self._lock:
current = session_id
seen = {current}
best = None # tracks the last (deepest) node with messages
for _ in range(32):
# Check if the current node has messages.
try:
row = self._conn.execute(
"SELECT 1 FROM messages WHERE session_id = ? LIMIT 1",
(current,),
).fetchone()
except Exception:
return session_id
if row is not None:
best = current
# Walk to the most-recently-started child — but skip explicit
# branch (`_branched_from`), delegate/subagent (`_delegate_from`),
# and tool children. They also carry a ``parent_session_id`` yet
# are NOT compression continuations; following them would hijack
# the resume target to an unrelated session (e.g. a subagent
# run). This mirrors the child-exclusion in ``get_compression_tip``.
try:
child_row = self._conn.execute(
"SELECT id FROM sessions "
"WHERE parent_session_id = ? "
" AND json_extract(COALESCE(model_config, '{}'), '$._branched_from') IS NULL "
" AND json_extract(COALESCE(model_config, '{}'), '$._delegate_from') IS NULL "
" AND COALESCE(source, '') != 'tool' "
"ORDER BY started_at DESC, id DESC LIMIT 1",
(current,),
).fetchone()
except Exception:
return session_id
if child_row is None:
break
child_id = child_row["id"] if hasattr(child_row, "keys") else child_row[0]
if not child_id or child_id in seen:
break
seen.add(child_id)
current = child_id
return best if best is not None else session_id
def get_messages_as_conversation(
self,
session_id: str,
include_ancestors: bool = False,
include_inactive: bool = False,
repair_alternation: bool = False,
) -> List[Dict[str, Any]]:
"""
Load messages in the OpenAI conversation format (role + content dicts).
Used by the gateway to restore conversation history.
By default only active messages are returned. Pass
``include_inactive=True`` to load soft-deleted (rewound) rows
as well. See :meth:`rewind_to_message`.
``repair_alternation=True`` runs ``repair_message_sequence`` over the
loaded list before returning it. Callers that restore a session for
LIVE REPLAY should pass it: a durable alternation violation (e.g. a
``user;user`` pair left by a turn that persisted no assistant row)
otherwise re-triggers the pre-request defensive repair on every
single request for the rest of the session's life — the repair
mutates only the per-request list, never the stored transcript.
Inspection/export consumers keep the default and see the transcript
verbatim.
"""
session_ids = [session_id]
if include_ancestors:
session_ids = self._session_lineage_root_to_tip(session_id)
active_clause = "" if include_inactive else " AND active = 1"
with self._lock:
placeholders = ",".join("?" for _ in session_ids)
rows = self._conn.execute(
"SELECT role, content, tool_call_id, tool_calls, tool_name, effect_disposition, "
"finish_reason, reasoning, reasoning_content, reasoning_details, "
"codex_reasoning_items, codex_message_items, platform_message_id, observed, timestamp, "
"api_content, display_kind, display_metadata "
f"FROM messages WHERE session_id IN ({placeholders})"
# Order by AUTOINCREMENT id (true insertion order), NOT timestamp:
# append_message stamps rows with time.time(), which is not
# monotonic (WSL2, NTP steps, VM/laptop sleep resume). A later
# row can carry an earlier timestamp than its predecessor, and
# ORDER BY timestamp would then sort an assistant tool_calls row
# after its tool response, breaking tool-call/response adjacency
# and triggering an HTTP 400 on replay. This matches get_messages
# — see c03acca50 for the original fix.
f"{active_clause} ORDER BY id",
tuple(session_ids),
).fetchall()
return self._rows_to_conversation(
rows,
session_id=session_id,
include_ancestors=include_ancestors,
repair_alternation=repair_alternation,
)
# Columns every conversation projection decodes. Shared by
# get_messages_as_conversation and get_resume_conversations so a single
# SELECT can feed both the model-fed and display views.
_CONVERSATION_ROW_COLUMNS = (
"role, content, tool_call_id, tool_calls, tool_name, effect_disposition, "
"finish_reason, reasoning, reasoning_content, reasoning_details, "
"codex_reasoning_items, codex_message_items, platform_message_id, observed, timestamp, "
"api_content, display_kind, display_metadata"
)
def _rows_to_conversation(
self,
rows,
*,
session_id: str,
include_ancestors: bool,
repair_alternation: bool,
) -> List[Dict[str, Any]]:
"""Decode fetched message rows into the OpenAI conversation format.
Extracted from get_messages_as_conversation so get_resume_conversations
can build the model-fed and display views from one SELECT. ``rows`` must
already be ordered by ``id`` (insertion order) and filtered to the
desired session set / active state by the caller.
"""
messages = []
for row in rows:
content = self._decode_content(row["content"])
if row["role"] in {"user", "assistant"} and isinstance(content, str):
content = sanitize_context(content).strip()
msg = {"role": row["role"], "content": content}
# api_content is the byte-fidelity sidecar: the exact string sent
# to the API when it differed from the clean content. Returned
# VERBATIM — no sanitize_context, no strip — because the replay
# path substitutes it for content to keep the provider prompt
# cache prefix byte-stable across turns. Cleaning it here would
# re-introduce the divergence it exists to remove.
if row["api_content"]:
msg["api_content"] = row["api_content"]
if row["display_kind"]:
msg["display_kind"] = row["display_kind"]
if row["display_metadata"]:
decoded = self._decode_display_metadata(row["display_metadata"])
if decoded is not None:
msg["display_metadata"] = decoded
if row["timestamp"]:
msg["timestamp"] = row["timestamp"]
if row["tool_call_id"]:
msg["tool_call_id"] = row["tool_call_id"]
if row["tool_name"]:
msg["tool_name"] = row["tool_name"]
if row["effect_disposition"]:
msg["effect_disposition"] = row["effect_disposition"]
if row["tool_calls"]:
try:
msg["tool_calls"] = json.loads(row["tool_calls"])
except (json.JSONDecodeError, TypeError):
logger.warning("Failed to deserialize tool_calls in conversation replay, falling back to []")
msg["tool_calls"] = []
# Surface the platform-side message id (e.g. yuanbao msg_id,
# telegram update_id) so platform-specific flows like recall
# can match by external identifier instead of having to fall
# back to content-match heuristics. Exposed as ``message_id``
# for backward compatibility with the JSONL transcript shape.
if row["platform_message_id"]:
msg["message_id"] = row["platform_message_id"]
if row["observed"]:
msg["observed"] = True
# Restore reasoning fields on assistant messages so providers
# that replay reasoning (OpenRouter, OpenAI, Nous) receive
# coherent multi-turn reasoning context.
if row["role"] == "assistant":
if row["finish_reason"]:
msg["finish_reason"] = row["finish_reason"]
if row["reasoning"]:
msg["reasoning"] = row["reasoning"]
if row["reasoning_content"] is not None:
msg["reasoning_content"] = row["reasoning_content"]
if row["reasoning_details"]:
try:
msg["reasoning_details"] = json.loads(row["reasoning_details"])
except (json.JSONDecodeError, TypeError):
logger.warning("Failed to deserialize reasoning_details, falling back to None")
msg["reasoning_details"] = None
if row["codex_reasoning_items"]:
try:
msg["codex_reasoning_items"] = json.loads(row["codex_reasoning_items"])
except (json.JSONDecodeError, TypeError):
logger.warning("Failed to deserialize codex_reasoning_items, falling back to None")
msg["codex_reasoning_items"] = None
if row["codex_message_items"]:
try:
msg["codex_message_items"] = json.loads(row["codex_message_items"])
except (json.JSONDecodeError, TypeError):
logger.warning("Failed to deserialize codex_message_items, falling back to None")
msg["codex_message_items"] = None
if include_ancestors and self._is_duplicate_replayed_user_message(messages, msg):
continue
messages.append(msg)
# DEFENSE-IN-DEPTH against background-review session pollution: a forked
# skill/memory review that (in older builds, before the _persist_disabled
# fix) shared the parent's session_id wrote its harness turn into this
# real session. The harness is a user/system message instructing the
# agent to "Review the conversation above and update the skill library /
# save to memory" under a hard tool restriction; re-loading it as live
# history makes the agent adopt the curator role and refuse the user's
# actual task. Strip any such harness message AND the curator-mode
# assistant reply immediately following it, so a polluted session
# resumes clean even if stray rows exist.
messages = _strip_background_review_harness(messages)
if repair_alternation and messages:
# Lazy import: hermes_state already depends on agent.* (see
# sanitize_context above), but keep this optional path from
# widening the import surface at module load.
from agent.agent_runtime_helpers import repair_message_sequence
repaired = repair_message_sequence(None, messages)
if repaired:
logger.info(
"Repaired %d message-alternation violation(s) while "
"restoring session %s — durable transcript kept them, "
"see repair_message_sequence",
repaired,
session_id,
)
return messages
def get_resume_conversations(
self, session_id: str
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
"""Return ``(model_history, display_history)`` for a session resume in ONE SELECT.
``session.resume`` needs two projections of the same lineage:
- ``model_history`` — the tip session's active rows, alternation-repaired
(the live-replay working conversation). Equivalent to
``get_messages_as_conversation(session_id, repair_alternation=True)``.
- ``display_history`` — the full lineage (ancestors → tip), verbatim, with
replayed-user dedup. Equivalent to
``get_messages_as_conversation(session_id, include_ancestors=True)``.
The display fetch already reads a superset of the model fetch (the tip
rows are part of the lineage), so serving both from one lineage SELECT
halves the resume's DB work versus two separate calls, with byte-identical
output (see test_get_resume_conversations_matches_separate_reads).
"""
session_ids = self._session_lineage_root_to_tip(session_id)
with self._lock:
placeholders = ",".join("?" for _ in session_ids)
rows = self._conn.execute(
f"SELECT session_id, {self._CONVERSATION_ROW_COLUMNS} "
f"FROM messages WHERE session_id IN ({placeholders}) AND active = 1 "
# ORDER BY id (insertion order) — see get_messages_as_conversation
# for why timestamp ordering is unsafe.
"ORDER BY id",
tuple(session_ids),
).fetchall()
# Tip rows are exactly the model-fed set (get_messages_as_conversation
# with session_ids=[session_id]); filtering the lineage fetch preserves
# their relative id order.
tip_rows = [r for r in rows if r["session_id"] == session_id]
model_history = self._rows_to_conversation(
tip_rows,
session_id=session_id,
include_ancestors=False,
repair_alternation=True,
)
display_history = self._rows_to_conversation(
rows,
session_id=session_id,
include_ancestors=True,
repair_alternation=False,
)
return model_history, display_history
def get_ancestor_display_prefix(self, session_id: str) -> List[Dict[str, Any]]:
"""Return the ancestor-only display messages for a session lineage.
These are messages from parent/grandparent sessions (compression
ancestors) that appear in the display transcript but NOT in the
tip session's model-fed history. Used by ``session.resume`` to
build the ``display_history_prefix`` that ``_live_session_payload``
prepends to the live model history.
Previously the prefix was calculated as
``display_history[:len(display) - len(raw)]``, but that overcounts
when ``repair_message_sequence`` removes messages from the MIDDLE
of the tip history (e.g. verification candidates collapsed by the
consecutive-assistant merge) — the length difference includes both
ancestor messages AND repair-removed tip messages, but the slice
only captures the first N display messages (which are tip messages
when there are no ancestors), causing duplication. This method
returns ONLY the genuine ancestor messages, identified by
``session_id != tip_session_id``. (#65919)
"""
session_ids = self._session_lineage_root_to_tip(session_id)
if len(session_ids) <= 1:
return []
with self._lock:
placeholders = ",".join("?" for _ in session_ids)
rows = self._conn.execute(
f"SELECT session_id, {self._CONVERSATION_ROW_COLUMNS} "
f"FROM messages WHERE session_id IN ({placeholders}) AND active = 1 "
"ORDER BY id",
tuple(session_ids),
).fetchall()
ancestor_rows = [r for r in rows if r["session_id"] != session_id]
if not ancestor_rows:
return []
return self._rows_to_conversation(
ancestor_rows,
session_id=session_id,
include_ancestors=True,
repair_alternation=False,
)
def get_conversation_root(self, session_id: str) -> str:
"""Return the ROOT id of *session_id*'s lineage chain.
The root is the stable "conversation id": context compression
rotates ``session_id`` to a new segment linked via
``parent_session_id``, and delegate subagents hang off their
parent the same way. Walking to the root gives every segment of
one user-facing conversation (and its delegation tree) a single
identifier — used for Nous Portal ``conversation=`` usage tagging.
Returns *session_id* unchanged when it has no recorded parent.
"""
chain = self._session_lineage_root_to_tip(session_id)
return (chain[0] if chain and chain[0] else session_id)
def _session_lineage_root_to_tip(self, session_id: str) -> List[str]:
if not session_id:
return [session_id]
chain = []
current = session_id
seen = set()
with self._lock:
for _ in range(100):
if not current or current in seen:
break
seen.add(current)
chain.append(current)
row = self._conn.execute(
"SELECT parent_session_id FROM sessions WHERE id = ?",
(current,),
).fetchone()
if row is None:
break
current = row["parent_session_id"] if hasattr(row, "keys") else row[0]
return list(reversed(chain)) or [session_id]
@staticmethod
def _is_duplicate_replayed_user_message(messages: List[Dict[str, Any]], msg: Dict[str, Any]) -> bool:
if msg.get("role") != "user":
return False
content = msg.get("content")
if not isinstance(content, str) or not content:
return False
for prev in reversed(messages):
if prev.get("role") == "user" and prev.get("content") == content:
return True
if prev.get("role") == "assistant" and (prev.get("content") or prev.get("tool_calls")):
return False
return False
# =========================================================================
# Rewind (soft-delete) — see /rewind slash command + issue #21910
# =========================================================================
def rewind_to_message(
self, session_id: str, target_message_id: int
) -> Dict[str, Any]:
"""Soft-delete all messages with id >= ``target_message_id`` in *session_id*.
The target message itself becomes inactive as well so the caller
can pre-fill it as the next user prompt without it appearing
twice in the replayed transcript. Rewound rows are kept on
disk with ``active=0`` for audit / forensic inspection — use
:meth:`get_messages` with ``include_inactive=True`` to see them.
Returns a dict::
{
"rewound_count": int, # number of rows newly flipped to active=0
"target_message": dict, # full row dict of the target
"new_head_id": int|None # id of the last still-active row, or None
}
Raises ``ValueError`` if the target message does not exist in
*session_id* or if its role is not ``"user"``.
Always increments ``sessions.rewind_count`` — even when the
target is already inactive — so the counter accurately reflects
the number of rewind operations performed against the session.
Idempotent on the ``active`` flag: re-rewinding past the same
target is a no-op on row state but still bumps the counter.
"""
# 1) Validate target up-front (read-only, outside the write txn).
with self._lock:
row = self._conn.execute(
"SELECT * FROM messages WHERE id = ? AND session_id = ?",
(target_message_id, session_id),
).fetchone()
if row is None:
raise ValueError(
f"message {target_message_id} not found in session {session_id}"
)
target_row = dict(row)
if target_row.get("role") != "user":
raise ValueError(
f"rewind target must be a 'user' message (got role="
f"{target_row.get('role')!r}, id={target_message_id})"
)
# Decode content for callers (prefill the prompt buffer).
target_row["content"] = self._decode_content(target_row.get("content"))
rewound: List[int] = []
def _do(conn):
cursor = conn.execute(
"SELECT id FROM messages "
"WHERE session_id = ? AND id >= ? AND active = 1",
(session_id, target_message_id),
)
ids = [r[0] for r in cursor.fetchall()]
if ids:
placeholders = ",".join("?" for _ in ids)
conn.execute(
f"UPDATE messages SET active = 0 WHERE id IN ({placeholders})",
ids,
)
conn.execute(
"UPDATE sessions SET rewind_count = COALESCE(rewind_count, 0) + 1 "
"WHERE id = ?",
(session_id,),
)
return ids
rewound = self._execute_write(_do)
# 2) Compute new head id (largest still-active row id in session).
with self._lock:
head_row = self._conn.execute(
"SELECT MAX(id) FROM messages WHERE session_id = ? AND active = 1",
(session_id,),
).fetchone()
new_head_id = head_row[0] if head_row and head_row[0] is not None else None
return {
"rewound_count": len(rewound),
"target_message": target_row,
"new_head_id": new_head_id,
}
def restore_rewound(self, session_id: str, since_message_id: int) -> int:
"""Mark inactive messages with id >= *since_message_id* active again.
Returns the number of rows flipped back to ``active=1``.
Intended for undo-of-rewind and test cleanup; not wired to a
slash command in v1.
"""
def _do(conn):
cursor = conn.execute(
"SELECT id FROM messages "
"WHERE session_id = ? AND id >= ? AND active = 0",
(session_id, since_message_id),
)
ids = [r[0] for r in cursor.fetchall()]
if ids:
placeholders = ",".join("?" for _ in ids)
conn.execute(
f"UPDATE messages SET active = 1 WHERE id IN ({placeholders})",
ids,
)
return len(ids)
return self._execute_write(_do)
# =========================================================================
# Search
# =========================================================================
def search_sessions(
self,
source: str = None,
limit: int = 20,
offset: int = 0,
workspace_key: str = None,
) -> List[Dict[str, Any]]:
"""List sessions, optionally filtered by source.
Returns rows enriched with a computed ``last_active`` column (latest
message timestamp for the session, falling back to ``started_at``),
ordered by most-recently-used first.
Pass ``workspace_key`` to scope rows to one workspace — matching
:func:`workspace_key` semantics (git repo root, else cwd). Used by
``hermes -c``/``--resume`` so the "last" session is the last one in
the *current* workspace, not the global MRU.
"""
select_with_last_active = (
"SELECT s.*, COALESCE(m.last_active, s.started_at) AS last_active "
"FROM sessions s "
"LEFT JOIN ("
"SELECT session_id, MAX(timestamp) AS last_active "
"FROM messages GROUP BY session_id"
") m ON m.session_id = s.id "
)
where_clauses = []
params: list = []
if source:
where_clauses.append("s.source = ?")
params.append(source)
if workspace_key:
ws_clause, ws_params = _workspace_key_clause(workspace_key)
where_clauses.append(ws_clause)
params.extend(ws_params)
where_sql = f" WHERE {' AND '.join(where_clauses)}" if where_clauses else ""
params.extend([limit, offset])
with self._lock:
cursor = self._conn.execute(
f"{select_with_last_active}"
f"{where_sql} "
"ORDER BY last_active DESC, s.started_at DESC, s.id DESC LIMIT ? OFFSET ?",
params,
)
return [dict(row) for row in cursor.fetchall()]
# =========================================================================
# Utility
# =========================================================================
def session_count(
self,
source: str = None,
sources: List[str] = None,
cwd_prefix: str = None,
min_message_count: int = 0,
include_archived: bool = False,
archived_only: bool = False,
exclude_children: bool = False,
exclude_sources: List[str] = None,
) -> int:
"""Count sessions, optionally filtered by source.
Pass ``exclude_children=True`` to count only the conversations that
``list_sessions_rich`` surfaces (root + branch sessions), hiding
sub-agent runs and compression continuations. Use it whenever the count
is paired with a ``list_sessions_rich`` page (e.g. sidebar "load more"
totals) so the total matches the number of listable rows — otherwise the
raw row count is inflated by children and "load more" never settles.
Pass ``exclude_sources`` to drop whole source classes from the count
(e.g. ``["cron"]`` so the recents "load more" total matches a
cron-excluded ``list_sessions_rich`` page and doesn't keep "load more"
stuck on for buried scheduler sessions).
"""
where_clauses = []
params = []
if exclude_children:
# Mirror list_sessions_rich's child-exclusion clause exactly so the
# count lines up with the rows: roots (no parent) plus branch
# children (parent ended with end_reason='branched').
where_clauses.append(_LISTABLE_CHILD_SQL)
where_clauses.append(f"{_delegate_from_json('s.model_config')} IS NULL")
include_sources = [source] if source else list(sources or [])
if include_sources:
placeholders = ",".join("?" for _ in include_sources)
where_clauses.append(f"s.source IN ({placeholders})")
params.extend(include_sources)
if exclude_sources:
placeholders = ",".join("?" for _ in exclude_sources)
where_clauses.append(f"s.source NOT IN ({placeholders})")
params.extend(exclude_sources)
if cwd_prefix:
clause, clause_params = _cwd_prefix_clause(cwd_prefix)
where_clauses.append(clause)
params.extend(clause_params)
if min_message_count > 0:
where_clauses.append("s.message_count >= ?")
params.append(min_message_count)
if archived_only:
where_clauses.append("s.archived = 1")
elif not include_archived:
where_clauses.append("s.archived = 0")
where_sql = f" WHERE {' AND '.join(where_clauses)}" if where_clauses else ""
with self._lock:
cursor = self._conn.execute(f"SELECT COUNT(*) FROM sessions s{where_sql}", params)
return cursor.fetchone()[0]
def session_count_by_source(
self,
*,
include_archived: bool = False,
archived_only: bool = False,
exclude_children: bool = False,
) -> Dict[str, int]:
"""Return a ``{source: count}`` dict via a single ``GROUP BY`` query.
Replaces the O(N) ``list_sessions_rich`` histogram loop with an
aggregate query. When ``exclude_children`` is False the query uses
``idx_sessions_source``; when True, the child-exclusion predicates
require a full table scan (same as ``session_count`` and
``list_sessions_rich``).
``exclude_children=True`` mirrors ``list_sessions_rich`` visibility
(roots + branch sessions, excluding sub-agent runs, delegates, and
compression continuations) so the source counts match what the
Sessions page actually lists.
"""
where_clauses = []
params: list = []
if exclude_children:
where_clauses.append(_LISTABLE_CHILD_SQL)
where_clauses.append(f"{_delegate_from_json('s.model_config')} IS NULL")
if archived_only:
where_clauses.append("s.archived = 1")
elif not include_archived:
where_clauses.append("s.archived = 0")
where_sql = f" WHERE {' AND '.join(where_clauses)}" if where_clauses else ""
with self._lock:
if self._conn is None:
raise RuntimeError("SessionDB connection is closed")
rows = self._conn.execute(
"SELECT COALESCE(NULLIF(s.source, ''), 'cli') AS source, COUNT(*) AS count "
f"FROM sessions s{where_sql} "
"GROUP BY COALESCE(NULLIF(s.source, ''), 'cli') "
"ORDER BY count DESC",
params,
).fetchall()
return {str(row["source"]): int(row["count"] or 0) for row in rows}
def message_count(self, session_id: str = None) -> int:
"""Count messages, optionally for a specific session."""
with self._lock:
if session_id:
cursor = self._conn.execute(
"SELECT COUNT(*) FROM messages WHERE session_id = ?", (session_id,)
)
else:
cursor = self._conn.execute("SELECT COUNT(*) FROM messages")
return cursor.fetchone()[0]
def has_platform_message_id(
self, session_id: str, platform_message_id: str
) -> bool:
"""Check if a message with the given platform_message_id exists.
Uses the idx_messages_platform_msg_id partial index for efficient
lookup. Used by the gateway's transient-failure dedupe guard (#47237)
to skip re-persisting a user message that was already saved on a
prior retry of the same inbound platform message.
"""
with self._lock:
cursor = self._conn.execute(
"SELECT 1 FROM messages "
"WHERE session_id = ? AND platform_message_id = ? LIMIT 1",
(session_id, platform_message_id),
)
return cursor.fetchone() is not None
# =========================================================================
# Export and cleanup
# =========================================================================
def _is_branch_child_row(self, session: Dict[str, Any]) -> bool:
raw = session.get("model_config")
if not raw:
return False
try:
cfg = json.loads(raw) if isinstance(raw, str) else raw
except (TypeError, json.JSONDecodeError):
return False
return isinstance(cfg, dict) and cfg.get("_branched_from") is not None
def _is_compression_child_row(self, child: Dict[str, Any]) -> bool:
parent_id = child.get("parent_session_id")
if not parent_id or self._is_branch_child_row(child):
return False
parent = self.get_session(parent_id)
return bool(parent and parent.get("end_reason") == "compression")
def get_compression_lineage(self, session_id: str) -> List[str]:
"""Return compression ancestors through tip in chronological order."""
session = self.get_session(session_id)
if not session or self._is_branch_child_row(session):
return [session_id] if session else []
root = session
ancestors = {root["id"]}
while self._is_compression_child_row(root):
parent = self.get_session(root["parent_session_id"])
if not parent or parent["id"] in ancestors:
break
root = parent
ancestors.add(root["id"])
lineage = [root["id"]]
seen = {root["id"]}
current = root
while current.get("end_reason") == "compression":
with self._lock:
rows = self._conn.execute(
"""
SELECT * FROM sessions
WHERE parent_session_id = ?
ORDER BY started_at ASC
""",
(current["id"],),
).fetchall()
next_child = None
for row in rows:
candidate = dict(row)
if not self._is_branch_child_row(candidate):
next_child = candidate
break
if not next_child or next_child["id"] in seen:
break
lineage.append(next_child["id"])
seen.add(next_child["id"])
current = next_child
if current["id"] == session_id:
# Continue to include later compression tips only when the
# requested session itself was compacted.
continue
return lineage if session_id in lineage else [session_id]
def clear_messages(self, session_id: str) -> None:
"""Delete all messages for a session and reset its counters."""
def _do(conn):
conn.execute(
"DELETE FROM messages WHERE session_id = ?", (session_id,)
)
conn.execute(
"UPDATE sessions SET message_count = 0, tool_call_count = 0 WHERE id = ?",
(session_id,),
)
self._execute_write(_do)
@staticmethod
def _remove_session_files(sessions_dir: Optional[Path], session_id: str) -> None:
"""Remove on-disk transcript files for a session.
Cleans up ``{session_id}.json``, ``{session_id}.jsonl``, and any
``request_dump_{session_id}_*.json`` files left by the gateway.
Silently skips files that don't exist and swallows OSError so a
filesystem hiccup never blocks a DB operation.
"""
if sessions_dir is None:
return
for suffix in (".json", ".jsonl"):
p = sessions_dir / f"{session_id}{suffix}"
try:
p.unlink(missing_ok=True)
except OSError:
pass
# request_dump files use session_id as a prefix component
try:
for p in sessions_dir.glob(f"request_dump_{session_id}_*.json"):
try:
p.unlink(missing_ok=True)
except OSError:
pass
except OSError:
pass
def get_session_delete_targets(self, session_id: str) -> List[str]:
"""Return every session row that :meth:`delete_session` would remove.
The requested session is first, followed by its recursively discovered
delegate/subagent children. Branch and compression children are not
included because deletion preserves them by orphaning their parent
reference.
"""
with self._lock:
exists = self._conn.execute(
"SELECT 1 FROM sessions WHERE id = ? LIMIT 1", (session_id,)
).fetchone()
if not exists:
return []
delegate_ids = _collect_delegate_child_ids(self._conn, [session_id])
return [session_id, *sorted(delegate_ids)]
def delete_session(
self,
session_id: str,
sessions_dir: Optional[Path] = None,
expected_delete_ids: Optional[List[str]] = None,
) -> bool:
"""Delete a session and all its messages.
Delegate subagent children (``model_config._delegate_from``) are
cascade-deleted with the parent so they never resurface in session
pickers as orphaned rows. Branch / compression children are orphaned
(``parent_session_id → NULL``) so they remain accessible independently.
When *sessions_dir* is provided, also removes on-disk transcript
files (``.json`` / ``.jsonl`` / ``request_dump_*``) for every deleted
session. When *expected_delete_ids* is provided, deletion proceeds only
if the parent plus delegate cascade still matches that exact set. This
lets export-before-delete callers fail closed if a new delegate appears
after they materialize their archive. The delegate tree is re-walked
inside the write transaction on purpose (TOCTOU guard); the cost is
accepted for correctness. Returns True if the session was found and
deleted.
"""
removed_delegate_ids: List[str] = []
expected_ids = (
set(expected_delete_ids) if expected_delete_ids is not None else None
)
def _do(conn):
cursor = conn.execute(
"SELECT COUNT(*) FROM sessions WHERE id = ?", (session_id,)
)
if cursor.fetchone()[0] == 0:
return False
if expected_ids is not None:
actual_ids = {
session_id,
*_collect_delegate_child_ids(conn, [session_id]),
}
if actual_ids != expected_ids:
return False
removed_delegate_ids.extend(_delete_delegate_children(conn, [session_id]))
# Orphan remaining child sessions (branches, etc.) so FK is satisfied.
conn.execute(
"UPDATE sessions SET parent_session_id = NULL "
"WHERE parent_session_id = ?",
(session_id,),
)
conn.execute("DELETE FROM messages WHERE session_id = ?", (session_id,))
conn.execute("DELETE FROM sessions WHERE id = ?", (session_id,))
return True
deleted = self._execute_write(_do)
if deleted:
for delegate_id in removed_delegate_ids:
self._remove_session_files(sessions_dir, delegate_id)
self._remove_session_files(sessions_dir, session_id)
return bool(deleted)
def delete_session_if_empty(
self,
session_id: str,
sessions_dir: Optional[Path] = None,
) -> bool:
"""Delete *session_id* only when it never gained resumable content.
A session is considered empty when it has no messages and no
user-assigned title. Used by CLI exit / session-rotation paths so
immediately-started-and-quit sessions don't pile up in ``/resume``
and ``hermes sessions list`` output. (Pattern ported from
google-gemini/gemini-cli#27770.)
The emptiness check and delete run in one transaction, so a message
flushed concurrently by another writer can't be lost. Sessions with
children (delegate subagent runs) are preserved — a parent that
spawned work is not "empty" even if its own transcript never
flushed. Returns True if the session was deleted.
"""
def _do(conn):
cursor = conn.execute(
"""
DELETE FROM sessions
WHERE id = ?
AND title IS NULL
AND NOT EXISTS (
SELECT 1 FROM messages WHERE messages.session_id = sessions.id
)
AND NOT EXISTS (
SELECT 1 FROM sessions child
WHERE child.parent_session_id = sessions.id
)
""",
(session_id,),
)
return cursor.rowcount > 0
deleted = self._execute_write(_do)
if deleted:
self._remove_session_files(sessions_dir, session_id)
return bool(deleted)
def delete_sessions(
self,
session_ids: List[str],
sessions_dir: Optional[Path] = None,
) -> int:
"""Delete every session in *session_ids* in a single transaction.
Backs the dashboard's bulk-select-then-delete flow on the
sessions page (``POST /api/sessions/bulk-delete``). Mirrors the
single-session :meth:`delete_session` contract per row:
* Unknown IDs are silently skipped (no 404) — selection state
in the UI can race against another tab's delete, and we'd
rather succeed-on-the-rest than fail-the-whole-batch.
* Delegate subagent children (``model_config._delegate_from``) are
cascade-deleted with their parent; branch children are orphaned
(``parent_session_id → NULL``) so they stay accessible.
* Messages and the session row both go in one
``_execute_write`` call so a partial failure can't leave the
DB in a "messages gone but session row still there" state.
* On-disk transcript / ``request_dump_*`` files are cleaned up
outside the DB transaction when *sessions_dir* is provided,
matching :meth:`prune_sessions` and
:meth:`delete_empty_sessions`.
Returns the count of sessions that actually existed and were
deleted (may be less than ``len(session_ids)`` if some IDs were
already gone).
"""
if not session_ids:
return 0
# Dedup + drop any non-string entries up-front. Avoids
# double-counting in the WHERE-IN list and protects against
# callers that pass a list with stray ``None`` values.
unique_ids = list({sid for sid in session_ids if isinstance(sid, str) and sid})
if not unique_ids:
return 0
removed_ids: list[str] = []
removed_delegate_ids: list[str] = []
def _do(conn):
placeholders = ",".join("?" * len(unique_ids))
# First, filter to IDs that actually exist — we want to
# return the real deleted count, not the input length.
cursor = conn.execute(
f"SELECT id FROM sessions WHERE id IN ({placeholders})",
unique_ids,
)
existing = [row["id"] for row in cursor.fetchall()]
if not existing:
return 0
existing_placeholders = ",".join("?" * len(existing))
removed_delegate_ids.extend(_delete_delegate_children(conn, existing))
# Orphan remaining children whose parent is in the kill list so the
# FK constraint stays satisfied. Pin children whose parent
# is itself in the kill list rather than NULL-ing parents
# of survivors — the IN list on ``parent_session_id`` does
# exactly this.
conn.execute(
f"UPDATE sessions SET parent_session_id = NULL "
f"WHERE parent_session_id IN ({existing_placeholders})",
existing,
)
conn.execute(
f"DELETE FROM messages WHERE session_id IN ({existing_placeholders})",
existing,
)
conn.execute(
f"DELETE FROM sessions WHERE id IN ({existing_placeholders})",
existing,
)
removed_ids.extend(existing)
return len(existing)
count = self._execute_write(_do)
for sid in removed_delegate_ids:
self._remove_session_files(sessions_dir, sid)
for sid in removed_ids:
self._remove_session_files(sessions_dir, sid)
return count
def count_empty_sessions(self) -> int:
"""Return the count of empty, non-active, non-archived sessions.
"Empty" = ``message_count = 0`` AND the session has ended
(``ended_at IS NOT NULL``) AND is not archived. The ``ended_at``
guard matches the safety contract used by :meth:`prune_sessions`:
only ended sessions are candidates for bulk deletion, so a freshly
spawned session whose first message hasn't landed yet — or one
held open by the live agent — is never sniped out from under
the runtime.
Backs the ``GET /api/sessions/empty/count`` endpoint that lets the
web dashboard hide its "Delete empty" button when there's nothing
to clean up, and pre-populate the confirm dialog with the actual
count.
"""
with self._lock:
cursor = self._conn.execute(
"SELECT COUNT(*) FROM sessions "
"WHERE message_count = 0 "
"AND ended_at IS NOT NULL "
"AND archived = 0"
)
return cursor.fetchone()[0]
def delete_empty_sessions(
self,
sessions_dir: Optional[Path] = None,
) -> int:
"""Delete every empty, ended, non-archived session.
Mirrors :meth:`prune_sessions`' transactional shape:
* Selects candidate IDs first (``message_count = 0`` AND
``ended_at IS NOT NULL`` AND ``archived = 0``) so we never
touch a live session or one the user deliberately archived.
* Orphans any child whose parent is in the kill list — children
of an empty parent are kept and re-parented to ``NULL`` rather
than cascade-deleted, matching ``delete_session`` /
``prune_sessions`` semantics so branch/subagent transcripts
survive an inadvertent parent cleanup.
* Deletes the rows in a single ``_execute_write`` callback so
the operation is atomic — a partial failure (e.g. SIGKILL
mid-loop) doesn't leave the DB in a "messages-deleted but
session-row-still-there" half-state.
* Cleans up on-disk transcript files (``.json`` / ``.jsonl`` /
``request_dump_*``) outside the DB transaction when
``sessions_dir`` is provided. Empty sessions don't typically
have transcript files, but the gateway can leave a stub
``request_dump_*`` if it crashed before the first reply —
so we still sweep, matching ``prune_sessions``.
Returns the number of sessions deleted.
"""
removed_ids: list[str] = []
def _do(conn):
cursor = conn.execute(
"SELECT id FROM sessions "
"WHERE message_count = 0 "
"AND ended_at IS NOT NULL "
"AND archived = 0"
)
session_ids = {row["id"] for row in cursor.fetchall()}
if not session_ids:
return 0
placeholders = ",".join("?" * len(session_ids))
conn.execute(
f"UPDATE sessions SET parent_session_id = NULL "
f"WHERE parent_session_id IN ({placeholders})",
list(session_ids),
)
for sid in session_ids:
# DELETE FROM messages is paranoia — by construction
# these rows have ``message_count = 0`` — but if a
# bookkeeping bug ever lets the counter drift below the
# real row count, we still leave a clean FK state.
conn.execute(
"DELETE FROM messages WHERE session_id = ?", (sid,)
)
conn.execute("DELETE FROM sessions WHERE id = ?", (sid,))
removed_ids.append(sid)
return len(session_ids)
count = self._execute_write(_do)
for sid in removed_ids:
self._remove_session_files(sessions_dir, sid)
return count
@staticmethod
def _prune_filter_where(
*,
last_active_before: Optional[float] = None,
last_active_after: Optional[float] = None,
started_before: Optional[float] = None,
started_after: Optional[float] = None,
source: Optional[str] = None,
title_like: Optional[str] = None,
end_reason: Optional[str] = None,
cwd_prefix: Optional[str] = None,
min_messages: Optional[int] = None,
max_messages: Optional[int] = None,
archived: Optional[bool] = None,
model_like: Optional[str] = None,
provider: Optional[str] = None,
user_id: Optional[str] = None,
chat_id: Optional[str] = None,
chat_type: Optional[str] = None,
branch_like: Optional[str] = None,
min_tokens: Optional[int] = None,
max_tokens: Optional[int] = None,
min_cost: Optional[float] = None,
max_cost: Optional[float] = None,
min_tool_calls: Optional[int] = None,
max_tool_calls: Optional[int] = None,
) -> Tuple[str, list]:
"""Build the shared WHERE clause for bulk prune/archive selection.
All filters AND together. Only ended sessions are ever candidates
(``ended_at IS NOT NULL``) so a live session is never selected.
``archived`` is a tri-state: ``None`` = both, ``True`` = only
archived rows, ``False`` = only unarchived rows.
String matching conventions: ``model_like`` / ``branch_like`` /
``title_like`` are case-insensitive substring matches (model slugs
and branch names vary in prefix format); ``provider`` / ``user_id``
/ ``chat_id`` / ``chat_type`` / ``source`` / ``end_reason`` are
exact (case-insensitive for provider). Token bounds apply to
``input_tokens + output_tokens``; cost bounds apply to
``COALESCE(actual_cost_usd, estimated_cost_usd)``.
The clause references the ``s`` table alias — callers must select
``FROM sessions s``.
"""
clauses = ["s.ended_at IS NOT NULL"]
params: list = []
if last_active_before is not None:
clauses.append(
"""COALESCE(
(SELECT MAX(m.timestamp) FROM messages m
WHERE m.session_id = s.id),
s.started_at
) < ?"""
)
params.append(last_active_before)
if last_active_after is not None:
clauses.append(
"""COALESCE(
(SELECT MAX(m.timestamp) FROM messages m
WHERE m.session_id = s.id),
s.started_at
) >= ?"""
)
params.append(last_active_after)
if started_before is not None:
clauses.append("s.started_at < ?")
params.append(started_before)
if started_after is not None:
clauses.append("s.started_at >= ?")
params.append(started_after)
if source:
clauses.append("s.source = ?")
params.append(source)
if title_like:
clauses.append("LOWER(COALESCE(s.title, '')) LIKE ?")
params.append(f"%{title_like.lower()}%")
if end_reason:
clauses.append("s.end_reason = ?")
params.append(end_reason)
if cwd_prefix:
clause, clause_params = _cwd_prefix_clause(cwd_prefix)
clauses.append(clause)
params.extend(clause_params)
if min_messages is not None:
clauses.append("s.message_count >= ?")
params.append(min_messages)
if max_messages is not None:
clauses.append("s.message_count <= ?")
params.append(max_messages)
if model_like:
clauses.append("LOWER(COALESCE(s.model, '')) LIKE ?")
params.append(f"%{model_like.lower()}%")
if provider:
clauses.append("LOWER(COALESCE(s.billing_provider, '')) = ?")
params.append(provider.lower())
if user_id:
clauses.append("s.user_id = ?")
params.append(user_id)
if chat_id:
clauses.append("s.chat_id = ?")
params.append(chat_id)
if chat_type:
clauses.append("s.chat_type = ?")
params.append(chat_type)
if branch_like:
clauses.append("LOWER(COALESCE(s.git_branch, '')) LIKE ?")
params.append(f"%{branch_like.lower()}%")
if min_tokens is not None:
clauses.append(
"(COALESCE(s.input_tokens, 0) + COALESCE(s.output_tokens, 0)) >= ?"
)
params.append(min_tokens)
if max_tokens is not None:
clauses.append(
"(COALESCE(s.input_tokens, 0) + COALESCE(s.output_tokens, 0)) <= ?"
)
params.append(max_tokens)
if min_cost is not None:
clauses.append(
"COALESCE(s.actual_cost_usd, s.estimated_cost_usd, 0) >= ?"
)
params.append(min_cost)
if max_cost is not None:
clauses.append(
"COALESCE(s.actual_cost_usd, s.estimated_cost_usd, 0) <= ?"
)
params.append(max_cost)
if min_tool_calls is not None:
clauses.append("COALESCE(s.tool_call_count, 0) >= ?")
params.append(min_tool_calls)
if max_tool_calls is not None:
clauses.append("COALESCE(s.tool_call_count, 0) <= ?")
params.append(max_tool_calls)
if archived is True:
clauses.append("s.archived = 1")
elif archived is False:
clauses.append("s.archived = 0")
return " AND ".join(clauses), params
def list_prune_candidates(
self,
older_than_days: Optional[float] = None,
source: str = None,
**filters,
) -> List[Dict[str, Any]]:
"""Return the sessions a matching :meth:`prune_sessions` /
:meth:`archive_sessions` call would touch, without modifying anything.
Backs ``--dry-run`` and pre-confirmation counts. Accepts the same
keyword filters as :meth:`_prune_filter_where` (unknown names raise
``TypeError`` there). Rows are ordered oldest-first and carry
``id, source, title, model, started_at, last_active, ended_at,
message_count, archived``. ``older_than_days`` is an inactivity
threshold: it uses the latest message timestamp, falling back to
``started_at`` for sessions without messages.
"""
if (
filters.get("last_active_before") is None
and filters.get("started_before") is None
and older_than_days is not None
):
filters["last_active_before"] = time.time() - (
older_than_days * 86400
)
where, params = self._prune_filter_where(source=source, **filters)
with self._lock:
cursor = self._conn.execute(
f"""SELECT s.id, s.source, s.title, s.model, s.started_at,
COALESCE(
(SELECT MAX(m.timestamp) FROM messages m
WHERE m.session_id = s.id),
s.started_at
) AS last_active,
s.ended_at, s.message_count, s.archived
FROM sessions s WHERE {where}
ORDER BY last_active ASC, s.started_at ASC""",
params,
)
return [dict(row) for row in cursor.fetchall()]
def archive_sessions(
self,
older_than_days: Optional[float] = None,
source: str = None,
**filters,
) -> int:
"""Bulk-archive (soft-hide) every session matching the filters.
Same filter surface as :meth:`prune_sessions`, but instead of deleting
rows it flips ``archived = 1`` via :meth:`set_session_archived` so
each match's compression lineage is archived as a unit (an unarchived
compression root would otherwise resurrect the conversation in
Desktop's projected list). Nothing is deleted; messages and transcript
files are untouched. Returns the number of sessions matched.
``archived`` defaults to ``False`` here (only select rows not yet
archived) so repeat runs are idempotent no-ops.
"""
filters.setdefault("archived", False)
rows = self.list_prune_candidates(
older_than_days=older_than_days, source=source, **filters
)
for row in rows:
self.set_session_archived(row["id"], True)
return len(rows)
def archive_stale_sessions(
self, idle_days: float, *, exclude_pinned: bool = True
) -> int:
"""Archive every session untouched for at least ``idle_days`` days.
"Touched" is the latest message timestamp (falling back to
``started_at``) — i.e. real recency, not creation time — so a session
created long ago but active yesterday is spared, while an old
abandoned one (even a still-open one) is swept. Unlike
:meth:`archive_sessions`, this method can also archive unended
sessions.
Guards:
* ``pinned = 0`` when ``exclude_pinned`` (the Desktop "keep" flag).
* ``archived = 0`` so repeat runs are idempotent no-ops.
* only lineage *tips* / standalone rows are candidates
(``end_reason <> 'compression'``); a stale tip archives its whole
chain via :meth:`set_session_archived`, so we never resurrect an
active conversation by matching an old compressed-away root whose
live continuation is recent.
Returns the number of sessions archived. Never raises for an empty or
non-positive ``idle_days`` — it simply archives nothing.
"""
if idle_days is None or idle_days < 0:
return 0
cutoff = time.time() - float(idle_days) * 86400.0
pin_clause = "AND s.pinned = 0" if exclude_pinned else ""
with self._lock:
rows = self._conn.execute(
f"""
SELECT s.id FROM sessions s
WHERE s.archived = 0
AND COALESCE(s.end_reason, '') <> 'compression'
{pin_clause}
AND COALESCE(
(SELECT MAX(m.timestamp) FROM messages m
WHERE m.session_id = s.id),
s.started_at
) < ?
ORDER BY s.started_at ASC
""",
(cutoff,),
).fetchall()
ids = [(r["id"] if isinstance(r, sqlite3.Row) else r[0]) for r in rows]
for sid in ids:
self.set_session_archived(sid, True)
return len(ids)
def prune_sessions(
self,
older_than_days: Optional[float] = 90,
source: str = None,
sessions_dir: Optional[Path] = None,
**filters,
) -> int:
"""Delete sessions matching the filters. Returns count deleted.
By default, delete ended sessions inactive for
``older_than_days`` days, optionally restricted to ``source``.
Activity is the latest message timestamp, falling back to
``started_at`` for sessions without messages. Additional keyword
filters AND together — the full set is defined by
:meth:`_prune_filter_where`:
* ``last_active_before`` / ``last_active_after`` — epoch bounds on
the latest message timestamp (falling back to ``started_at``).
* ``started_before`` / ``started_after`` — epoch bounds on
``started_at``. An explicit ``started_before`` overrides the
default ``older_than_days`` inactivity cutoff; pass
``older_than_days=None`` for no implicit upper age bound.
* ``title_like`` / ``model_like`` / ``branch_like`` —
case-insensitive substring matches.
* ``end_reason`` / ``provider`` / ``user_id`` / ``chat_id`` /
``chat_type`` — exact matches (provider case-insensitive, against
``billing_provider``).
* ``cwd_prefix`` — session cwd equals or is under this path.
* ``min_messages`` / ``max_messages`` — bounds on message_count.
* ``min_tokens`` / ``max_tokens`` — bounds on input+output tokens.
* ``min_cost`` / ``max_cost`` — bounds on USD cost
(actual, falling back to estimated).
* ``min_tool_calls`` / ``max_tool_calls`` — bounds on tool_call_count.
* ``archived`` — tri-state: None = both (default), True = only
archived, False = only unarchived.
Only prunes ended sessions (not active ones). Child sessions outside
the prune window are orphaned (parent_session_id set to NULL) rather
than cascade-deleted. When *sessions_dir* is provided, also removes
on-disk transcript files (``.json`` / ``.jsonl`` /
``request_dump_*``) for every pruned session, outside the DB
transaction.
"""
if (
filters.get("last_active_before") is None
and filters.get("started_before") is None
and older_than_days is not None
):
filters["last_active_before"] = time.time() - (
older_than_days * 86400
)
where, where_params = self._prune_filter_where(source=source, **filters)
removed_ids: list[str] = []
def _do(conn):
cursor = conn.execute(
f"SELECT s.id FROM sessions s WHERE {where}", where_params
)
session_ids = {row["id"] for row in cursor.fetchall()}
if not session_ids:
return 0
# Orphan any sessions whose parent is about to be deleted
placeholders = ",".join("?" * len(session_ids))
conn.execute(
f"UPDATE sessions SET parent_session_id = NULL "
f"WHERE parent_session_id IN ({placeholders})",
list(session_ids),
)
for sid in session_ids:
conn.execute("DELETE FROM messages WHERE session_id = ?", (sid,))
conn.execute("DELETE FROM sessions WHERE id = ?", (sid,))
removed_ids.append(sid)
return len(session_ids)
count = self._execute_write(_do)
# Clean up on-disk files outside the DB transaction
for sid in removed_ids:
self._remove_session_files(sessions_dir, sid)
return count
# ── Meta key/value (for scheduler bookkeeping) ──
def get_meta(self, key: str) -> Optional[str]:
"""Read a value from the state_meta key/value store."""
# Kept on self._lock (not _read_ctx) because callers like
# fts_rebuild_step read progress before entering a write
# transaction, and the read-only WAL connection sees only
# committed data — a pending write transaction's uncommitted
# meta writes would be invisible. This is a cheap point lookup,
# not the convoy bottleneck the read-path split targets.
with self._lock:
row = self._conn.execute(
"SELECT value FROM state_meta WHERE key = ?", (key,)
).fetchone()
if row is None:
return None
return row["value"] if isinstance(row, sqlite3.Row) else row[0]
def set_meta(
self, key: str, value: str, *, cursor: Optional[sqlite3.Cursor] = None
) -> None:
"""Write a value to the state_meta key/value store.
When ``cursor`` is provided the write is issued on that cursor
inline (used during ``_init_schema``, which already holds an open
transaction — routing through ``_execute_write`` there would nest
BEGIN IMMEDIATE and deadlock). Otherwise a normal write transaction
is used.
"""
if cursor is not None:
cursor.execute(
"INSERT INTO state_meta (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
(key, value),
)
return
def _do(conn):
conn.execute(
"INSERT INTO state_meta (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
(key, value),
)
self._execute_write(_do)
def apply_telegram_topic_migration(self) -> None:
"""Create Telegram DM topic-mode tables on explicit /topic opt-in.
This migration is deliberately not part of automatic SessionDB startup
reconciliation. Operators must be able to upgrade Hermes, keep the old
Telegram bot behavior running, and only mutate topic-mode state when the
user executes /topic to opt into the feature.
Schema versions:
v1 — initial shape (no ON DELETE CASCADE on session_id FK)
v2 — session_id FK gets ON DELETE CASCADE so session pruning
automatically clears bindings.
"""
def _do(conn):
conn.executescript(
"""
CREATE TABLE IF NOT EXISTS telegram_dm_topic_mode (
chat_id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
enabled INTEGER NOT NULL DEFAULT 1,
activated_at REAL NOT NULL,
updated_at REAL NOT NULL,
has_topics_enabled INTEGER,
allows_users_to_create_topics INTEGER,
capability_checked_at REAL,
intro_message_id TEXT,
pinned_message_id TEXT
);
CREATE TABLE IF NOT EXISTS telegram_dm_topic_bindings (
chat_id TEXT NOT NULL,
thread_id TEXT NOT NULL,
user_id TEXT NOT NULL,
session_key TEXT NOT NULL,
session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
managed_mode TEXT NOT NULL DEFAULT 'auto',
linked_at REAL NOT NULL,
updated_at REAL NOT NULL,
PRIMARY KEY (chat_id, thread_id)
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_session
ON telegram_dm_topic_bindings(session_id);
CREATE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_user
ON telegram_dm_topic_bindings(user_id, chat_id);
"""
)
# v1 → v2: rebuild telegram_dm_topic_bindings if its session_id FK
# lacks ON DELETE CASCADE. SQLite can't ALTER a foreign key, so we
# rebuild the table. Only runs once per DB (version gate).
current = conn.execute(
"SELECT value FROM state_meta WHERE key = ?",
("telegram_dm_topic_schema_version",),
).fetchone()
current_version = int(current[0]) if current and str(current[0]).isdigit() else 0
if current_version < 2:
fk_rows = conn.execute(
"PRAGMA foreign_key_list('telegram_dm_topic_bindings')"
).fetchall()
needs_rebuild = any(
row[2] == "sessions" and (row[6] or "") != "CASCADE"
for row in fk_rows
)
if needs_rebuild:
conn.executescript(
"""
CREATE TABLE telegram_dm_topic_bindings_new (
chat_id TEXT NOT NULL,
thread_id TEXT NOT NULL,
user_id TEXT NOT NULL,
session_key TEXT NOT NULL,
session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
managed_mode TEXT NOT NULL DEFAULT 'auto',
linked_at REAL NOT NULL,
updated_at REAL NOT NULL,
PRIMARY KEY (chat_id, thread_id)
);
INSERT INTO telegram_dm_topic_bindings_new
SELECT chat_id, thread_id, user_id, session_key,
session_id, managed_mode, linked_at, updated_at
FROM telegram_dm_topic_bindings;
DROP TABLE telegram_dm_topic_bindings;
ALTER TABLE telegram_dm_topic_bindings_new
RENAME TO telegram_dm_topic_bindings;
CREATE UNIQUE INDEX idx_telegram_dm_topic_bindings_session
ON telegram_dm_topic_bindings(session_id);
CREATE INDEX idx_telegram_dm_topic_bindings_user
ON telegram_dm_topic_bindings(user_id, chat_id);
"""
)
conn.execute(
"INSERT INTO state_meta (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
("telegram_dm_topic_schema_version", "2"),
)
self._execute_write(_do)
def enable_telegram_topic_mode(
self,
*,
chat_id: str,
user_id: str,
has_topics_enabled: Optional[bool] = None,
allows_users_to_create_topics: Optional[bool] = None,
) -> None:
"""Enable Telegram DM topic mode for one private chat/user.
This method intentionally owns the explicit topic migration. Ordinary
SessionDB startup must not create these side tables.
"""
self.apply_telegram_topic_migration()
now = time.time()
def _to_int(value: Optional[bool]) -> Optional[int]:
if value is None:
return None
return 1 if value else 0
def _do(conn):
conn.execute(
"""
INSERT INTO telegram_dm_topic_mode (
chat_id, user_id, enabled, activated_at, updated_at,
has_topics_enabled, allows_users_to_create_topics,
capability_checked_at
) VALUES (?, ?, 1, ?, ?, ?, ?, ?)
ON CONFLICT(chat_id) DO UPDATE SET
user_id = excluded.user_id,
enabled = 1,
updated_at = excluded.updated_at,
has_topics_enabled = excluded.has_topics_enabled,
allows_users_to_create_topics = excluded.allows_users_to_create_topics,
capability_checked_at = excluded.capability_checked_at
""",
(
str(chat_id),
str(user_id),
now,
now,
_to_int(has_topics_enabled),
_to_int(allows_users_to_create_topics),
now,
),
)
self._execute_write(_do)
def disable_telegram_topic_mode(
self,
*,
chat_id: str,
clear_bindings: bool = True,
) -> None:
"""Disable Telegram DM topic mode for one private chat.
When ``clear_bindings`` is True (default) the (chat_id, thread_id)
bindings for this chat are also cleared so re-enabling later
starts from a clean slate. Set to False if the operator wants to
preserve bindings for a later re-enable.
Never creates the topic-mode tables from scratch; if they don't
exist there is nothing to disable and the call is a no-op.
"""
def _do(conn):
try:
conn.execute(
"UPDATE telegram_dm_topic_mode SET enabled = 0, updated_at = ? "
"WHERE chat_id = ?",
(time.time(), str(chat_id)),
)
if clear_bindings:
conn.execute(
"DELETE FROM telegram_dm_topic_bindings WHERE chat_id = ?",
(str(chat_id),),
)
except sqlite3.OperationalError:
# Tables don't exist yet — nothing to disable.
return
self._execute_write(_do)
def is_telegram_topic_mode_enabled(self, *, chat_id: str, user_id: str) -> bool:
"""Return whether Telegram DM topic mode is enabled for this chat/user."""
with self._lock:
try:
row = self._conn.execute(
"""
SELECT enabled FROM telegram_dm_topic_mode
WHERE chat_id = ? AND user_id = ?
""",
(str(chat_id), str(user_id)),
).fetchone()
except sqlite3.OperationalError:
return False
if row is None:
return False
enabled = row["enabled"] if isinstance(row, sqlite3.Row) else row[0]
return bool(enabled)
def get_telegram_topic_binding(
self,
*,
chat_id: str,
thread_id: str,
) -> Optional[Dict[str, Any]]:
"""Return the session binding for a Telegram DM topic, if present."""
with self._lock:
try:
row = self._conn.execute(
"""
SELECT * FROM telegram_dm_topic_bindings
WHERE chat_id = ? AND thread_id = ?
""",
(str(chat_id), str(thread_id)),
).fetchone()
except sqlite3.OperationalError:
return None
return dict(row) if row else None
def list_telegram_topic_bindings_for_chat(
self,
*,
chat_id: str,
) -> List[Dict[str, Any]]:
"""All Telegram DM topic bindings for one chat, newest first.
Read-only; returns [] if the bindings table doesn't exist yet
(does not trigger the topic-mode migration).
"""
with self._lock:
try:
rows = self._conn.execute(
"SELECT * FROM telegram_dm_topic_bindings "
"WHERE chat_id = ? ORDER BY updated_at DESC",
(str(chat_id),),
).fetchall()
except sqlite3.OperationalError:
return []
return [dict(row) for row in rows]
def get_telegram_topic_binding_by_session(
self,
*,
session_id: str,
) -> Optional[Dict[str, Any]]:
"""Return the Telegram DM topic binding for a given session_id, if present.
Uses the UNIQUE INDEX on telegram_dm_topic_bindings(session_id) for an
efficient reverse lookup. Returns None when the session has no binding or
the table does not exist yet.
"""
with self._lock:
try:
row = self._conn.execute(
"""
SELECT * FROM telegram_dm_topic_bindings
WHERE session_id = ?
""",
(str(session_id),),
).fetchone()
except sqlite3.OperationalError:
return None
return dict(row) if row else None
def delete_telegram_topic_binding(
self,
*,
chat_id: str,
thread_id: str,
) -> int:
"""Remove the binding row for a single (chat, thread) pair.
Called when the Telegram Bot API confirms a topic was deleted
externally (``Thread not found`` after the same-thread retry
already failed). Without this prune, the stale row keeps
living in ``telegram_dm_topic_bindings`` and the
recovery logic in ``gateway.run._recover_telegram_topic_thread_id``
cheerfully redirects future inbound messages to the deleted
topic, causing tool progress, approvals, and replies to land
in the wrong place. Issue #31501.
When this prune removes the chat's *last* remaining binding,
the chat's row in ``telegram_dm_topic_mode`` is also flipped to
``enabled = 0`` in the same transaction. Otherwise the chat
would be left in topic mode with zero lanes — and
``gateway.run._recover_telegram_topic_thread_id`` keeps treating
the chat as topic-enabled, lobby messages keep hunting for a
binding that no longer exists, and a user who disabled topics in
the Telegram client (rather than via ``/topic off``) stays stuck
until the next send happens to fail. Clearing the flag makes
recovery fully stand down once the dead topics are gone.
Returns the number of binding rows deleted (0 when the binding
was already absent or the topic-mode tables haven't been
migrated yet — both are silent no-ops; we never raise from
a cleanup hot path).
"""
chat_id = str(chat_id)
thread_id = str(thread_id)
deleted = {"count": 0}
def _do(conn):
try:
cursor = conn.execute(
"""
DELETE FROM telegram_dm_topic_bindings
WHERE chat_id = ? AND thread_id = ?
""",
(chat_id, thread_id),
)
deleted["count"] = cursor.rowcount or 0
except sqlite3.OperationalError:
# Tables don't exist yet — nothing to prune.
deleted["count"] = 0
return
if not deleted["count"]:
return
# If that was the chat's last binding, disable topic mode for
# the chat so recovery stops steering lobby messages at a now
# empty lane set. Same transaction → no read-after-prune race.
try:
remaining = conn.execute(
"""
SELECT 1 FROM telegram_dm_topic_bindings
WHERE chat_id = ? LIMIT 1
""",
(chat_id,),
).fetchone()
if remaining is None:
conn.execute(
"UPDATE telegram_dm_topic_mode "
"SET enabled = 0, updated_at = ? WHERE chat_id = ?",
(time.time(), chat_id),
)
except sqlite3.OperationalError:
# telegram_dm_topic_mode absent — binding prune still stands.
pass
self._execute_write(_do)
return deleted["count"]
def bind_telegram_topic(
self,
*,
chat_id: str,
thread_id: str,
user_id: str,
session_key: str,
session_id: str,
managed_mode: str = "auto",
) -> None:
"""Bind one Telegram DM topic thread to one Hermes session.
A Hermes session may only be linked to one Telegram topic in MVP.
Rebinding the same topic to the same session is idempotent; trying to
link the same session to a different topic raises ValueError.
"""
self.apply_telegram_topic_migration()
now = time.time()
chat_id = str(chat_id)
thread_id = str(thread_id)
user_id = str(user_id)
session_key = str(session_key)
session_id = str(session_id)
def _do(conn):
existing_session = conn.execute(
"""
SELECT chat_id, thread_id FROM telegram_dm_topic_bindings
WHERE session_id = ?
""",
(session_id,),
).fetchone()
if existing_session is not None:
linked_chat = existing_session["chat_id"] if isinstance(existing_session, sqlite3.Row) else existing_session[0]
linked_thread = existing_session["thread_id"] if isinstance(existing_session, sqlite3.Row) else existing_session[1]
if str(linked_chat) != chat_id or str(linked_thread) != thread_id:
raise ValueError("session is already linked to another Telegram topic")
conn.execute(
"""
INSERT INTO telegram_dm_topic_bindings (
chat_id, thread_id, user_id, session_key, session_id,
managed_mode, linked_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(chat_id, thread_id) DO UPDATE SET
user_id = excluded.user_id,
session_key = excluded.session_key,
session_id = excluded.session_id,
managed_mode = excluded.managed_mode,
updated_at = excluded.updated_at
""",
(
chat_id,
thread_id,
user_id,
session_key,
session_id,
managed_mode,
now,
now,
),
)
self._execute_write(_do)
def is_telegram_session_linked_to_topic(self, *, session_id: str) -> bool:
"""Return True if a Hermes session is already bound to any Telegram DM topic.
Read-only: does NOT trigger the telegram-topic migration. If the
topic-mode tables have not been created yet (i.e. nobody has run
``/topic`` in this profile), the session is by definition unbound
and we return False.
"""
with self._lock:
try:
row = self._conn.execute(
"""
SELECT 1 FROM telegram_dm_topic_bindings
WHERE session_id = ?
LIMIT 1
""",
(str(session_id),),
).fetchone()
except sqlite3.OperationalError:
return False
return row is not None
def list_unlinked_telegram_sessions_for_user(
self,
*,
chat_id: str,
user_id: str,
limit: int = 10,
) -> List[Dict[str, Any]]:
"""List previous Telegram sessions for this user that are not bound to a topic.
Read-only: does NOT trigger the telegram-topic migration. If the
topic-mode tables are absent, fall back to a simpler query that
just returns this user's Telegram sessions — there can't be any
bindings yet.
"""
with self._lock:
try:
rows = self._conn.execute(
f"""
SELECT s.*,
COALESCE(
(SELECT {_PREVIEW_RAW_SELECT}
FROM messages m
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
ORDER BY m.timestamp, m.id LIMIT 1),
''
) AS _preview_raw,
COALESCE(
(SELECT MAX(m2.timestamp) FROM messages m2 WHERE m2.session_id = s.id),
s.started_at
) AS last_active
FROM sessions s
WHERE s.source = 'telegram'
AND s.user_id = ?
AND NOT EXISTS (
SELECT 1 FROM telegram_dm_topic_bindings b
WHERE b.session_id = s.id
)
ORDER BY last_active DESC, s.started_at DESC
LIMIT ?
""",
(str(user_id), int(limit)),
).fetchall()
except sqlite3.OperationalError:
# telegram_dm_topic_bindings doesn't exist yet — no bindings
# means every telegram session for this user is "unlinked".
rows = self._conn.execute(
f"""
SELECT s.*,
COALESCE(
(SELECT {_PREVIEW_RAW_SELECT}
FROM messages m
WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL
ORDER BY m.timestamp, m.id LIMIT 1),
''
) AS _preview_raw,
COALESCE(
(SELECT MAX(m2.timestamp) FROM messages m2 WHERE m2.session_id = s.id),
s.started_at
) AS last_active
FROM sessions s
WHERE s.source = 'telegram'
AND s.user_id = ?
ORDER BY last_active DESC, s.started_at DESC
LIMIT ?
""",
(str(user_id), int(limit)),
).fetchall()
sessions: List[Dict[str, Any]] = []
for row in rows:
session = dict(row)
session["preview"] = _shape_preview(session.pop("_preview_raw", ""))
sessions.append(session)
return sessions
# ── Space reclamation ──
# FTS5 virtual tables whose b-tree segments we merge on optimize. The
# trigram table is created lazily / may be disabled, and the cjk-bigram
# table only exists (and is only queryable) when the loadable tokenizer
# is present — so we probe each before touching it (see optimize_fts).
_FTS_TABLES = ("messages_fts", "messages_fts_trigram", "messages_fts_cjk")
def logical_size_bytes(self) -> Optional[int]:
"""Database size in bytes as SQLite itself accounts for it.
``page_count * page_size`` — the size the main DB file will have once
the WAL is checkpointed back into it.
Prefer this over ``os.path.getsize(db_path)`` when reporting the effect
of a VACUUM. In WAL mode a VACUUM's rewrite lands in the ``-wal`` file,
and the checkpoint that folds it back is refused while any other
connection (a live gateway) holds a read-mark. Until that happens the
main file on disk still carries its pre-VACUUM size and keeps growing,
so a stat()-based before/after delta understates the win and can go
negative — the "reclaimed -3820.1 MB" report on a database that had
actually shrunk 60%.
Returns None if the pragmas cannot be read.
"""
try:
with self._lock:
if self._conn is None:
return None
page_count = self._conn.execute("PRAGMA page_count").fetchone()[0]
page_size = self._conn.execute("PRAGMA page_size").fetchone()[0]
return int(page_count) * int(page_size)
except Exception as exc:
logger.debug("Could not read logical DB size: %s", exc)
return None
def vacuum(self) -> int:
"""Run VACUUM to reclaim disk space after large deletes.
SQLite does not shrink the database file when rows are deleted —
freed pages just get reused on the next insert. After a prune that
removed hundreds of sessions, the file stays bloated unless we
explicitly VACUUM.
VACUUM rewrites the entire DB, so it's expensive (seconds per
100MB) and cannot run inside a transaction. It also acquires an
exclusive lock, so callers must ensure no other writers are
active. Safe to call at startup before the gateway/CLI starts
serving traffic.
FTS5 segments are merged first via :meth:`optimize_fts` so the
subsequent VACUUM reclaims the pages freed by the merge. This is a
layout-only optimization — search results are unchanged.
Returns the number of FTS indexes that were optimized (0 if the
merge step failed or no FTS tables exist).
"""
# Merge FTS5 segments before VACUUM so the freed pages are returned
# to the OS in the same pass. optimize_fts() manages its own lock.
optimized = 0
try:
optimized = self.optimize_fts()
except Exception as exc:
logger.warning("FTS optimize before VACUUM failed: %s", exc)
# VACUUM cannot be executed inside a transaction.
with self._lock:
# Best-effort WAL checkpoint first, then VACUUM.
try:
self._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
except Exception as exc:
logger.debug("WAL checkpoint (TRUNCATE) before VACUUM failed: %s", exc)
self._conn.execute("VACUUM")
return optimized
def maybe_auto_prune_and_vacuum(
self,
retention_days: int = 90,
min_interval_hours: int = 24,
vacuum: bool = True,
sessions_dir: Optional[Path] = None,
) -> Dict[str, Any]:
"""Idempotent auto-maintenance: prune inactive sessions + optional VACUUM.
Records the last run timestamp in state_meta so subsequent calls
within ``min_interval_hours`` no-op. Designed to be called once at
startup from long-lived entrypoints (CLI, gateway, cron scheduler).
When *sessions_dir* is provided, on-disk transcript files
(``.json`` / ``.jsonl`` / ``request_dump_*``) for pruned sessions
are removed as part of the same sweep (issue #3015).
Never raises. On any failure, logs a warning and returns a dict
with ``"error"`` set.
Returns a dict with keys:
- ``"skipped"`` (bool) — true if within min_interval_hours of last run
- ``"pruned"`` (int) — number of sessions deleted
- ``"vacuumed"`` (bool) — true if VACUUM ran
- ``"error"`` (str, optional) — present only on failure
"""
result: Dict[str, Any] = {"skipped": False, "pruned": 0, "vacuumed": False}
try:
# Skip if another process/call did maintenance recently.
last_raw = self.get_meta("last_auto_prune")
now = time.time()
if last_raw:
try:
last_ts = float(last_raw)
if now - last_ts < min_interval_hours * 3600:
result["skipped"] = True
return result
except (TypeError, ValueError):
pass # corrupt meta; treat as no prior run
pruned = self.prune_sessions(
older_than_days=retention_days,
sessions_dir=sessions_dir,
)
result["pruned"] = pruned
# Only VACUUM if we actually freed rows — VACUUM on a tight DB
# is wasted I/O. Threshold keeps small DBs from paying the cost.
if vacuum and pruned > 0:
try:
self.vacuum()
result["vacuumed"] = True
except Exception as exc:
logger.warning("state.db VACUUM failed: %s", exc)
# Record the attempt even if pruned == 0, so we don't retry
# every startup within the min_interval_hours window.
self.set_meta("last_auto_prune", str(now))
if pruned > 0:
logger.info(
"state.db auto-maintenance: pruned %d session(s) inactive for %d days%s",
pruned,
retention_days,
" + VACUUM" if result["vacuumed"] else "",
)
except Exception as exc:
# Maintenance must never block startup. Log and return error marker.
logger.warning("state.db auto-maintenance failed: %s", exc)
result["error"] = str(exc)
return result
def maybe_auto_archive(
self,
idle_days: float = 3,
min_interval_hours: int = 24,
exclude_pinned: bool = True,
) -> Dict[str, Any]:
"""Idempotent auto-archive: soft-hide sessions idle for ``idle_days``.
Sibling of :meth:`maybe_auto_prune_and_vacuum` but non-destructive —
it archives (hides) rather than deletes, and ages on last activity
(see :meth:`archive_stale_sessions`) rather than creation. Records the
last run in ``state_meta['last_auto_archive']`` so calls within
``min_interval_hours`` no-op; safe to call opportunistically (startup
hooks, or when the Desktop backend lists sessions).
Never raises. Returns a dict with:
- ``"skipped"`` (bool) — within min_interval_hours of last run
- ``"archived"`` (int) — sessions archived this run
- ``"error"`` (str, optional) — present only on failure
"""
result: Dict[str, Any] = {"skipped": False, "archived": 0}
try:
last_raw = self.get_meta("last_auto_archive")
now = time.time()
if last_raw:
try:
if now - float(last_raw) < min_interval_hours * 3600:
result["skipped"] = True
return result
except (TypeError, ValueError):
pass # corrupt meta; treat as no prior run
archived = self.archive_stale_sessions(
idle_days, exclude_pinned=exclude_pinned
)
result["archived"] = archived
# Record even a zero-archive run so we don't re-sweep every call
# within the interval window.
self.set_meta("last_auto_archive", str(now))
if archived > 0:
logger.info(
"state.db auto-archive: archived %d session(s) idle >= %s days",
archived,
idle_days,
)
except Exception as exc:
logger.warning("state.db auto-archive failed: %s", exc)
result["error"] = str(exc)
return result
# ── Handoff (cross-platform session transfer) ──────────────────────────
#
# State machine:
# None — no handoff in flight
# "pending" — CLI requested handoff, gateway hasn't picked it up yet
# "running" — gateway is processing (session switch + synthetic turn)
# "completed"— gateway successfully delivered the synthetic turn
# "failed" — gateway hit an error; reason in handoff_error
#
# The CLI writes "pending" then poll-waits for terminal state. The gateway
# watcher transitions pending→running→{completed,failed}.
def request_handoff(self, session_id: str, platform: str) -> bool:
"""Mark a session as pending handoff to the given platform.
Returns True if the row was found and not already in flight; False if
the session is already in a non-terminal handoff state.
"""
def _do(conn):
cur = conn.execute(
"UPDATE sessions "
"SET handoff_state = 'pending', "
" handoff_platform = ?, "
" handoff_error = NULL "
"WHERE id = ? AND (handoff_state IS NULL "
" OR handoff_state IN ('completed', 'failed'))",
(platform, session_id),
)
return cur.rowcount > 0
return self._execute_write(_do)
def get_handoff_state(self, session_id: str) -> Optional[Dict[str, Any]]:
"""Read the current handoff state for a session.
Returns ``{"state", "platform", "error"}`` or None if the session has
no handoff record.
"""
try:
cur = self._conn.execute(
"SELECT handoff_state, handoff_platform, handoff_error "
"FROM sessions WHERE id = ?",
(session_id,),
)
row = cur.fetchone()
if not row:
return None
return {
"state": row["handoff_state"],
"platform": row["handoff_platform"],
"error": row["handoff_error"],
}
except Exception:
return None
def list_pending_handoffs(self) -> List[Dict[str, Any]]:
"""Return all sessions in handoff_state='pending', oldest first.
Used by the gateway's handoff watcher.
"""
try:
cur = self._conn.execute(
"SELECT * FROM sessions "
"WHERE handoff_state = 'pending' "
"ORDER BY started_at ASC"
)
return [dict(r) for r in cur.fetchall()]
except Exception:
return []
def claim_handoff(self, session_id: str) -> bool:
"""Atomically transition pending → running. Returns True if claimed."""
def _do(conn):
cur = conn.execute(
"UPDATE sessions SET handoff_state = 'running' "
"WHERE id = ? AND handoff_state = 'pending'",
(session_id,),
)
return cur.rowcount > 0
return self._execute_write(_do)
def complete_handoff(self, session_id: str) -> None:
"""Mark a handoff as completed."""
def _do(conn):
conn.execute(
"UPDATE sessions SET handoff_state = 'completed', "
"handoff_error = NULL WHERE id = ?",
(session_id,),
)
self._execute_write(_do)
def fail_handoff(self, session_id: str, error: str) -> None:
"""Mark a handoff as failed and record the reason."""
def _do(conn):
conn.execute(
"UPDATE sessions SET handoff_state = 'failed', "
"handoff_error = ? WHERE id = ?",
(error[:500], session_id),
)
self._execute_write(_do)
class AsyncSessionDB:
"""Async door onto SessionDB: offloads each call via asyncio.to_thread so a blocking SQLite call never freezes the event loop. Generic forwarder — the audit confirms no method returns a live cursor/generator."""
def __init__(self, db: "SessionDB") -> None:
self._db = db
def __getattr__(self, name: str):
attr = getattr(self._db, name)
if not callable(attr):
return attr
async def _offloaded(*args, **kwargs):
return await asyncio.to_thread(attr, *args, **kwargs)
return _offloaded
Reference in a new issue View git blame Copy permalink