diff --git a/apps/desktop/electron/main.ts b/apps/desktop/electron/main.ts index e83fa416fcb..573f867ff45 100644 --- a/apps/desktop/electron/main.ts +++ b/apps/desktop/electron/main.ts @@ -2772,6 +2772,11 @@ async function applyUpdates(opts = {}) { const venvBin = path.join(updateRoot, 'venv', IS_WINDOWS ? 'Scripts' : 'bin') + // ── Pre-flight state.db integrity guard (#68474) ───────────────── + // Emergency backup and header verification before the update touches + // anything. Runs while the backend is still alive. + preflightStateDb(HERMES_HOME, rememberLog) + // Stop our own backend(s) and wait for the venv shim to unlock BEFORE we // spawn the updater. Without this the updater races a still-locked // hermes.exe (held by the backend child / its grandchildren) and the update @@ -2969,6 +2974,101 @@ function runningAppBundle() { return dir.endsWith('.app') ? dir : null } +// ── Pre-flight state.db integrity guard (#68474) ───────────────────── +// Take an emergency snapshot of state.db and verify the live copy is +// intact before any update process mutates the install. Runs in the +// desktop Electron process itself, before the backend is killed and +// before the updater is spawned — a separate safety net from the +// Python-level pre-update snapshot inside `hermes update`. +function preflightStateDb(hermesHome, rememberLog) { + const stateDbPath = path.join(hermesHome, 'state.db') + + if (!fileExists(stateDbPath)) { + rememberLog('[updates] state.db pre-flight: not found (fresh install?)') + return + } + + try { + const stat = fs.statSync(stateDbPath) + + if (stat.size > 100) { + const fd = fs.openSync(stateDbPath, 'r') + const header = Buffer.alloc(16) + + fs.readSync(fd, header, 0, 16, 0) + fs.closeSync(fd) + + const expectedHeader = Buffer.from('SQLite format 3\0') + const headerOk = header.equals(expectedHeader) + + rememberLog( + `[updates] state.db pre-flight: size=${stat.size}, ` + + `headerOk=${headerOk}, headerHex=${header.toString('hex')}` + ) + + if (!headerOk) { + rememberLog( + '[updates] state.db header is INVALID before update — ' + + 'this indicates pre-existing corruption or a concurrent write issue' + ) + } + + // Emergency timestamped backup, separate from the Python-level snapshot. + const ts = new Date().toISOString().replace(/[:.]/g, '-') + const emergencyPath = path.join( + hermesHome, + `state.db.pre-update-emergency-${ts}.bak` + ) + + try { + fs.copyFileSync(stateDbPath, emergencyPath) + const emergStat = fs.statSync(emergencyPath) + + rememberLog( + `[updates] emergency state.db backup: ${emergencyPath} ` + + `(${emergStat.size} bytes)` + ) + + // Prune to the 2 most recent emergency backups. + try { + const homeDir = fs.readdirSync(hermesHome) + const backups = homeDir + .filter( + f => + f.startsWith('state.db.pre-update-emergency-') && + f.endsWith('.bak') && + f !== path.basename(emergencyPath) + ) + .sort() + .reverse() + + for (const old of backups.slice(2)) { + try { + fs.unlinkSync(path.join(hermesHome, old)) + } catch { + void 0 + } + } + } catch { + void 0 + } + } catch (copyErr) { + rememberLog( + `[updates] emergency state.db backup failed: ${copyErr.message}` + ) + } + } else { + rememberLog( + `[updates] state.db too small (${stat.size} bytes) for a valid SQLite database` + ) + } + } catch (statErr) { + rememberLog( + `[updates] could not stat state.db before update: ${statErr.message}` + ) + } +} + function shellQuote(value) { return `'${String(value).replace(/'/g, `'\\''`)}'` } @@ -2983,13 +3083,15 @@ async function applyUpdatesPosixInApp(opts: any) { if (!hermes) { emitUpdateProgress({ stage: 'manual', message: 'hermes update', percent: null }) - return { ok: true, manual: true, command: 'hermes update', hermesRoot: updateRoot } } + // ── Pre-flight state.db integrity guard (#68474) ── + preflightStateDb(HERMES_HOME, rememberLog) + // Put the Hermes-managed Node and the venv on PATH so `hermes desktop`'s // npm build can find them on a machine with no system Node. Windows portable - // Node lives directly under %LOCALAPPDATA%\hermes\node, not node\bin. + // Node lives directly under %LOCALAPPDATA%\\hermes\\node, not node\\bin. // PYTHONUNBUFFERED: `hermes update` writes to a pipe here, so CPython // block-buffers stdout and long quiet steps (the pre-update backup can zip // multi-GB archives for minutes) stream nothing to the progress UI — users diff --git a/hermes_cli/backup.py b/hermes_cli/backup.py index 2a25217160c..5898b1a56ae 100644 --- a/hermes_cli/backup.py +++ b/hermes_cli/backup.py @@ -283,6 +283,134 @@ def _safe_copy_db(src: Path, dst: Path) -> bool: pass +# --------------------------------------------------------------------------- +# SQLite integrity verification +# --------------------------------------------------------------------------- + +_SQLITE_HEADER = b"SQLite format 3\0" + + +def verify_sqlite_integrity( + path: Path, + *, + check_header: bool = True, + run_pragma: bool = True, + max_bytes: int = 0, +) -> dict: + """Verify that a SQLite database at *path* is intact. + + Checks, in order: + 1. File exists and has an expected minimum size. + 2. SQLite header magic bytes are present. + 3. A read-only ``PRAGMA integrity_check`` execution passes. + + Args: + path: Path to the database file. + check_header: When true (default), verify the SQLite header magic. + run_pragma: When true (default), run ``PRAGMA integrity_check`` via + a read-only connection and verify the result is ``"ok"``. + max_bytes: When > 0, the file must be at most this many bytes. + Useful to catch a multi-GB DB before running ``PRAGMA integrity_check`` + on it (which reads the whole file into the pager). + + Returns: + A dict with keys: + - ``valid`` (bool): true when all requested checks passed. + - ``message`` (str): human-readable outcome or error detail. + - ``size`` (int | None): file size in bytes, or None if stat failed. + """ + result: dict = {"valid": False, "message": "", "size": None} + + try: + st = path.stat() + except FileNotFoundError: + result["message"] = f"not found: {path}" + return result + except OSError as exc: + result["message"] = f"cannot stat: {exc}" + return result + + result["size"] = st.st_size + + if st.st_size < 100: # SQLite minimum viable size (header + 1 page) + result["message"] = f"too small ({st.st_size} bytes) to be a valid SQLite database" + return result + + if max_bytes > 0 and st.st_size > max_bytes: + result["message"] = ( + f"size {st.st_size:,} bytes exceeds max_bytes {max_bytes:,}; " + "skipping PRAGMA integrity_check to avoid reading a very large file" + ) + result["valid"] = True + result["message"] += " (size-only check passed)" + + if check_header: + try: + with open(path, "rb") as f: + head = f.read(len(_SQLITE_HEADER)) + if head != _SQLITE_HEADER: + result["message"] = ( + f"missing SQLite header magic (got {head[:16].hex()!r})" + ) + return result + except OSError as exc: + result["message"] = f"cannot read header: {exc}" + return result + + if run_pragma and max_bytes > 0 and st.st_size > max_bytes: + run_pragma = False + + if run_pragma: + conn = None + try: + conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=1.0) + cursor = conn.execute("PRAGMA integrity_check") + rows = cursor.fetchall() + if len(rows) == 1 and rows[0][0] == "ok": + result["valid"] = True + result["message"] = "integrity check passed" + return result + errors = [str(r[0]) for r in rows] + result["message"] = f"integrity check failed: {'; '.join(errors[:5])}" + return result + except sqlite3.DatabaseError as exc: + result["message"] = f"cannot open database: {exc}" + return result + except Exception as exc: + result["message"] = f"integrity check error: {exc}" + return result + finally: + if conn is not None: + try: + conn.close() + except Exception: + pass + + result["valid"] = True + if not result["message"]: + result["message"] = "header check passed" + return result + + +def copy_db_and_verify(src: Path, dst: Path) -> bool: + """Like :func:`_safe_copy_db` but verifies the destination after copy. + + Returns True only when the copy succeeded AND the destination is valid + SQLite (header + integrity check). + """ + if not _safe_copy_db(src, dst): + return False + integrity = verify_sqlite_integrity(dst, run_pragma=True, max_bytes=0) + if not integrity.get("valid"): + try: + dst.unlink(missing_ok=True) + except OSError: + pass + logger.warning("Backup of %s failed integrity verification: %s", src, integrity.get("message")) + return False + return True + + # --------------------------------------------------------------------------- # Backup # --------------------------------------------------------------------------- diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 84557c26655..585e3e44465 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -7022,6 +7022,66 @@ def _update_via_zip(args): except Exception as e: logger.debug("Model catalog seed during zip update failed: %s", e) + # ── Post-update state.db integrity guard (#68474) ───────────────── + # Same as the git-pull path: verify state.db survived the ZIP update + # and auto-restore from the most recent pre-update snapshot if needed. + try: + from hermes_cli.backup import verify_sqlite_integrity + + _state_path = get_hermes_home() / "state.db" + if _state_path.exists(): + _state_ok = verify_sqlite_integrity( + _state_path, check_header=True, run_pragma=True + ) + if not _state_ok.get("valid"): + print() + print( + "⚠ state.db is corrupted after update: " + + _state_ok.get("message", "unknown error") + ) + _snap_root = _quick_snapshot_root(get_hermes_home()) + if _snap_root.exists(): + _snap_dirs = sorted( + (d for d in _snap_root.iterdir() if d.is_dir()), + reverse=True, + ) + for _snap_dir in _snap_dirs: + _snap_state = _snap_dir / "state.db" + if _snap_state.exists(): + _snap_ok = verify_sqlite_integrity( + _snap_state, check_header=True, run_pragma=True + ) + if _snap_ok.get("valid"): + try: + import shutil as _shutil + + _shutil.copy2(_snap_state, _state_path) + _restored_ok = verify_sqlite_integrity( + _state_path, + check_header=True, + run_pragma=True, + ) + if _restored_ok.get("valid"): + print( + " ✓ Auto-restored from snapshot " + f"{_snap_dir.name}" + ) + else: + print( + " ✗ Auto-restore FAILED — restored " + "copy also failed integrity" + ) + break + except OSError as _exc: + print( + f" ✗ Auto-restore file copy failed: {_exc}" + ) + break + except Exception as exc: + logger.debug( + "Post-update state.db integrity check (zip path) failed: %s", exc + ) + print() if node_failures: print( @@ -10013,13 +10073,58 @@ def _run_pre_update_backup(args) -> Optional[str]: snapshot_id = None try: - from hermes_cli.backup import create_quick_snapshot + from hermes_cli.backup import create_quick_snapshot, verify_sqlite_integrity snapshot_id = create_quick_snapshot( label="pre-update", keep=_PRE_UPDATE_SNAPSHOT_KEEP, max_file_size=_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE, ) + + # After the snapshot, verify the source state.db is still intact. + # The snapshot was taken via _safe_copy_db (read-only SQLite backup + # API), but a concurrent process (antivirus, force-killed gateway + # releasing file handles, Windows filter driver) can corrupt the live + # file at any point. A silent zeroing at this point would proceed with + # the update and exit code 0 — exactly the #68474 symptom. + if snapshot_id: + _src_path = get_hermes_home() / "state.db" + if _src_path.exists(): + _integrity = verify_sqlite_integrity( + _src_path, + check_header=True, + run_pragma=True, + max_bytes=_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE, + ) + if not _integrity.get("valid"): + _msg = _integrity.get("message", "unknown error") + print( + f" ⚠ state.db integrity check FAILED after snapshot: {_msg}" + ) + # Check if the snapshot itself is valid. + _snap_root = _quick_snapshot_root(get_hermes_home()) + _snap_state = _snap_root / snapshot_id / "state.db" + if _snap_state.exists(): + _snap_ok = verify_sqlite_integrity( + _snap_state, check_header=True, run_pragma=True + ) + if _snap_ok.get("valid"): + print( + " ✓ Snapshot copy is valid — continuing update." + ) + print( + " If state.db is lost after update it will be auto-restored." + ) + else: + print( + " ✗ Snapshot copy ALSO failed integrity — " + "the source was already corrupted before the backup." + ) + else: + print( + " ⚠ Snapshot does not contain state.db (was skipped or too large)." + ) + print() if snapshot_id: print(f"◆ Pre-update snapshot: {snapshot_id}") except Exception as exc: @@ -11323,6 +11428,82 @@ def _cmd_update_impl(args, gateway_mode: bool): print() print("✓ Code updated!") + # ── Post-update state.db integrity guard (#68474) ───────────────── + # Verify that state.db survived the update intact. If the live file + # is now corrupted (zeroed, missing header, integrity failure), + # automatically restore from the pre-update snapshot rather than + # letting the user discover silently that their sessions are gone. + try: + from hermes_cli.backup import verify_sqlite_integrity + + _state_path = get_hermes_home() / "state.db" + if _state_path.exists(): + _state_ok = verify_sqlite_integrity( + _state_path, + check_header=True, + run_pragma=True, + max_bytes=0, + ) + if _state_ok.get("valid"): + logger.debug( + "Post-update state.db integrity check: %s", + _state_ok.get("message"), + ) + else: + print() + print( + "⚠ state.db is corrupted after update: " + + _state_ok.get("message", "unknown error") + ) + _pre_snap_id = pre_update_snapshot_id + if _pre_snap_id: + _snap_state = ( + _quick_snapshot_root(get_hermes_home()) + / _pre_snap_id + / "state.db" + ) + if _snap_state.exists(): + _snap_ok = verify_sqlite_integrity( + _snap_state, check_header=True, run_pragma=True + ) + if _snap_ok.get("valid"): + try: + import shutil as _shutil + + _shutil.copy2(_snap_state, _state_path) + _restored_ok = verify_sqlite_integrity( + _state_path, + check_header=True, + run_pragma=True, + ) + if _restored_ok.get("valid"): + print( + " ✓ Auto-restored from pre-update " + f"snapshot ({_pre_snap_id})" + ) + else: + print( + " ✗ Auto-restore FAILED — restored " + "copy also failed integrity" + ) + except OSError as _exc: + print( + f" ✗ Auto-restore file copy failed: {_exc}" + ) + else: + print( + " ✗ Pre-update snapshot also failed integrity" + ) + else: + print( + " ⚠ Pre-update snapshot does not contain state.db" + ) + else: + print(" ⚠ No pre-update snapshot was taken") + print() + except Exception as exc: + logger.debug("Post-update state.db integrity check failed: %s", exc) + # Seed the model-catalog disk cache from the freshly-pulled checkout. # The repo ships the canonical catalog at # website/static/api/model-catalog.json, and `git pull` just made it