fix(backup): fail closed on sqlite snapshot errors

This commit is contained in:
teknium1 2026-07-09 18:56:15 -07:00 committed by Teknium
parent b4221c6db2
commit 332fbadd7b
2 changed files with 147 additions and 14 deletions

View file

@ -257,23 +257,30 @@ def _safe_copy_db(src: Path, dst: Path) -> bool:
"""Copy a SQLite database safely using the backup() API.
Handles WAL mode produces a consistent snapshot even while
the DB is being written to. Falls back to raw copy on failure.
the DB is being written to. Fail closed if a consistent snapshot cannot
be created: copying only the live main file can omit committed WAL data.
"""
conn = None
backup_conn = None
try:
conn = sqlite3.connect(f"file:{src}?mode=ro", uri=True)
backup_conn = sqlite3.connect(str(dst))
conn.backup(backup_conn)
backup_conn.close()
conn.close()
return True
except Exception as exc:
logger.warning("SQLite safe copy failed for %s: %s", src, exc)
try:
shutil.copy2(src, dst)
return True
except Exception as exc2:
logger.error("Raw copy also failed for %s: %s", src, exc2)
return False
dst.unlink(missing_ok=True)
except OSError:
pass
return False
finally:
for connection in (backup_conn, conn):
if connection is not None:
try:
connection.close()
except Exception:
pass
# ---------------------------------------------------------------------------
@ -429,7 +436,10 @@ def run_backup(args) -> None:
# Summary
print()
print(f"Backup complete: {out_path}")
if errors:
print(f"Backup incomplete: {out_path}")
else:
print(f"Backup complete: {out_path}")
print(f" Files: {file_count}")
print(f" Original: {_format_size(total_bytes)}")
print(f" Compressed: {_format_size(zip_size)}")
@ -461,7 +471,8 @@ def run_backup(args) -> None:
if len(errors) > 10:
print(f" ... and {len(errors) - 10} more")
print(f"\nRestore with: hermes import {out_path.name}")
if not errors:
print(f"\nRestore with: hermes import {out_path.name}")
# ---------------------------------------------------------------------------
@ -1215,6 +1226,7 @@ def _write_full_zip_backup(out_path: Path, hermes_root: Path) -> Optional[Path]:
if not files_to_add:
return None
sqlite_snapshot_failed = False
try:
with zipfile.ZipFile(out_path, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as zf:
for abs_path, rel_path in files_to_add:
@ -1229,8 +1241,14 @@ def _write_full_zip_backup(out_path: Path, hermes_root: Path) -> Optional[Path]:
) as tmp:
tmp_db = Path(tmp.name)
try:
if _safe_copy_db(abs_path, tmp_db):
zf.write(tmp_db, arcname=str(rel_path))
if not _safe_copy_db(abs_path, tmp_db):
logger.warning(
"Full-zip backup aborted: SQLite snapshot failed for %s",
rel_path,
)
sqlite_snapshot_failed = True
break
zf.write(tmp_db, arcname=str(rel_path))
finally:
tmp_db.unlink(missing_ok=True)
else:
@ -1247,6 +1265,13 @@ def _write_full_zip_backup(out_path: Path, hermes_root: Path) -> Optional[Path]:
pass
return None
if sqlite_snapshot_failed:
try:
out_path.unlink(missing_ok=True)
except OSError:
pass
return None
return out_path

View file

@ -19,8 +19,10 @@ def _make_hermes_tree(root: Path) -> None:
"""Create a realistic ~/.hermes directory structure for testing."""
(root / "config.yaml").write_text("model:\n provider: openrouter\n")
(root / ".env").write_text("OPENROUTER_API_KEY=sk-test-123\n")
(root / "memory_store.db").write_bytes(b"fake-sqlite")
(root / "hermes_state.db").write_bytes(b"fake-state")
for db_name in ("memory_store.db", "hermes_state.db"):
with sqlite3.connect(root / db_name) as conn:
conn.execute("CREATE TABLE sample (value TEXT)")
conn.execute("INSERT INTO sample VALUES ('test')")
# Sessions
(root / "sessions").mkdir(exist_ok=True)
@ -225,6 +227,65 @@ class TestBackup:
# Skins
assert "skins/cyber.yaml" in names
def test_failed_sqlite_backup_never_raw_copies_live_wal_db(self, tmp_path, monkeypatch, capsys):
"""A failed backup() must not silently archive the stale main DB file.
Keep a real, uncheckpointed WAL transaction live so a raw copy of only
``state.db`` would be a valid-looking but torn snapshot.
"""
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
(hermes_home / "config.yaml").write_text("model: test\n")
db_path = hermes_home / "state.db"
writer = sqlite3.connect(db_path)
writer.execute("PRAGMA journal_mode=WAL")
writer.execute("PRAGMA wal_autocheckpoint=0")
writer.execute("CREATE TABLE events (value TEXT)")
writer.commit()
writer.execute("PRAGMA wal_checkpoint(TRUNCATE)")
writer.execute("INSERT INTO events VALUES ('only-in-wal')")
writer.commit()
assert Path(f"{db_path}-wal").stat().st_size > 0
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
monkeypatch.setattr(Path, "home", lambda: tmp_path)
import hermes_cli.backup as backup_mod
real_connect = backup_mod.sqlite3.connect
class FailingBackupConnection:
def __init__(self, connection):
self._connection = connection
def backup(self, _destination):
raise sqlite3.OperationalError("forced backup failure")
def close(self):
self._connection.close()
def connect_with_failed_backup(database, *args, **kwargs):
connection = real_connect(database, *args, **kwargs)
if str(database).startswith(f"file:{db_path}"):
return FailingBackupConnection(connection)
return connection
monkeypatch.setattr(backup_mod.sqlite3, "connect", connect_with_failed_backup)
out_zip = tmp_path / "backup.zip"
try:
backup_mod.run_backup(Namespace(output=str(out_zip)))
finally:
writer.close()
with zipfile.ZipFile(out_zip) as zf:
assert "config.yaml" in zf.namelist()
assert "state.db" not in zf.namelist()
output = capsys.readouterr().out
assert "Backup incomplete" in output
assert "state.db: SQLite safe copy failed" in output
assert "Restore with:" not in output
def test_db_snapshots_staged_beside_output_zip(self, tmp_path, monkeypatch):
"""SQLite staging temp files must be created on the output zip's
filesystem (dir=out_path.parent), NOT the system /tmp default a
@ -1902,6 +1963,53 @@ class TestPreUpdateBackup:
"""Tests for create_pre_update_backup — the auto-backup ``hermes update``
runs before touching anything."""
def test_failed_sqlite_snapshot_removes_incomplete_archive(self, tmp_path, monkeypatch):
"""The non-interactive full-zip helper must fail the entire archive
rather than return success after omitting a live WAL database."""
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
(hermes_home / "config.yaml").write_text("model: test\n")
db_path = hermes_home / "state.db"
writer = sqlite3.connect(db_path)
writer.execute("PRAGMA journal_mode=WAL")
writer.execute("PRAGMA wal_autocheckpoint=0")
writer.execute("CREATE TABLE events (value TEXT)")
writer.commit()
writer.execute("PRAGMA wal_checkpoint(TRUNCATE)")
writer.execute("INSERT INTO events VALUES ('only-in-wal')")
writer.commit()
assert Path(f"{db_path}-wal").stat().st_size > 0
import hermes_cli.backup as backup_mod
real_connect = backup_mod.sqlite3.connect
class FailingBackupConnection:
def __init__(self, connection):
self._connection = connection
def backup(self, _destination):
raise sqlite3.OperationalError("forced backup failure")
def close(self):
self._connection.close()
def connect_with_failed_backup(database, *args, **kwargs):
connection = real_connect(database, *args, **kwargs)
if str(database).startswith(f"file:{db_path}"):
return FailingBackupConnection(connection)
return connection
monkeypatch.setattr(backup_mod.sqlite3, "connect", connect_with_failed_backup)
out_zip = tmp_path / "pre-update.zip"
try:
result = backup_mod._write_full_zip_backup(out_zip, hermes_home)
finally:
writer.close()
assert result is None
assert not out_zip.exists()
@pytest.fixture
def hermes_home(self, tmp_path):
root = tmp_path / ".hermes"