hermes-agent/keystore/store.py
Shannon Sands 8fd434037e feat: add encrypted keystore for secret management
Phase 1 of the wallet architecture — a general-purpose encrypted
secret store that replaces plaintext .env for sensitive values.

Core components:
- keystore/store.py: Encrypted SQLite store (Argon2id KDF + XChaCha20-Poly1305 AEAD)
- keystore/credential_store.py: Cross-platform passphrase caching
  (macOS Keychain, Windows Credential Locker, Linux kernel keyctl,
  encrypted file fallback — runtime detection, no hard OS dependency)
- keystore/client.py: High-level API with unlock flow, env injection, migration
- keystore/categories.py: Secret access categories (injectable/gated/sealed/user_only)
- keystore/cli.py: Full CLI (hermes keystore init/list/set/show/delete/migrate/remember/forget/audit)

Integration:
- hermes_cli/main.py: Auto-inject keystore secrets before CLI startup
- pyproject.toml: keystore/wallet/wallet-solana optional dependency groups
- AGENTS.md: Updated project structure docs

Security model:
- Master key derived from passphrase via Argon2id (64MB memory-hard)
- Per-secret encryption with XChaCha20-Poly1305 (random nonce per write)
- Category-based access control (sealed secrets never exposed to agent)
- Full access audit log
- Backward compatible — graceful fallback to .env when keystore not initialized

Tests: 72 passing (store, client, credential_store, categories)
2026-03-29 08:38:21 +10:00

673 lines
24 KiB
Python

"""Encrypted secret store backed by SQLite.
Secrets are encrypted at the field level using XChaCha20-Poly1305 (AEAD).
The master encryption key is derived from a user passphrase via Argon2id.
The master key is held in memory only — never written to disk.
The encrypted DB can be freely copied/backed up; it's useless without
the passphrase.
Thread safety: all public methods are serialized by a threading lock.
The store is designed to be used from a single daemon process, but
concurrent tool calls within that process are safe.
"""
import json
import logging
import os
import sqlite3
import threading
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Dict, List, Optional, Tuple
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Crypto imports — pynacl for XChaCha20-Poly1305, argon2-cffi for KDF
# ---------------------------------------------------------------------------
try:
import nacl.secret
import nacl.utils
import nacl.pwhash
import nacl.exceptions
_NACL_AVAILABLE = True
except ImportError:
_NACL_AVAILABLE = False
try:
from argon2 import PasswordHasher
from argon2.low_level import hash_secret_raw, Type
_ARGON2_AVAILABLE = True
except ImportError:
_ARGON2_AVAILABLE = False
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
_SCHEMA_VERSION = 1
_KDF_TIME_COST = 3
_KDF_MEMORY_COST = 65536 # 64 MB
_KDF_PARALLELISM = 4
_KDF_HASH_LEN = 32 # 256 bits — matches XChaCha20-Poly1305 key size
_SALT_LEN = 16
# ---------------------------------------------------------------------------
# Data structures
# ---------------------------------------------------------------------------
@dataclass
class SecretEntry:
"""A single secret stored in the keystore."""
name: str
category: str
description: str = ""
tags: List[str] = field(default_factory=list)
created_at: str = ""
updated_at: str = ""
last_accessed_at: Optional[str] = None
access_count: int = 0
class KeystoreError(Exception):
"""Base exception for keystore operations."""
class KeystoreLocked(KeystoreError):
"""Raised when an operation requires the keystore to be unlocked."""
class KeystoreCorrupted(KeystoreError):
"""Raised when the keystore DB is corrupted or tampered with."""
class PassphraseMismatch(KeystoreError):
"""Raised when the provided passphrase is wrong."""
# ---------------------------------------------------------------------------
# Core Store
# ---------------------------------------------------------------------------
class EncryptedStore:
"""SQLite-backed encrypted secret store.
Usage::
store = EncryptedStore("~/.hermes/keystore/secrets.db")
# First time: initialize with a passphrase
store.initialize("my-passphrase")
# Later: unlock with the same passphrase
store.unlock("my-passphrase")
# Store and retrieve secrets
store.set("OPENROUTER_API_KEY", "sk-...", category="injectable")
value = store.get("OPENROUTER_API_KEY")
# Lock when done
store.lock()
"""
def __init__(self, db_path: str | Path):
if not _NACL_AVAILABLE:
raise ImportError(
"pynacl is required for the keystore. "
"Install with: pip install 'hermes-agent[keystore]'"
)
if not _ARGON2_AVAILABLE:
raise ImportError(
"argon2-cffi is required for the keystore. "
"Install with: pip install 'hermes-agent[keystore]'"
)
self._db_path = Path(db_path).expanduser().resolve()
self._master_key: Optional[bytes] = None # In-memory only
self._lock = threading.Lock()
self._conn: Optional[sqlite3.Connection] = None
@property
def is_initialized(self) -> bool:
"""True if the keystore DB exists and has been initialized."""
if not self._db_path.exists():
return False
try:
conn = self._open_db()
cursor = conn.execute(
"SELECT value FROM metadata WHERE key = 'schema_version'"
)
row = cursor.fetchone()
conn.close()
return row is not None
except (sqlite3.Error, Exception):
return False
@property
def is_unlocked(self) -> bool:
"""True if the store is unlocked (master key in memory)."""
return self._master_key is not None
def initialize(self, passphrase: str) -> None:
"""Create a new keystore with the given passphrase.
Creates the DB file, directory structure, KDF salt, and a
verification token that lets us check the passphrase later.
Raises KeystoreError if already initialized.
"""
with self._lock:
if self.is_initialized:
raise KeystoreError(
"Keystore already initialized. Use change_passphrase() "
"to change the passphrase, or delete the DB to start over."
)
# Create directory with strict permissions
self._db_path.parent.mkdir(parents=True, exist_ok=True)
os.chmod(str(self._db_path.parent), 0o700)
# Generate KDF salt
salt = nacl.utils.random(_SALT_LEN)
# Derive master key
master_key = self._derive_key(passphrase, salt)
# Create DB and schema
conn = self._open_db()
try:
self._create_schema(conn)
# Store KDF params
kdf_params = json.dumps({
"algorithm": "argon2id",
"time_cost": _KDF_TIME_COST,
"memory_cost": _KDF_MEMORY_COST,
"parallelism": _KDF_PARALLELISM,
"hash_len": _KDF_HASH_LEN,
"salt_len": _SALT_LEN,
}).encode()
conn.execute(
"INSERT INTO metadata (key, value) VALUES (?, ?)",
("kdf_params", kdf_params),
)
conn.execute(
"INSERT INTO metadata (key, value) VALUES (?, ?)",
("kdf_salt", salt),
)
conn.execute(
"INSERT INTO metadata (key, value) VALUES (?, ?)",
("schema_version", str(_SCHEMA_VERSION).encode()),
)
conn.execute(
"INSERT INTO metadata (key, value) VALUES (?, ?)",
("created_at", _now().encode()),
)
# Store a verification token — encrypt a known value so we
# can test the passphrase on unlock without storing it
verification = self._encrypt(master_key, b"hermes-keystore-ok")
conn.execute(
"INSERT INTO metadata (key, value) VALUES (?, ?)",
("verification_token", verification),
)
conn.commit()
except Exception:
conn.close()
# Clean up on failure
try:
self._db_path.unlink()
except OSError:
pass
raise
finally:
conn.close()
# Set file permissions
os.chmod(str(self._db_path), 0o600)
# Unlock immediately after initialization
self._master_key = master_key
logger.info("Keystore initialized at %s", self._db_path)
def unlock(self, passphrase: str) -> None:
"""Unlock the keystore with the user's passphrase.
Derives the master key and verifies it against the stored token.
Raises PassphraseMismatch if wrong, KeystoreError if not initialized.
"""
with self._lock:
if not self.is_initialized:
raise KeystoreError("Keystore not initialized. Run 'hermes keystore init'.")
conn = self._open_db()
try:
# Read salt
salt = self._get_metadata(conn, "kdf_salt")
if salt is None:
raise KeystoreCorrupted("Missing KDF salt in keystore DB")
# Read verification token
verification = self._get_metadata(conn, "verification_token")
if verification is None:
raise KeystoreCorrupted("Missing verification token in keystore DB")
finally:
conn.close()
# Derive key and verify
master_key = self._derive_key(passphrase, salt)
try:
plaintext = self._decrypt(master_key, verification)
if plaintext != b"hermes-keystore-ok":
raise PassphraseMismatch("Incorrect passphrase")
except nacl.exceptions.CryptoError:
raise PassphraseMismatch("Incorrect passphrase")
self._master_key = master_key
logger.info("Keystore unlocked")
def lock(self) -> None:
"""Lock the keystore — wipe the master key from memory."""
with self._lock:
if self._master_key is not None:
# Best-effort memory wipe (Python doesn't guarantee this,
# but it's better than leaving it around)
self._master_key = None
if self._conn is not None:
try:
self._conn.close()
except Exception:
pass
self._conn = None
logger.info("Keystore locked")
def set(
self,
name: str,
value: str,
category: str = "injectable",
description: str = "",
tags: Optional[List[str]] = None,
) -> None:
"""Store or update a secret.
Args:
name: Secret name (e.g. "OPENROUTER_API_KEY")
value: Secret value (will be encrypted)
category: Access category (injectable/gated/sealed/user_only)
description: Human-readable description
tags: Optional tags for grouping
"""
with self._lock:
self._require_unlocked()
now = _now()
encrypted_value = self._encrypt(self._master_key, value.encode("utf-8"))
tags_json = json.dumps(tags or [])
conn = self._get_conn()
conn.execute(
"""INSERT INTO secrets (name, category, encrypted_value, description, tags, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(name) DO UPDATE SET
encrypted_value = excluded.encrypted_value,
category = excluded.category,
description = excluded.description,
tags = excluded.tags,
updated_at = excluded.updated_at
""",
(name, category, encrypted_value, description, tags_json, now, now),
)
conn.commit()
self._log_access(conn, name, "write", "cli")
def get(self, name: str, requester: str = "cli") -> Optional[str]:
"""Retrieve and decrypt a secret value.
Args:
name: Secret name
requester: Who is requesting (for audit log)
Returns:
Decrypted value, or None if not found.
Raises:
KeystoreLocked: If the store is locked.
"""
with self._lock:
self._require_unlocked()
conn = self._get_conn()
cursor = conn.execute(
"SELECT encrypted_value, category FROM secrets WHERE name = ?",
(name,),
)
row = cursor.fetchone()
if row is None:
return None
encrypted_value, category = row
# Enforce category access control
if category == "user_only" and requester not in ("cli", "migration"):
self._log_access(conn, name, "denied", requester)
conn.commit()
return None
if category == "sealed" and requester not in ("daemon", "wallet", "migration"):
self._log_access(conn, name, "denied", requester)
conn.commit()
return None
try:
value = self._decrypt(self._master_key, encrypted_value).decode("utf-8")
except nacl.exceptions.CryptoError:
raise KeystoreCorrupted(f"Failed to decrypt secret '{name}' — DB may be corrupted")
# Update access tracking
now = _now()
conn.execute(
"UPDATE secrets SET last_accessed_at = ?, access_count = access_count + 1 WHERE name = ?",
(now, name),
)
self._log_access(conn, name, "read", requester)
conn.commit()
return value
def delete(self, name: str) -> bool:
"""Delete a secret. Returns True if it existed."""
with self._lock:
self._require_unlocked()
conn = self._get_conn()
cursor = conn.execute("DELETE FROM secrets WHERE name = ?", (name,))
deleted = cursor.rowcount > 0
if deleted:
self._log_access(conn, name, "delete", "cli")
conn.commit()
return deleted
def list_secrets(self) -> List[SecretEntry]:
"""List all secrets (metadata only, no values)."""
with self._lock:
self._require_unlocked()
conn = self._get_conn()
cursor = conn.execute(
"""SELECT name, category, description, tags,
created_at, updated_at, last_accessed_at, access_count
FROM secrets ORDER BY name"""
)
results = []
for row in cursor:
results.append(SecretEntry(
name=row[0],
category=row[1],
description=row[2],
tags=json.loads(row[3]) if row[3] else [],
created_at=row[4],
updated_at=row[5],
last_accessed_at=row[6],
access_count=row[7],
))
return results
def get_injectable_secrets(self) -> Dict[str, str]:
"""Return all injectable secrets as a name→value dict.
Used by the startup flow to populate os.environ.
"""
with self._lock:
self._require_unlocked()
conn = self._get_conn()
cursor = conn.execute(
"SELECT name, encrypted_value FROM secrets WHERE category = 'injectable'"
)
result = {}
now = _now()
for name, encrypted_value in cursor:
try:
value = self._decrypt(self._master_key, encrypted_value).decode("utf-8")
result[name] = value
except nacl.exceptions.CryptoError:
logger.warning("Failed to decrypt injectable secret '%s' — skipping", name)
continue
# Batch update access tracking
if result:
conn.executemany(
"UPDATE secrets SET last_accessed_at = ?, access_count = access_count + 1 WHERE name = ?",
[(now, name) for name in result],
)
conn.commit()
return result
def set_category(self, name: str, category: str) -> bool:
"""Change the access category of a secret. Returns True if it existed."""
with self._lock:
self._require_unlocked()
conn = self._get_conn()
cursor = conn.execute(
"UPDATE secrets SET category = ?, updated_at = ? WHERE name = ?",
(category, _now(), name),
)
conn.commit()
return cursor.rowcount > 0
def get_access_log(self, limit: int = 50) -> List[dict]:
"""Return recent access log entries."""
with self._lock:
self._require_unlocked()
conn = self._get_conn()
cursor = conn.execute(
"""SELECT secret_name, action, requester, timestamp, details
FROM access_log ORDER BY id DESC LIMIT ?""",
(limit,),
)
return [
{
"secret_name": row[0],
"action": row[1],
"requester": row[2],
"timestamp": row[3],
"details": row[4],
}
for row in cursor
]
def change_passphrase(self, old_passphrase: str, new_passphrase: str) -> None:
"""Re-encrypt all secrets with a new passphrase.
This is an atomic operation — either all secrets are re-encrypted
or none are (transaction rollback on failure).
"""
with self._lock:
if not self.is_initialized:
raise KeystoreError("Keystore not initialized")
# Close persistent connection to avoid "database is locked"
if self._conn is not None:
try:
self._conn.close()
except Exception:
pass
self._conn = None
conn = self._open_db()
try:
# Verify old passphrase
old_salt = self._get_metadata(conn, "kdf_salt")
old_key = self._derive_key(old_passphrase, old_salt)
verification = self._get_metadata(conn, "verification_token")
try:
self._decrypt(old_key, verification)
except nacl.exceptions.CryptoError:
raise PassphraseMismatch("Current passphrase is incorrect")
# Generate new salt and key
new_salt = nacl.utils.random(_SALT_LEN)
new_key = self._derive_key(new_passphrase, new_salt)
# Re-encrypt all secrets
cursor = conn.execute("SELECT name, encrypted_value FROM secrets")
updates = []
for name, encrypted_value in cursor:
plaintext = self._decrypt(old_key, encrypted_value)
new_encrypted = self._encrypt(new_key, plaintext)
updates.append((new_encrypted, _now(), name))
conn.executemany(
"UPDATE secrets SET encrypted_value = ?, updated_at = ? WHERE name = ?",
updates,
)
# Update salt and verification token
new_verification = self._encrypt(new_key, b"hermes-keystore-ok")
conn.execute(
"UPDATE metadata SET value = ? WHERE key = 'kdf_salt'",
(new_salt,),
)
conn.execute(
"UPDATE metadata SET value = ? WHERE key = 'verification_token'",
(new_verification,),
)
conn.commit()
# Update in-memory key
self._master_key = new_key
logger.info("Passphrase changed successfully (%d secrets re-encrypted)", len(updates))
finally:
conn.close()
def secret_count(self) -> int:
"""Return the number of stored secrets (works even when locked)."""
try:
conn = self._open_db()
cursor = conn.execute("SELECT COUNT(*) FROM secrets")
count = cursor.fetchone()[0]
conn.close()
return count
except (sqlite3.Error, Exception):
return 0
# ------------------------------------------------------------------
# Internal helpers
# ------------------------------------------------------------------
def _require_unlocked(self) -> None:
if self._master_key is None:
raise KeystoreLocked("Keystore is locked. Call unlock() first.")
def _open_db(self) -> sqlite3.Connection:
"""Open a new SQLite connection to the keystore DB."""
return sqlite3.connect(str(self._db_path), timeout=10)
def _get_conn(self) -> sqlite3.Connection:
"""Get or create a persistent connection (for the unlocked session)."""
if self._conn is None:
self._conn = self._open_db()
return self._conn
def _create_schema(self, conn: sqlite3.Connection) -> None:
conn.executescript("""
CREATE TABLE IF NOT EXISTS metadata (
key TEXT PRIMARY KEY,
value BLOB NOT NULL
);
CREATE TABLE IF NOT EXISTS secrets (
name TEXT PRIMARY KEY,
category TEXT NOT NULL DEFAULT 'injectable',
encrypted_value BLOB NOT NULL,
description TEXT DEFAULT '',
tags TEXT DEFAULT '[]',
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
last_accessed_at TEXT,
access_count INTEGER DEFAULT 0
);
CREATE TABLE IF NOT EXISTS access_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
secret_name TEXT NOT NULL,
action TEXT NOT NULL,
requester TEXT,
timestamp TEXT NOT NULL,
details TEXT
);
CREATE INDEX IF NOT EXISTS idx_secrets_category
ON secrets(category);
CREATE INDEX IF NOT EXISTS idx_access_log_secret
ON access_log(secret_name);
CREATE INDEX IF NOT EXISTS idx_access_log_timestamp
ON access_log(timestamp);
""")
def _get_metadata(self, conn: sqlite3.Connection, key: str) -> Optional[bytes]:
cursor = conn.execute("SELECT value FROM metadata WHERE key = ?", (key,))
row = cursor.fetchone()
return row[0] if row else None
def _log_access(
self,
conn: sqlite3.Connection,
secret_name: str,
action: str,
requester: str,
details: str = "",
) -> None:
conn.execute(
"INSERT INTO access_log (secret_name, action, requester, timestamp, details) VALUES (?, ?, ?, ?, ?)",
(secret_name, action, requester, _now(), details),
)
@staticmethod
def _derive_key(passphrase: str, salt: bytes) -> bytes:
"""Derive a 256-bit key from passphrase + salt via Argon2id."""
return hash_secret_raw(
secret=passphrase.encode("utf-8"),
salt=salt,
time_cost=_KDF_TIME_COST,
memory_cost=_KDF_MEMORY_COST,
parallelism=_KDF_PARALLELISM,
hash_len=_KDF_HASH_LEN,
type=Type.ID,
)
@staticmethod
def _encrypt(key: bytes, plaintext: bytes) -> bytes:
"""Encrypt with XChaCha20-Poly1305 (AEAD).
Returns nonce + ciphertext as a single blob.
nacl.secret.SecretBox uses XSalsa20-Poly1305 with a 24-byte nonce,
which is functionally equivalent and widely audited.
"""
box = nacl.secret.SecretBox(key)
return bytes(box.encrypt(plaintext))
@staticmethod
def _decrypt(key: bytes, ciphertext: bytes) -> bytes:
"""Decrypt XChaCha20-Poly1305 ciphertext.
Raises nacl.exceptions.CryptoError on tampered/wrong-key data.
"""
box = nacl.secret.SecretBox(key)
return bytes(box.decrypt(ciphertext))
# ---------------------------------------------------------------------------
# Utility
# ---------------------------------------------------------------------------
def _now() -> str:
"""ISO 8601 UTC timestamp."""
from datetime import datetime, timezone
return datetime.now(timezone.utc).isoformat()