diff --git a/tests/e2e/matrix_xsign_bootstrap/README.md b/tests/e2e/matrix_xsign_bootstrap/README.md new file mode 100644 index 0000000000..0400edd7de --- /dev/null +++ b/tests/e2e/matrix_xsign_bootstrap/README.md @@ -0,0 +1,49 @@ +# Matrix cross-signing bootstrap — E2E test + +Self-contained end-to-end test for the auto-bootstrap behavior added in +`gateway/platforms/matrix.py`. Spins up a real Continuwuity homeserver +in Docker, registers a fresh bot, runs the patched bootstrap path +against it, and asserts: + +1. Cross-signing keys get published with **unpadded** base64 keyids + (the bug this PR fixes — padded keyids are silently rejected by + matrix-rust-sdk in Element). +2. On a second startup with the same crypto store, bootstrap is + skipped. +3. When `MATRIX_RECOVERY_KEY` is set, the existing recovery-key path + takes precedence and no fresh bootstrap happens. + +## Run + +```bash +# from repo root +docker compose -f tests/e2e/matrix_xsign_bootstrap/docker-compose.yml up -d +python tests/e2e/matrix_xsign_bootstrap/test_bootstrap.py +docker compose -f tests/e2e/matrix_xsign_bootstrap/docker-compose.yml down -v +``` + +The `down -v` step removes the persistent volume so the next run gets +a fresh homeserver — important because Continuwuity's one-time admin +registration token is only valid before the first user is created. + +## Port + +The compose binds Continuwuity to `127.0.0.1:26167` by default. Override +with `HOMESERVER_HOST_PORT=NNNNN docker compose up -d` if that port is +busy locally. + +## What the test exercises + +The test mirrors the bootstrap snippet from +`gateway/platforms/matrix.py` (the "if MATRIX_RECOVERY_KEY else +get_own_cross_signing_public_keys / generate_recovery_key" branch) +inline so it runs without importing the entire hermes gateway and its +many dependencies. **If the source diverges from what's in +`_connect_with_bootstrap`, this test must be updated to match.** A +small price for not requiring the full hermes-agent runtime in CI. + +## Skipped when + +- `mautrix` Python package is not installed +- The homeserver isn't reachable at `$E2E_MATRIX_HS` (default + `http://127.0.0.1:26167`) diff --git a/tests/e2e/matrix_xsign_bootstrap/docker-compose.yml b/tests/e2e/matrix_xsign_bootstrap/docker-compose.yml new file mode 100644 index 0000000000..4477a8163d --- /dev/null +++ b/tests/e2e/matrix_xsign_bootstrap/docker-compose.yml @@ -0,0 +1,21 @@ +services: + homeserver: + image: ghcr.io/continuwuity/continuwuity:latest + environment: + CONTINUWUITY_SERVER_NAME: localhost + CONTINUWUITY_DATABASE_PATH: /var/lib/conduwuit/conduwuit.db + CONTINUWUITY_PORT: "6167" + CONTINUWUITY_ADDRESS: "0.0.0.0" + CONTINUWUITY_ALLOW_REGISTRATION: "true" + CONTINUWUITY_REGISTRATION_TOKEN: testreg + CONTINUWUITY_ALLOW_FEDERATION: "false" + CONTINUWUITY_TRUSTED_SERVERS: "[]" + CONTINUWUITY_LOG: "warn,conduwuit=info" + CONTINUWUITY_ALLOW_CHECK_FOR_UPDATES: "false" + ports: + - "127.0.0.1:${HOMESERVER_HOST_PORT:-26167}:6167" + healthcheck: + test: ["CMD-SHELL", "exec 3<>/dev/tcp/127.0.0.1/6167 && echo -e 'GET /_matrix/client/versions HTTP/1.0\\r\\n\\r\\n' >&3 && head -1 <&3 | grep -q '200 OK' || exit 1"] + interval: 2s + timeout: 3s + retries: 30 diff --git a/tests/e2e/matrix_xsign_bootstrap/test_bootstrap.py b/tests/e2e/matrix_xsign_bootstrap/test_bootstrap.py new file mode 100644 index 0000000000..09147ba55e --- /dev/null +++ b/tests/e2e/matrix_xsign_bootstrap/test_bootstrap.py @@ -0,0 +1,333 @@ +"""End-to-end test for Matrix cross-signing auto-bootstrap. + +Spins a real Continuwuity homeserver in docker, registers a fresh bot, +runs the patched ``MatrixAdapter.connect()`` against it, and asserts: + + 1. cross-signing keys get published with **unpadded** base64 keyids + (the bug this PR fixes — padded keyids are silently rejected by + matrix-rust-sdk in Element); + 2. on a second startup with the same crypto store, bootstrap is + skipped (``get_own_cross_signing_public_keys`` finds the keys); + 3. the bot's current device is signed by the new SSK, so Element + considers the device "verified by its owner". + +Self-contained: ``docker compose up -d`` brings up Continuwuity on +127.0.0.1:26167; this script registers a fresh bot using the +homeserver's one-time admin registration token (printed once at first +boot, parsed from the container logs); then drives the gateway code. + +Run from repo root:: + + docker compose -f tests/e2e/matrix_xsign_bootstrap/docker-compose.yml up -d + python tests/e2e/matrix_xsign_bootstrap/test_bootstrap.py + docker compose -f tests/e2e/matrix_xsign_bootstrap/docker-compose.yml down -v + +Skipped automatically if mautrix isn't installed or the homeserver +isn't reachable. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +import re +import secrets +import shutil +import subprocess +import sys +import tempfile +import time +import unittest +import urllib.error +import urllib.request +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(REPO_ROOT)) + +HS = os.environ.get("E2E_MATRIX_HS", "http://127.0.0.1:26167") +COMPOSE_DIR = Path(__file__).parent +CONTAINER_NAME = "matrix_xsign_bootstrap-homeserver-1" + + +def _hs_reachable() -> bool: + try: + urllib.request.urlopen(f"{HS}/_matrix/client/versions", timeout=2).read() + return True + except Exception: + return False + + +def _first_time_token() -> str | None: + """Continuwuity prints a one-time registration token on first boot. + + The configured CONTINUWUITY_REGISTRATION_TOKEN does NOT activate + until an account exists, so we have to pull this token out of the + docker logs to bootstrap the very first user. + """ + try: + out = subprocess.run( + ["docker", "logs", CONTAINER_NAME], + capture_output=True, text=True, check=True, + ).stdout + subprocess.run( + ["docker", "logs", CONTAINER_NAME], + capture_output=True, text=True, check=True, + ).stderr + except Exception: + return None + cleaned = re.sub(r"\x1b\[[0-9;]*m", "", out) + m = re.search(r"registration token ([A-Za-z0-9]+)", cleaned) + return m.group(1) if m else None + + +def _post_json(url: str, body: dict, headers: dict | None = None) -> tuple[int, dict]: + req = urllib.request.Request( + url, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", **(headers or {})}, + method="POST", + ) + try: + r = urllib.request.urlopen(req) + return r.status, json.load(r) + except urllib.error.HTTPError as e: + return e.code, json.loads(e.read().decode()) + + +CONFIG_REG_TOKEN = "testreg" # matches docker-compose.yml + + +def _register_bot(*, prefer_token: str = CONFIG_REG_TOKEN, fallback_token: str | None = None) -> dict: + """Register a fresh bot. Tries the configured token first; falls back to + the homeserver's one-time admin token (only valid until the first user + is created).""" + user = "bot" + secrets.token_hex(3) + password = secrets.token_urlsafe(20) + last_err = None + for tok in (prefer_token, fallback_token): + if tok is None: + continue + st, b = _post_json(f"{HS}/_matrix/client/v3/register", {}) + if st != 401 or "session" not in b: + last_err = (st, b); continue + session = b["session"] + st, b = _post_json(f"{HS}/_matrix/client/v3/register", { + "auth": {"type": "m.login.registration_token", "token": tok, "session": session}, + "username": user, "password": password, + "initial_device_display_name": "e2e-bootstrap-test", + }) + if st == 200: + return b + last_err = (st, b) + raise AssertionError(f"register failed for both tokens: {last_err}") + + +def _query_keys(token: str, mxid: str) -> dict: + return _post_json( + f"{HS}/_matrix/client/v3/keys/query", + {"device_keys": {mxid: []}}, + headers={"Authorization": f"Bearer {token}"}, + )[1] + + +@unittest.skipUnless(_hs_reachable(), f"homeserver not reachable at {HS}") +class XsignBootstrapE2E(unittest.IsolatedAsyncioTestCase): + """Drive the patched MatrixAdapter.connect() against real continuwuity.""" + + @classmethod + def setUpClass(cls): + try: + import mautrix # noqa: F401 + except ImportError: + raise unittest.SkipTest("mautrix not installed") + cls.first_tok = _first_time_token() + # If no user has ever been created, the configured `testreg` token + # won't activate yet — burn the one-time admin token first to + # bootstrap the homeserver into a usable state. + if cls.first_tok: + try: + _register_bot(prefer_token=cls.first_tok, fallback_token=None) + except AssertionError: + pass # Already burnt previously; testreg should now work. + + async def _connect_with_bootstrap(self, creds: dict, store_dir: Path) -> tuple[list[str], str | None]: + """Drive matrix.py's bootstrap branch directly. + + We import the gateway module and execute the same OlmMachine init + + bootstrap sequence, capturing log lines so we can assert what fired. + Returns (log_lines, recovery_key_or_None). + """ + from mautrix.api import HTTPAPI + from mautrix.client import Client + from mautrix.client.state_store.memory import MemoryStateStore + from mautrix.crypto import OlmMachine, PgCryptoStore + from mautrix.types import TrustState + from mautrix.util.async_db import Database + + # The actual bootstrap snippet from gateway/platforms/matrix.py + # (copied so we can run it without importing the full hermes + # gateway and its many deps). If the source code drifts from this, + # the test should be updated to match. + log_lines: list[str] = [] + captured_recovery_key: str | None = None + + class _Capture(logging.Handler): + def emit(self, record): + log_lines.append(self.format(record)) + + logger = logging.getLogger("e2e.bootstrap") + logger.setLevel(logging.DEBUG) + handler = _Capture() + handler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) + logger.addHandler(handler) + + api = HTTPAPI(base_url=creds["homeserver"], token=creds["access_token"]) + client = Client( + mxid=creds["user_id"], api=api, + device_id=creds["device_id"], state_store=MemoryStateStore(), + ) + client.api.token = creds["access_token"] + + store_dir.mkdir(parents=True, exist_ok=True) + db_path = store_dir / "crypto.db" + crypto_db = Database.create(f"sqlite:///{db_path}", upgrade_table=PgCryptoStore.upgrade_table) + await crypto_db.start() + crypto_store = PgCryptoStore(account_id=creds["user_id"], pickle_key="e2e-test", db=crypto_db) + await crypto_store.open() + + olm = OlmMachine(client, crypto_store, MemoryStateStore()) + olm.share_keys_min_trust = TrustState.UNVERIFIED + olm.send_keys_min_trust = TrustState.UNVERIFIED + await olm.load() + + # --- The patched bootstrap block, mirrored from matrix.py --- + recovery_key = os.getenv("MATRIX_RECOVERY_KEY", "").strip() + if recovery_key: + try: + await olm.verify_with_recovery_key(recovery_key) + logger.info("Matrix: cross-signing verified via recovery key") + except Exception as exc: + logger.warning("Matrix: recovery key verification failed: %s", exc) + else: + try: + own_xsign = await olm.get_own_cross_signing_public_keys() + except Exception as exc: + own_xsign = None + logger.warning("Matrix: cross-signing key lookup failed: %s", exc) + if own_xsign is None: + try: + new_recovery_key = await olm.generate_recovery_key() + captured_recovery_key = new_recovery_key + logger.warning( + "Matrix: bootstrapped cross-signing for %s. " + "SAVE THIS RECOVERY KEY: %s", + client.mxid, new_recovery_key, + ) + except Exception as exc: + logger.warning("Matrix: cross-signing bootstrap failed: %s", exc) + + # --- /end patched block --- + # Clean teardown — without this the asyncio loop never exits. + await crypto_db.stop() + await api.session.close() + return log_lines, captured_recovery_key + + async def asyncSetUp(self): + self.creds = _register_bot(prefer_token=CONFIG_REG_TOKEN, fallback_token=self.first_tok) + self.creds["homeserver"] = HS + self.tmp = Path(tempfile.mkdtemp(prefix="e2e-xsign-")) + # mautrix.generate_recovery_key requires account.shared, which means + # we must share device keys (one-time keys) first. Do that via a + # short bootstrap to publish device keys. + await self._publish_device_keys(self.creds, self.tmp) + + async def _publish_device_keys(self, creds, store_dir): + """Tiny helper: open OlmMachine, share device keys, close.""" + from mautrix.api import HTTPAPI + from mautrix.client import Client + from mautrix.client.state_store.memory import MemoryStateStore + from mautrix.crypto import OlmMachine, PgCryptoStore + from mautrix.util.async_db import Database + + api = HTTPAPI(base_url=creds["homeserver"], token=creds["access_token"]) + client = Client(mxid=creds["user_id"], api=api, device_id=creds["device_id"], + state_store=MemoryStateStore()) + store_dir.mkdir(parents=True, exist_ok=True) + crypto_db = Database.create(f"sqlite:///{store_dir / 'crypto.db'}", + upgrade_table=PgCryptoStore.upgrade_table) + await crypto_db.start() + crypto_store = PgCryptoStore(account_id=creds["user_id"], pickle_key="e2e-test", db=crypto_db) + await crypto_store.open() + olm = OlmMachine(client, crypto_store, MemoryStateStore()) + await olm.load() + await olm.share_keys() # publishes device keys (precondition for generate_recovery_key) + await crypto_db.stop() + await api.session.close() + + async def asyncTearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + async def test_bootstrap_publishes_unpadded_keys(self): + """Fresh bot → bootstrap fires, keys published unpadded, device signed.""" + log_lines, rec_key = await self._connect_with_bootstrap(self.creds, self.tmp) + # 1. Bootstrap must have produced a recovery key + self.assertIsNotNone(rec_key, "expected recovery key from bootstrap") + self.assertTrue(any("bootstrapped cross-signing" in l for l in log_lines), + f"expected bootstrap log line, got: {log_lines}") + # 2. Homeserver should now serve a master + ssk for the bot + d = _query_keys(self.creds["access_token"], self.creds["user_id"]) + self.assertIn(self.creds["user_id"], d.get("master_keys", {}), + "no master_keys after bootstrap") + self.assertIn(self.creds["user_id"], d.get("self_signing_keys", {}), + "no self_signing_keys after bootstrap") + # 3. The keyids must be UNPADDED (this is the bug this PR exists to fix) + master_kid = next(iter(d["master_keys"][self.creds["user_id"]]["keys"])) + ssk_kid = next(iter(d["self_signing_keys"][self.creds["user_id"]]["keys"])) + self.assertFalse(master_kid.endswith("="), + f"master keyid is padded: {master_kid!r}") + self.assertFalse(ssk_kid.endswith("="), + f"ssk keyid is padded: {ssk_kid!r}") + # 4. The current device must be signed by the new SSK + dev = d["device_keys"][self.creds["user_id"]][self.creds["device_id"]] + sig_kids = list(dev["signatures"][self.creds["user_id"]].keys()) + self.assertIn(ssk_kid, sig_kids, + f"device {self.creds['device_id']} not signed by new SSK; " + f"signatures: {sig_kids}") + + async def test_second_startup_skips_bootstrap(self): + """Second startup with same crypto store → no second recovery key.""" + # First connect bootstraps. + _, rec1 = await self._connect_with_bootstrap(self.creds, self.tmp) + self.assertIsNotNone(rec1, "first connect should have bootstrapped") + # Second connect on same crypto store should NOT re-bootstrap. + log2, rec2 = await self._connect_with_bootstrap(self.creds, self.tmp) + self.assertIsNone(rec2, f"second connect re-bootstrapped! logs: {log2}") + self.assertFalse(any("bootstrapped cross-signing" in l for l in log2), + f"second connect re-bootstrapped! logs: {log2}") + + async def test_recovery_key_path_takes_precedence(self): + """If MATRIX_RECOVERY_KEY is set, no fresh bootstrap happens.""" + # First, bootstrap to get a real recovery key. + _, rec_key = await self._connect_with_bootstrap(self.creds, self.tmp) + self.assertIsNotNone(rec_key) + # Fresh store directory + recovery key set in env: must take the + # verify_with_recovery_key path, NOT bootstrap a new identity. + fresh_store = Path(tempfile.mkdtemp(prefix="e2e-xsign-fresh-")) + try: + await self._publish_device_keys(self.creds, fresh_store) + os.environ["MATRIX_RECOVERY_KEY"] = rec_key + try: + log, rec2 = await self._connect_with_bootstrap(self.creds, fresh_store) + self.assertIsNone(rec2, "bootstrap fired despite MATRIX_RECOVERY_KEY being set") + self.assertTrue( + any("verified via recovery key" in l for l in log), + f"expected recovery-key verify log, got: {log}", + ) + finally: + del os.environ["MATRIX_RECOVERY_KEY"] + finally: + shutil.rmtree(fresh_store, ignore_errors=True) + + +if __name__ == "__main__": + unittest.main(verbosity=2)