From e79d316a043ce188be97c8e2c4de2ade1d7253ac Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:12:20 -0700 Subject: [PATCH] fix(photon): persist sidecar runtime record so cron/standalone sends work The sidecar auth token is generated at spawn (secrets.token_hex) and existed only in the gateway process memory + sidecar child env, so _standalone_send from cron subprocesses, hermes send, or the dashboard structurally could not authenticate (#69960). The adapter now writes /runtime/photon-sidecar.json ({port, token, pid}, 0600, atomic tempfile+os.replace) once the sidecar passes its /healthz readiness check, and deletes it in _stop_sidecar, on every startup-failure path, and at disconnect so a stale record never outlives a dead sidecar. _standalone_send falls back to the record when PHOTON_SIDECAR_TOKEN is unset, validating the recorded pid is alive first; a stale record yields a clear 'gateway appears to be down' error. Docs note the gateway-must-be-running requirement and the Photon-side shared-line initiation policy (#51897). --- plugins/platforms/photon/README.md | 8 + plugins/platforms/photon/adapter.py | 149 +++++++- .../platforms/photon/test_runtime_record.py | 326 ++++++++++++++++++ website/docs/user-guide/messaging/photon.md | 13 + 4 files changed, 489 insertions(+), 7 deletions(-) create mode 100644 tests/plugins/platforms/photon/test_runtime_record.py diff --git a/plugins/platforms/photon/README.md b/plugins/platforms/photon/README.md index 91680ebd1a6c..c1f558ad6e5f 100644 --- a/plugins/platforms/photon/README.md +++ b/plugins/platforms/photon/README.md @@ -155,6 +155,14 @@ All env vars are documented in `plugin.yaml`. The most important: reachable across restarts via spectrum-ts' `space.get(id)`. - **Message effects, polls** — supported by `spectrum-ts` but not yet exposed; the sidecar is the natural place to add them. +- **Cron/standalone sends require a running gateway.** Processes outside + the gateway (cron subprocesses, `hermes send`) cannot spawn the sidecar; + they authenticate to the gateway's live sidecar via the runtime record at + `/runtime/photon-sidecar.json` (written after the sidecar's + `/healthz` readiness check, `0600`, removed on stop/failed start). Also + note that shared/free-tier Photon lines cannot INITIATE conversations + with numbers that never texted the line — that's Photon-side policy, not + a Hermes limitation. ## Upgrading spectrum-ts diff --git a/plugins/platforms/photon/adapter.py b/plugins/platforms/photon/adapter.py index ed3e966d58da..39f27db22920 100644 --- a/plugins/platforms/photon/adapter.py +++ b/plugins/platforms/photon/adapter.py @@ -78,6 +78,103 @@ _DEFAULT_SIDECAR_BIND = "127.0.0.1" # size to ~16 KB. Keep a conservative cap that matches BlueBubbles. _MAX_MESSAGE_LENGTH = 8000 +# --------------------------------------------------------------------------- +# Sidecar runtime record +# +# Out-of-process senders (cron subprocesses, `hermes send`, the dashboard) +# go through ``_standalone_send`` and need the live sidecar's port + token — +# but the token is generated at spawn time and otherwise exists only in the +# gateway process memory and the sidecar child env (issue #69960). The +# gateway persists this record once the sidecar passes its /healthz +# readiness check, and removes it on every stop / failed-start path so a +# stale record never outlives a dead sidecar. + +_RUNTIME_RECORD_NAME = "photon-sidecar.json" + + +def _runtime_record_path() -> Path: + # get_hermes_home() honors profile overrides — never hardcode ~/.hermes. + from hermes_constants import get_hermes_home + + return get_hermes_home() / "runtime" / _RUNTIME_RECORD_NAME + + +def _write_runtime_record(port: int, token: str, pid: int) -> None: + """Atomically persist ``{port, token, pid}`` with owner-only perms.""" + import tempfile + + try: + path = _runtime_record_path() + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp( + dir=str(path.parent), prefix=".photon-sidecar.", suffix=".tmp" + ) + try: + # Restrict perms BEFORE the token hits disk (mkstemp is already + # 0600 on POSIX; the explicit chmod is a belt-and-braces guard, + # wrapped so Windows/exotic filesystems can't crash the write). + try: + os.chmod(tmp, 0o600) + except OSError: # pragma: no cover - Windows / odd fs + pass + with os.fdopen(fd, "w", encoding="utf-8") as fh: + json.dump({"port": port, "token": token, "pid": pid}, fh) + os.replace(tmp, path) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise + except Exception as e: # best-effort: gateway sends still work without it + logger.warning("[photon] failed to write sidecar runtime record: %s", e) + + +def _read_runtime_record() -> Optional[Dict[str, Any]]: + try: + raw = json.loads(_runtime_record_path().read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + return raw if isinstance(raw, dict) else None + + +def _delete_runtime_record() -> None: + try: + _runtime_record_path().unlink(missing_ok=True) + except OSError: + pass + + +def _sidecar_pid_alive(pid: Any) -> bool: + """Best-effort liveness check for the recorded sidecar pid.""" + try: + pid_int = int(pid) + except (TypeError, ValueError): + return False + if pid_int <= 0: + return False + try: + # Cross-platform (psutil-backed, Windows-safe — see its docstring). + from gateway.status import _pid_exists + + return bool(_pid_exists(pid_int)) + except Exception: + pass + if os.name == "posix": + try: + os.kill(pid_int, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return False + return True + # Windows without gateway.status/psutil: can't probe safely + # (os.kill(pid, 0) is destructive there) — assume alive and let the + # HTTP send itself be the arbiter. + return True + # Dedup parameters — the gRPC stream is at-least-once, and a sidecar # reconnect can replay, so keep at least 1k ids for ~48h. _DEDUP_MAX_SIZE = 4000 @@ -499,6 +596,9 @@ class PhotonAdapter(BasePlatformAdapter): f"failed to start Photon sidecar: {e}", retryable=True, ) + # No live sidecar — make sure no stale runtime record + # survives to mislead standalone senders. + _delete_runtime_record() await client.aclose() self._http_client = None return False @@ -1110,6 +1210,7 @@ class PhotonAdapter(BasePlatformAdapter): async with httpx.AsyncClient(timeout=2.0, trust_env=False) as client: while time.time() < deadline: if self._sidecar_proc.poll() is not None: + _delete_runtime_record() raise RuntimeError( f"Photon sidecar exited with code " f"{self._sidecar_proc.returncode} before becoming ready" @@ -1120,10 +1221,19 @@ class PhotonAdapter(BasePlatformAdapter): headers={"X-Hermes-Sidecar-Token": self._sidecar_token}, ) if resp.status_code == 200: + # Persist port/token/pid so out-of-process senders + # (cron, `hermes send`) can reach this sidecar + # (see _standalone_send / issue #69960). + _write_runtime_record( + self._sidecar_port, + self._sidecar_token, + self._sidecar_proc.pid, + ) return except httpx.RequestError as e: last_err = e await asyncio.sleep(0.2) + _delete_runtime_record() raise RuntimeError( f"Photon sidecar did not become ready within 15s: {last_err}" ) @@ -1161,6 +1271,9 @@ class PhotonAdapter(BasePlatformAdapter): async def _stop_sidecar(self) -> None: proc = self._sidecar_proc if proc is None: + # Nothing to stop, but never leave a record behind on disconnect + # (e.g. autostart was disabled or startup failed earlier). + _delete_runtime_record() return try: # Closing our end of the stdin pipe is itself a shutdown signal @@ -1197,6 +1310,9 @@ class PhotonAdapter(BasePlatformAdapter): proc.kill() finally: self._sidecar_proc = None + # The sidecar is gone (or going) — remove the runtime record so + # standalone senders don't chase a dead port/token. + _delete_runtime_record() if self._sidecar_supervisor_task is not None: self._sidecar_supervisor_task.cancel() self._sidecar_supervisor_task = None @@ -1799,13 +1915,32 @@ async def _standalone_send( ) token = os.getenv("PHOTON_SIDECAR_TOKEN") if not token: - return { - "error": ( - "Photon standalone send requires a running sidecar with " - "PHOTON_SIDECAR_TOKEN set in the environment. Cron processes " - "cannot spawn the sidecar themselves." - ) - } + # Fall back to the runtime record the gateway persists once its + # sidecar passes /healthz (issue #69960) — the token only exists in + # the gateway process env otherwise, so cron/`hermes send` would be + # structurally unable to authenticate. + record = _read_runtime_record() + stale_hint = "" + if record and record.get("token"): + if _sidecar_pid_alive(record.get("pid")): + token = str(record["token"]) + port = _coerce_port(record.get("port"), port) + else: + stale_hint = ( + " A stale sidecar runtime record was found (pid " + f"{record.get('pid')} is not running) — the gateway " + "appears to be down." + ) + if not token: + return { + "error": ( + "Photon standalone send requires a running sidecar. " + "Start the Hermes gateway (which spawns the sidecar and " + "records its address under /runtime/" + f"{_RUNTIME_RECORD_NAME}), or set PHOTON_SIDECAR_TOKEN " + "in this process's environment." + stale_hint + ) + } base = f"http://{_DEFAULT_SIDECAR_BIND}:{port}" headers = {"X-Hermes-Sidecar-Token": token} last_message_id: Optional[str] = None diff --git a/tests/plugins/platforms/photon/test_runtime_record.py b/tests/plugins/platforms/photon/test_runtime_record.py new file mode 100644 index 000000000000..1b1e650cfc5e --- /dev/null +++ b/tests/plugins/platforms/photon/test_runtime_record.py @@ -0,0 +1,326 @@ +"""Sidecar runtime-record persistence tests (issue #69960). + +The sidecar token is generated at spawn and used to exist only in the +gateway process memory + sidecar child env — so cron/`hermes send` +standalone sends structurally could not authenticate. The adapter now +persists ``/runtime/photon-sidecar.json`` after the sidecar +passes its /healthz readiness check, deletes it on stop/failed-start, and +``_standalone_send`` falls back to it when PHOTON_SIDECAR_TOKEN is unset. +No Node, no ports, no network. +""" +from __future__ import annotations + +import asyncio +import json +import os +import stat +import sys +from pathlib import Path +from typing import Any + +import pytest + +from gateway.config import PlatformConfig +from plugins.platforms.photon import adapter as photon_adapter +from plugins.platforms.photon.adapter import PhotonAdapter + + +@pytest.fixture() +def record_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + path = tmp_path / "runtime" / "photon-sidecar.json" + monkeypatch.setattr(photon_adapter, "_runtime_record_path", lambda: path) + return path + + +def _make_adapter(monkeypatch: pytest.MonkeyPatch) -> PhotonAdapter: + monkeypatch.setenv("PHOTON_PROJECT_ID", "test-project-id") + monkeypatch.setenv("PHOTON_PROJECT_SECRET", "test-project-secret") + monkeypatch.delenv("PHOTON_SIDECAR_TOKEN", raising=False) + cfg = PlatformConfig(enabled=True, token="", extra={}) + return PhotonAdapter(cfg) + + +# -- record helpers ---------------------------------------------------------- + + +def test_write_read_delete_roundtrip(record_path: Path) -> None: + photon_adapter._write_runtime_record(8789, "tok123", 4242) + + assert record_path.exists() + data = json.loads(record_path.read_text(encoding="utf-8")) + assert data == {"port": 8789, "token": "tok123", "pid": 4242} + assert photon_adapter._read_runtime_record() == data + + photon_adapter._delete_runtime_record() + assert not record_path.exists() + # Idempotent: deleting a missing record must not raise. + photon_adapter._delete_runtime_record() + assert photon_adapter._read_runtime_record() is None + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits") +def test_record_written_with_0600(record_path: Path) -> None: + photon_adapter._write_runtime_record(8789, "secret", 1) + mode = stat.S_IMODE(record_path.stat().st_mode) + assert mode == 0o600 + + +def test_read_tolerates_corrupt_record(record_path: Path) -> None: + record_path.parent.mkdir(parents=True, exist_ok=True) + record_path.write_text("{not json", encoding="utf-8") + assert photon_adapter._read_runtime_record() is None + + +# -- lifecycle: written after healthz success, removed on stop/failure ------- + + +class _HealthzClient: + """Fake httpx.AsyncClient whose /healthz response is injectable.""" + + status_code = 200 + + def __init__(self, *a: Any, **k: Any) -> None: + pass + + async def __aenter__(self) -> "_HealthzClient": + return self + + async def __aexit__(self, *a: Any) -> bool: + return False + + async def post(self, *a: Any, **k: Any) -> Any: + cls = type(self) + + class _Resp: + status_code = cls.status_code + + return _Resp() + + +class _FakeProc: + pid = 4242 + stdin = None + returncode: int | None = None + + def poll(self) -> int | None: + return None + + def wait(self, timeout: float | None = None) -> int: + return 0 + + def terminate(self) -> None: + pass + + def kill(self) -> None: + pass + + +def _patch_spawn( + monkeypatch: pytest.MonkeyPatch, adapter: PhotonAdapter, tmp_path: Path +) -> None: + """Stub everything _start_sidecar touches before the healthz loop.""" + sidecar_dir = tmp_path / "sidecar" + (sidecar_dir / "node_modules").mkdir(parents=True) + monkeypatch.setattr(photon_adapter, "_SIDECAR_DIR", sidecar_dir) + monkeypatch.setattr(photon_adapter, "_sidecar_deps_stale", lambda: False) + + async def _no_reap(self: PhotonAdapter) -> None: + return None + + monkeypatch.setattr(PhotonAdapter, "_reap_stale_sidecar", _no_reap) + monkeypatch.setattr( + photon_adapter.subprocess, + "run", + lambda *a, **k: type("R", (), {"returncode": 0, "stdout": "", "stderr": ""})(), + ) + monkeypatch.setattr( + photon_adapter.subprocess, "Popen", lambda *a, **k: _FakeProc() + ) + + async def _no_supervise(self: PhotonAdapter, proc: Any) -> None: + return None + + monkeypatch.setattr(PhotonAdapter, "_supervise_sidecar", _no_supervise) + + +@pytest.mark.asyncio +async def test_record_written_after_healthz_success( + monkeypatch: pytest.MonkeyPatch, record_path: Path, tmp_path: Path +) -> None: + adapter = _make_adapter(monkeypatch) + _patch_spawn(monkeypatch, adapter, tmp_path) + _HealthzClient.status_code = 200 + monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _HealthzClient) + + await adapter._start_sidecar() + + data = json.loads(record_path.read_text(encoding="utf-8")) + assert data["port"] == adapter._sidecar_port + assert data["token"] == adapter._sidecar_token + assert data["pid"] == 4242 + + # Cleanup so the fake supervisor task doesn't leak between tests. + if adapter._sidecar_supervisor_task is not None: + adapter._sidecar_supervisor_task.cancel() + + +@pytest.mark.asyncio +async def test_record_removed_on_failed_startup( + monkeypatch: pytest.MonkeyPatch, record_path: Path, tmp_path: Path +) -> None: + adapter = _make_adapter(monkeypatch) + _patch_spawn(monkeypatch, adapter, tmp_path) + monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _HealthzClient) + + class _DeadProc(_FakeProc): + returncode = 1 + + def poll(self) -> int | None: + return 1 + + monkeypatch.setattr( + photon_adapter.subprocess, "Popen", lambda *a, **k: _DeadProc() + ) + + # Pre-seed a stale record; the failed startup must clear it. + photon_adapter._write_runtime_record(1234, "stale", 1) + + with pytest.raises(RuntimeError, match="before becoming ready"): + await adapter._start_sidecar() + + assert not record_path.exists() + if adapter._sidecar_supervisor_task is not None: + adapter._sidecar_supervisor_task.cancel() + + +@pytest.mark.asyncio +async def test_record_removed_on_stop( + monkeypatch: pytest.MonkeyPatch, record_path: Path +) -> None: + adapter = _make_adapter(monkeypatch) + photon_adapter._write_runtime_record(8789, "tok", 4242) + adapter._sidecar_proc = _FakeProc() # type: ignore[assignment] + + await adapter._stop_sidecar() + + assert not record_path.exists() + assert adapter._sidecar_proc is None + + +@pytest.mark.asyncio +async def test_stop_without_proc_still_clears_record( + monkeypatch: pytest.MonkeyPatch, record_path: Path +) -> None: + adapter = _make_adapter(monkeypatch) + photon_adapter._write_runtime_record(8789, "tok", 4242) + adapter._sidecar_proc = None + + await adapter._stop_sidecar() + + assert not record_path.exists() + + +# -- _standalone_send fallback ------------------------------------------------ + + +class _SendClient: + """Fake httpx.AsyncClient capturing /send calls.""" + + calls: list = [] + + def __init__(self, *a: Any, **k: Any) -> None: + pass + + async def __aenter__(self) -> "_SendClient": + return self + + async def __aexit__(self, *a: Any) -> bool: + return False + + async def post(self, url: str, json: Any = None, headers: Any = None) -> Any: + type(self).calls.append((url, json, headers)) + + class _Resp: + status_code = 200 + text = "" + + @staticmethod + def json() -> dict: + return {"ok": True, "messageId": "m1"} + + return _Resp() + + +@pytest.mark.asyncio +async def test_standalone_send_consumes_record_when_env_missing( + monkeypatch: pytest.MonkeyPatch, record_path: Path +) -> None: + monkeypatch.delenv("PHOTON_SIDECAR_TOKEN", raising=False) + monkeypatch.delenv("PHOTON_SIDECAR_PORT", raising=False) + photon_adapter._write_runtime_record(9111, "record-token", os.getpid()) + _SendClient.calls = [] + monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _SendClient) + + result = await photon_adapter._standalone_send( + PlatformConfig(enabled=True, token="", extra={}), "+15551234567", "hi" + ) + + assert result == {"success": True, "message_id": "m1"} + url, _body, headers = _SendClient.calls[0] + assert ":9111/" in url + assert headers["X-Hermes-Sidecar-Token"] == "record-token" + + +@pytest.mark.asyncio +async def test_standalone_send_env_token_still_wins( + monkeypatch: pytest.MonkeyPatch, record_path: Path +) -> None: + monkeypatch.setenv("PHOTON_SIDECAR_TOKEN", "env-token") + monkeypatch.delenv("PHOTON_SIDECAR_PORT", raising=False) + photon_adapter._write_runtime_record(9111, "record-token", os.getpid()) + _SendClient.calls = [] + monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _SendClient) + + result = await photon_adapter._standalone_send( + PlatformConfig(enabled=True, token="", extra={}), "+15551234567", "hi" + ) + + assert result["success"] is True + _url, _body, headers = _SendClient.calls[0] + assert headers["X-Hermes-Sidecar-Token"] == "env-token" + + +@pytest.mark.asyncio +async def test_standalone_send_stale_record_errors_cleanly( + monkeypatch: pytest.MonkeyPatch, record_path: Path +) -> None: + monkeypatch.delenv("PHOTON_SIDECAR_TOKEN", raising=False) + photon_adapter._write_runtime_record(9111, "record-token", 2**22 + 12345) + monkeypatch.setattr(photon_adapter, "_sidecar_pid_alive", lambda pid: False) + _SendClient.calls = [] + monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _SendClient) + + result = await photon_adapter._standalone_send( + PlatformConfig(enabled=True, token="", extra={}), "+15551234567", "hi" + ) + + assert "error" in result + assert "gateway" in result["error"] + assert _SendClient.calls == [] # never hit the sidecar + + +@pytest.mark.asyncio +async def test_standalone_send_no_record_no_env_errors( + monkeypatch: pytest.MonkeyPatch, record_path: Path +) -> None: + monkeypatch.delenv("PHOTON_SIDECAR_TOKEN", raising=False) + _SendClient.calls = [] + monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _SendClient) + + result = await photon_adapter._standalone_send( + PlatformConfig(enabled=True, token="", extra={}), "+15551234567", "hi" + ) + + assert "error" in result + assert "gateway" in result["error"].lower() or "sidecar" in result["error"].lower() + assert _SendClient.calls == [] diff --git a/website/docs/user-guide/messaging/photon.md b/website/docs/user-guide/messaging/photon.md index 7e5ffad83a9c..1022cb91dc9d 100644 --- a/website/docs/user-guide/messaging/photon.md +++ b/website/docs/user-guide/messaging/photon.md @@ -205,6 +205,19 @@ Common issues: - **Photon's free quotas:** 5,000 messages per server per day, 50 new-conversation initiations per shared line per day. Increases available — email `help@photon.codes`. +- **Cron and standalone sends need the gateway running.** Out-of-process + senders (cron jobs, `hermes send`, the dashboard) reuse the sidecar the + gateway spawned — they read its port/token from + `/runtime/photon-sidecar.json`, written once the sidecar + passes its health check and removed when it stops. If a standalone send + reports the gateway appears to be down, start (or restart) the gateway + first. +- **Shared/free-tier lines can't initiate conversations with new + targets.** Photon-side policy: a shared line can only message a number + after that number has texted the line first. A cron/standalone send to a + brand-new recipient will be rejected by Photon even when Hermes is set + up correctly — either have the recipient message the line once, or move + to a dedicated line. ## Env vars