fix(photon): persist sidecar runtime record so cron/standalone sends work

The sidecar auth token is generated at spawn (secrets.token_hex) and
existed only in the gateway process memory + sidecar child env, so
_standalone_send from cron subprocesses, hermes send, or the dashboard
structurally could not authenticate (#69960).

The adapter now writes <hermes-home>/runtime/photon-sidecar.json
({port, token, pid}, 0600, atomic tempfile+os.replace) once the sidecar
passes its /healthz readiness check, and deletes it in _stop_sidecar,
on every startup-failure path, and at disconnect so a stale record
never outlives a dead sidecar. _standalone_send falls back to the
record when PHOTON_SIDECAR_TOKEN is unset, validating the recorded pid
is alive first; a stale record yields a clear 'gateway appears to be
down' error. Docs note the gateway-must-be-running requirement and the
Photon-side shared-line initiation policy (#51897).

plugins/platforms/photon/README.md

@@ -155,6 +155,14 @@ All env vars are documented in `plugin.yaml`. The most important:
   reachable across restarts via spectrum-ts' `space.get(id)`.
 - **Message effects, polls** — supported by `spectrum-ts` but not yet
   exposed; the sidecar is the natural place to add them.
+- **Cron/standalone sends require a running gateway.** Processes outside
+  the gateway (cron subprocesses, `hermes send`) cannot spawn the sidecar;
+  they authenticate to the gateway's live sidecar via the runtime record at
+  `<hermes-home>/runtime/photon-sidecar.json` (written after the sidecar's
+  `/healthz` readiness check, `0600`, removed on stop/failed start). Also
+  note that shared/free-tier Photon lines cannot INITIATE conversations
+  with numbers that never texted the line — that's Photon-side policy, not
+  a Hermes limitation.
 
 ## Upgrading spectrum-ts
 

plugins/platforms/photon/adapter.py

@@ -78,6 +78,103 @@ _DEFAULT_SIDECAR_BIND = "127.0.0.1"
 # size to ~16 KB.  Keep a conservative cap that matches BlueBubbles.
 _MAX_MESSAGE_LENGTH = 8000
 
+# ---------------------------------------------------------------------------
+# Sidecar runtime record
+#
+# Out-of-process senders (cron subprocesses, `hermes send`, the dashboard)
+# go through ``_standalone_send`` and need the live sidecar's port + token —
+# but the token is generated at spawn time and otherwise exists only in the
+# gateway process memory and the sidecar child env (issue #69960). The
+# gateway persists this record once the sidecar passes its /healthz
+# readiness check, and removes it on every stop / failed-start path so a
+# stale record never outlives a dead sidecar.
+
+_RUNTIME_RECORD_NAME = "photon-sidecar.json"
+
+
+def _runtime_record_path() -> Path:
+    # get_hermes_home() honors profile overrides — never hardcode ~/.hermes.
+    from hermes_constants import get_hermes_home
+
+    return get_hermes_home() / "runtime" / _RUNTIME_RECORD_NAME
+
+
+def _write_runtime_record(port: int, token: str, pid: int) -> None:
+    """Atomically persist ``{port, token, pid}`` with owner-only perms."""
+    import tempfile
+
+    try:
+        path = _runtime_record_path()
+        path.parent.mkdir(parents=True, exist_ok=True)
+        fd, tmp = tempfile.mkstemp(
+            dir=str(path.parent), prefix=".photon-sidecar.", suffix=".tmp"
+        )
+        try:
+            # Restrict perms BEFORE the token hits disk (mkstemp is already
+            # 0600 on POSIX; the explicit chmod is a belt-and-braces guard,
+            # wrapped so Windows/exotic filesystems can't crash the write).
+            try:
+                os.chmod(tmp, 0o600)
+            except OSError:  # pragma: no cover - Windows / odd fs
+                pass
+            with os.fdopen(fd, "w", encoding="utf-8") as fh:
+                json.dump({"port": port, "token": token, "pid": pid}, fh)
+            os.replace(tmp, path)
+        except BaseException:
+            try:
+                os.unlink(tmp)
+            except OSError:
+                pass
+            raise
+    except Exception as e:  # best-effort: gateway sends still work without it
+        logger.warning("[photon] failed to write sidecar runtime record: %s", e)
+
+
+def _read_runtime_record() -> Optional[Dict[str, Any]]:
+    try:
+        raw = json.loads(_runtime_record_path().read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+    return raw if isinstance(raw, dict) else None
+
+
+def _delete_runtime_record() -> None:
+    try:
+        _runtime_record_path().unlink(missing_ok=True)
+    except OSError:
+        pass
+
+
+def _sidecar_pid_alive(pid: Any) -> bool:
+    """Best-effort liveness check for the recorded sidecar pid."""
+    try:
+        pid_int = int(pid)
+    except (TypeError, ValueError):
+        return False
+    if pid_int <= 0:
+        return False
+    try:
+        # Cross-platform (psutil-backed, Windows-safe — see its docstring).
+        from gateway.status import _pid_exists
+
+        return bool(_pid_exists(pid_int))
+    except Exception:
+        pass
+    if os.name == "posix":
+        try:
+            os.kill(pid_int, 0)
+        except ProcessLookupError:
+            return False
+        except PermissionError:
+            return True
+        except OSError:
+            return False
+        return True
+    # Windows without gateway.status/psutil: can't probe safely
+    # (os.kill(pid, 0) is destructive there) — assume alive and let the
+    # HTTP send itself be the arbiter.
+    return True
+
 # Dedup parameters — the gRPC stream is at-least-once, and a sidecar
 # reconnect can replay, so keep at least 1k ids for ~48h.
 _DEDUP_MAX_SIZE = 4000
@@ -499,6 +596,9 @@ class PhotonAdapter(BasePlatformAdapter):
                     f"failed to start Photon sidecar: {e}",
                     retryable=True,
                 )
+                # No live sidecar — make sure no stale runtime record
+                # survives to mislead standalone senders.
+                _delete_runtime_record()
                 await client.aclose()
                 self._http_client = None
                 return False
@@ -1110,6 +1210,7 @@ class PhotonAdapter(BasePlatformAdapter):
         async with httpx.AsyncClient(timeout=2.0, trust_env=False) as client:
             while time.time() < deadline:
                 if self._sidecar_proc.poll() is not None:
+                    _delete_runtime_record()
                     raise RuntimeError(
                         f"Photon sidecar exited with code "
                         f"{self._sidecar_proc.returncode} before becoming ready"
@@ -1120,10 +1221,19 @@ class PhotonAdapter(BasePlatformAdapter):
                         headers={"X-Hermes-Sidecar-Token": self._sidecar_token},
                     )
                     if resp.status_code == 200:
+                        # Persist port/token/pid so out-of-process senders
+                        # (cron, `hermes send`) can reach this sidecar
+                        # (see _standalone_send / issue #69960).
+                        _write_runtime_record(
+                            self._sidecar_port,
+                            self._sidecar_token,
+                            self._sidecar_proc.pid,
+                        )
                         return
                 except httpx.RequestError as e:
                     last_err = e
                 await asyncio.sleep(0.2)
+        _delete_runtime_record()
         raise RuntimeError(
             f"Photon sidecar did not become ready within 15s: {last_err}"
         )
@@ -1161,6 +1271,9 @@ class PhotonAdapter(BasePlatformAdapter):
     async def _stop_sidecar(self) -> None:
         proc = self._sidecar_proc
         if proc is None:
+            # Nothing to stop, but never leave a record behind on disconnect
+            # (e.g. autostart was disabled or startup failed earlier).
+            _delete_runtime_record()
             return
         try:
             # Closing our end of the stdin pipe is itself a shutdown signal
@@ -1197,6 +1310,9 @@ class PhotonAdapter(BasePlatformAdapter):
                     proc.kill()
         finally:
             self._sidecar_proc = None
+            # The sidecar is gone (or going) — remove the runtime record so
+            # standalone senders don't chase a dead port/token.
+            _delete_runtime_record()
             if self._sidecar_supervisor_task is not None:
                 self._sidecar_supervisor_task.cancel()
                 self._sidecar_supervisor_task = None
@@ -1799,13 +1915,32 @@ async def _standalone_send(
     )
     token = os.getenv("PHOTON_SIDECAR_TOKEN")
     if not token:
-        return {
-            "error": (
-                "Photon standalone send requires a running sidecar with "
-                "PHOTON_SIDECAR_TOKEN set in the environment. Cron processes "
-                "cannot spawn the sidecar themselves."
-            )
-        }
+        # Fall back to the runtime record the gateway persists once its
+        # sidecar passes /healthz (issue #69960) — the token only exists in
+        # the gateway process env otherwise, so cron/`hermes send` would be
+        # structurally unable to authenticate.
+        record = _read_runtime_record()
+        stale_hint = ""
+        if record and record.get("token"):
+            if _sidecar_pid_alive(record.get("pid")):
+                token = str(record["token"])
+                port = _coerce_port(record.get("port"), port)
+            else:
+                stale_hint = (
+                    " A stale sidecar runtime record was found (pid "
+                    f"{record.get('pid')} is not running) — the gateway "
+                    "appears to be down."
+                )
+        if not token:
+            return {
+                "error": (
+                    "Photon standalone send requires a running sidecar. "
+                    "Start the Hermes gateway (which spawns the sidecar and "
+                    "records its address under <hermes-home>/runtime/"
+                    f"{_RUNTIME_RECORD_NAME}), or set PHOTON_SIDECAR_TOKEN "
+                    "in this process's environment." + stale_hint
+                )
+            }
     base = f"http://{_DEFAULT_SIDECAR_BIND}:{port}"
     headers = {"X-Hermes-Sidecar-Token": token}
     last_message_id: Optional[str] = None

tests/plugins/platforms/photon/test_runtime_record.py

@@ -0,0 +1,326 @@
+"""Sidecar runtime-record persistence tests (issue #69960).
+
+The sidecar token is generated at spawn and used to exist only in the
+gateway process memory + sidecar child env — so cron/`hermes send`
+standalone sends structurally could not authenticate. The adapter now
+persists ``<hermes-home>/runtime/photon-sidecar.json`` after the sidecar
+passes its /healthz readiness check, deletes it on stop/failed-start, and
+``_standalone_send`` falls back to it when PHOTON_SIDECAR_TOKEN is unset.
+No Node, no ports, no network.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import stat
+import sys
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from gateway.config import PlatformConfig
+from plugins.platforms.photon import adapter as photon_adapter
+from plugins.platforms.photon.adapter import PhotonAdapter
+
+
+@pytest.fixture()
+def record_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+    path = tmp_path / "runtime" / "photon-sidecar.json"
+    monkeypatch.setattr(photon_adapter, "_runtime_record_path", lambda: path)
+    return path
+
+
+def _make_adapter(monkeypatch: pytest.MonkeyPatch) -> PhotonAdapter:
+    monkeypatch.setenv("PHOTON_PROJECT_ID", "test-project-id")
+    monkeypatch.setenv("PHOTON_PROJECT_SECRET", "test-project-secret")
+    monkeypatch.delenv("PHOTON_SIDECAR_TOKEN", raising=False)
+    cfg = PlatformConfig(enabled=True, token="", extra={})
+    return PhotonAdapter(cfg)
+
+
+# -- record helpers ----------------------------------------------------------
+
+
+def test_write_read_delete_roundtrip(record_path: Path) -> None:
+    photon_adapter._write_runtime_record(8789, "tok123", 4242)
+
+    assert record_path.exists()
+    data = json.loads(record_path.read_text(encoding="utf-8"))
+    assert data == {"port": 8789, "token": "tok123", "pid": 4242}
+    assert photon_adapter._read_runtime_record() == data
+
+    photon_adapter._delete_runtime_record()
+    assert not record_path.exists()
+    # Idempotent: deleting a missing record must not raise.
+    photon_adapter._delete_runtime_record()
+    assert photon_adapter._read_runtime_record() is None
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits")
+def test_record_written_with_0600(record_path: Path) -> None:
+    photon_adapter._write_runtime_record(8789, "secret", 1)
+    mode = stat.S_IMODE(record_path.stat().st_mode)
+    assert mode == 0o600
+
+
+def test_read_tolerates_corrupt_record(record_path: Path) -> None:
+    record_path.parent.mkdir(parents=True, exist_ok=True)
+    record_path.write_text("{not json", encoding="utf-8")
+    assert photon_adapter._read_runtime_record() is None
+
+
+# -- lifecycle: written after healthz success, removed on stop/failure -------
+
+
+class _HealthzClient:
+    """Fake httpx.AsyncClient whose /healthz response is injectable."""
+
+    status_code = 200
+
+    def __init__(self, *a: Any, **k: Any) -> None:
+        pass
+
+    async def __aenter__(self) -> "_HealthzClient":
+        return self
+
+    async def __aexit__(self, *a: Any) -> bool:
+        return False
+
+    async def post(self, *a: Any, **k: Any) -> Any:
+        cls = type(self)
+
+        class _Resp:
+            status_code = cls.status_code
+
+        return _Resp()
+
+
+class _FakeProc:
+    pid = 4242
+    stdin = None
+    returncode: int | None = None
+
+    def poll(self) -> int | None:
+        return None
+
+    def wait(self, timeout: float | None = None) -> int:
+        return 0
+
+    def terminate(self) -> None:
+        pass
+
+    def kill(self) -> None:
+        pass
+
+
+def _patch_spawn(
+    monkeypatch: pytest.MonkeyPatch, adapter: PhotonAdapter, tmp_path: Path
+) -> None:
+    """Stub everything _start_sidecar touches before the healthz loop."""
+    sidecar_dir = tmp_path / "sidecar"
+    (sidecar_dir / "node_modules").mkdir(parents=True)
+    monkeypatch.setattr(photon_adapter, "_SIDECAR_DIR", sidecar_dir)
+    monkeypatch.setattr(photon_adapter, "_sidecar_deps_stale", lambda: False)
+
+    async def _no_reap(self: PhotonAdapter) -> None:
+        return None
+
+    monkeypatch.setattr(PhotonAdapter, "_reap_stale_sidecar", _no_reap)
+    monkeypatch.setattr(
+        photon_adapter.subprocess,
+        "run",
+        lambda *a, **k: type("R", (), {"returncode": 0, "stdout": "", "stderr": ""})(),
+    )
+    monkeypatch.setattr(
+        photon_adapter.subprocess, "Popen", lambda *a, **k: _FakeProc()
+    )
+
+    async def _no_supervise(self: PhotonAdapter, proc: Any) -> None:
+        return None
+
+    monkeypatch.setattr(PhotonAdapter, "_supervise_sidecar", _no_supervise)
+
+
+@pytest.mark.asyncio
+async def test_record_written_after_healthz_success(
+    monkeypatch: pytest.MonkeyPatch, record_path: Path, tmp_path: Path
+) -> None:
+    adapter = _make_adapter(monkeypatch)
+    _patch_spawn(monkeypatch, adapter, tmp_path)
+    _HealthzClient.status_code = 200
+    monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _HealthzClient)
+
+    await adapter._start_sidecar()
+
+    data = json.loads(record_path.read_text(encoding="utf-8"))
+    assert data["port"] == adapter._sidecar_port
+    assert data["token"] == adapter._sidecar_token
+    assert data["pid"] == 4242
+
+    # Cleanup so the fake supervisor task doesn't leak between tests.
+    if adapter._sidecar_supervisor_task is not None:
+        adapter._sidecar_supervisor_task.cancel()
+
+
+@pytest.mark.asyncio
+async def test_record_removed_on_failed_startup(
+    monkeypatch: pytest.MonkeyPatch, record_path: Path, tmp_path: Path
+) -> None:
+    adapter = _make_adapter(monkeypatch)
+    _patch_spawn(monkeypatch, adapter, tmp_path)
+    monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _HealthzClient)
+
+    class _DeadProc(_FakeProc):
+        returncode = 1
+
+        def poll(self) -> int | None:
+            return 1
+
+    monkeypatch.setattr(
+        photon_adapter.subprocess, "Popen", lambda *a, **k: _DeadProc()
+    )
+
+    # Pre-seed a stale record; the failed startup must clear it.
+    photon_adapter._write_runtime_record(1234, "stale", 1)
+
+    with pytest.raises(RuntimeError, match="before becoming ready"):
+        await adapter._start_sidecar()
+
+    assert not record_path.exists()
+    if adapter._sidecar_supervisor_task is not None:
+        adapter._sidecar_supervisor_task.cancel()
+
+
+@pytest.mark.asyncio
+async def test_record_removed_on_stop(
+    monkeypatch: pytest.MonkeyPatch, record_path: Path
+) -> None:
+    adapter = _make_adapter(monkeypatch)
+    photon_adapter._write_runtime_record(8789, "tok", 4242)
+    adapter._sidecar_proc = _FakeProc()  # type: ignore[assignment]
+
+    await adapter._stop_sidecar()
+
+    assert not record_path.exists()
+    assert adapter._sidecar_proc is None
+
+
+@pytest.mark.asyncio
+async def test_stop_without_proc_still_clears_record(
+    monkeypatch: pytest.MonkeyPatch, record_path: Path
+) -> None:
+    adapter = _make_adapter(monkeypatch)
+    photon_adapter._write_runtime_record(8789, "tok", 4242)
+    adapter._sidecar_proc = None
+
+    await adapter._stop_sidecar()
+
+    assert not record_path.exists()
+
+
+# -- _standalone_send fallback ------------------------------------------------
+
+
+class _SendClient:
+    """Fake httpx.AsyncClient capturing /send calls."""
+
+    calls: list = []
+
+    def __init__(self, *a: Any, **k: Any) -> None:
+        pass
+
+    async def __aenter__(self) -> "_SendClient":
+        return self
+
+    async def __aexit__(self, *a: Any) -> bool:
+        return False
+
+    async def post(self, url: str, json: Any = None, headers: Any = None) -> Any:
+        type(self).calls.append((url, json, headers))
+
+        class _Resp:
+            status_code = 200
+            text = ""
+
+            @staticmethod
+            def json() -> dict:
+                return {"ok": True, "messageId": "m1"}
+
+        return _Resp()
+
+
+@pytest.mark.asyncio
+async def test_standalone_send_consumes_record_when_env_missing(
+    monkeypatch: pytest.MonkeyPatch, record_path: Path
+) -> None:
+    monkeypatch.delenv("PHOTON_SIDECAR_TOKEN", raising=False)
+    monkeypatch.delenv("PHOTON_SIDECAR_PORT", raising=False)
+    photon_adapter._write_runtime_record(9111, "record-token", os.getpid())
+    _SendClient.calls = []
+    monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _SendClient)
+
+    result = await photon_adapter._standalone_send(
+        PlatformConfig(enabled=True, token="", extra={}), "+15551234567", "hi"
+    )
+
+    assert result == {"success": True, "message_id": "m1"}
+    url, _body, headers = _SendClient.calls[0]
+    assert ":9111/" in url
+    assert headers["X-Hermes-Sidecar-Token"] == "record-token"
+
+
+@pytest.mark.asyncio
+async def test_standalone_send_env_token_still_wins(
+    monkeypatch: pytest.MonkeyPatch, record_path: Path
+) -> None:
+    monkeypatch.setenv("PHOTON_SIDECAR_TOKEN", "env-token")
+    monkeypatch.delenv("PHOTON_SIDECAR_PORT", raising=False)
+    photon_adapter._write_runtime_record(9111, "record-token", os.getpid())
+    _SendClient.calls = []
+    monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _SendClient)
+
+    result = await photon_adapter._standalone_send(
+        PlatformConfig(enabled=True, token="", extra={}), "+15551234567", "hi"
+    )
+
+    assert result["success"] is True
+    _url, _body, headers = _SendClient.calls[0]
+    assert headers["X-Hermes-Sidecar-Token"] == "env-token"
+
+
+@pytest.mark.asyncio
+async def test_standalone_send_stale_record_errors_cleanly(
+    monkeypatch: pytest.MonkeyPatch, record_path: Path
+) -> None:
+    monkeypatch.delenv("PHOTON_SIDECAR_TOKEN", raising=False)
+    photon_adapter._write_runtime_record(9111, "record-token", 2**22 + 12345)
+    monkeypatch.setattr(photon_adapter, "_sidecar_pid_alive", lambda pid: False)
+    _SendClient.calls = []
+    monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _SendClient)
+
+    result = await photon_adapter._standalone_send(
+        PlatformConfig(enabled=True, token="", extra={}), "+15551234567", "hi"
+    )
+
+    assert "error" in result
+    assert "gateway" in result["error"]
+    assert _SendClient.calls == []  # never hit the sidecar
+
+
+@pytest.mark.asyncio
+async def test_standalone_send_no_record_no_env_errors(
+    monkeypatch: pytest.MonkeyPatch, record_path: Path
+) -> None:
+    monkeypatch.delenv("PHOTON_SIDECAR_TOKEN", raising=False)
+    _SendClient.calls = []
+    monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _SendClient)
+
+    result = await photon_adapter._standalone_send(
+        PlatformConfig(enabled=True, token="", extra={}), "+15551234567", "hi"
+    )
+
+    assert "error" in result
+    assert "gateway" in result["error"].lower() or "sidecar" in result["error"].lower()
+    assert _SendClient.calls == []

website/docs/user-guide/messaging/photon.md

@@ -205,6 +205,19 @@ Common issues:
 - **Photon's free quotas:** 5,000 messages per server per day,
   50 new-conversation initiations per shared line per day. Increases
   available — email `help@photon.codes`.
+- **Cron and standalone sends need the gateway running.** Out-of-process
+  senders (cron jobs, `hermes send`, the dashboard) reuse the sidecar the
+  gateway spawned — they read its port/token from
+  `<hermes-home>/runtime/photon-sidecar.json`, written once the sidecar
+  passes its health check and removed when it stops. If a standalone send
+  reports the gateway appears to be down, start (or restart) the gateway
+  first.
+- **Shared/free-tier lines can't initiate conversations with new
+  targets.** Photon-side policy: a shared line can only message a number
+  after that number has texted the line first. A cron/standalone send to a
+  brand-new recipient will be rejected by Photon even when Hermes is set
+  up correctly — either have the recipient message the line once, or move
+  to a dedicated line.
 
 ## Env vars