From 1289f12812a9c6a3f3e6fbdf39e39ed7e1b7bd50 Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Mon, 29 Jun 2026 13:36:46 +1000 Subject: [PATCH] fix(memory): lazy-install supermemory + mem0 SDKs like honcho/hindsight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The supermemory and mem0 memory providers shipped third-party SDKs (supermemory / mem0ai) that are not core dependencies, but — unlike the honcho and hindsight providers — they imported those SDKs directly with no tools.lazy_deps.ensure() preflight and had no LAZY_DEPS allowlist entry. On the published Docker image the agent venv is sealed (HERMES_DISABLE_LAZY_INSTALLS=1) and lazy installs are redirected to a writable durable target (HERMES_LAZY_INSTALL_TARGET). honcho/hindsight route through ensure() and install fine there; supermemory/mem0 never called it, so their SDK was never installed on a hosted instance and the provider silently reported itself unavailable even with the API key set. Fixes: - Add memory.supermemory + memory.mem0 to the LAZY_DEPS allowlist (tools/lazy_deps.py), pinned to current PyPI releases. - Call ensure('memory.', prompt=False) at each SDK-import chokepoint (_SupermemoryClient.__init__; Mem0MemoryProvider._create_backend), mirroring honcho's wrapped try/except shape. - Drop the SDK-import gate from supermemory's is_available() — it was a chicken-and-egg trap (provider never loaded on a sealed venv, so ensure() never ran). Now key-presence only, like honcho/mem0. - Add matching pyproject extras [supermemory]/[mem0]; update the lazy-covered-extras contract test (excluded from [all] by policy). Tests prove each path fails without the fix and the real sealed-venv durable-target gate accepts both features. --- plugins/memory/mem0/__init__.py | 12 + plugins/memory/supermemory/__init__.py | 29 +- pyproject.toml | 7 + .../memory/test_memory_lazy_install.py | 254 ++++++++++++++++++ .../memory/test_supermemory_provider.py | 17 +- tests/test_project_metadata.py | 1 + tools/lazy_deps.py | 9 + 7 files changed, 319 insertions(+), 10 deletions(-) create mode 100644 tests/plugins/memory/test_memory_lazy_install.py diff --git a/plugins/memory/mem0/__init__.py b/plugins/memory/mem0/__init__.py index eccf6ad53fe..f11d1f0cd83 100644 --- a/plugins/memory/mem0/__init__.py +++ b/plugins/memory/mem0/__init__.py @@ -250,6 +250,18 @@ class Mem0MemoryProvider(MemoryProvider): post_setup(hermes_home, config) def _create_backend(self): + # Lazy-install the mem0 SDK on demand before either backend imports + # it. ensure() honors security.allow_lazy_installs (default true) and, + # on a sealed Docker venv, redirects the install to the durable + # target. On failure we fall through so the import inside the backend + # produces the canonical error, captured below. + try: + from tools.lazy_deps import ensure as _lazy_ensure + _lazy_ensure("memory.mem0", prompt=False) + except ImportError: + pass + except Exception: + pass try: if self._mode == "oss": from ._backend import OSSBackend diff --git a/plugins/memory/supermemory/__init__.py b/plugins/memory/supermemory/__init__.py index 14afcff9ac0..1d086f47df8 100644 --- a/plugins/memory/supermemory/__init__.py +++ b/plugins/memory/supermemory/__init__.py @@ -264,6 +264,19 @@ def _is_trivial_message(text: str) -> bool: class _SupermemoryClient: def __init__(self, api_key: str, timeout: float, container_tag: str, search_mode: str = "hybrid"): + # Lazy-install the supermemory SDK on demand. ensure() honors + # security.allow_lazy_installs (default true) and, on a sealed Docker + # venv, redirects the install to the durable target. On failure we + # fall through so the raw import below produces the canonical + # ImportError message. + try: + from tools.lazy_deps import ensure as _lazy_ensure + _lazy_ensure("memory.supermemory", prompt=False) + except ImportError: + pass + except Exception: + pass + from supermemory import Supermemory self._api_key = api_key @@ -533,14 +546,14 @@ class SupermemoryMemoryProvider(MemoryProvider): return "supermemory" def is_available(self) -> bool: - api_key = os.environ.get("SUPERMEMORY_API_KEY", "") - if not api_key: - return False - try: - __import__("supermemory") - return True - except Exception: - return False + # Key presence only — no SDK import check. The supermemory SDK is + # lazy-installed when the client is first constructed in initialize() + # (see _SupermemoryClient.__init__). Gating availability on the SDK + # being importable here would be a chicken-and-egg trap: on a sealed + # Docker venv the package isn't present until ensure() runs, but + # ensure() only runs once the provider is loaded — which this gates. + # Mirrors honcho/mem0, which check config only. No network calls. + return bool(os.environ.get("SUPERMEMORY_API_KEY", "")) def get_config_schema(self): # Only prompt for the API key during `hermes memory setup`. diff --git a/pyproject.toml b/pyproject.toml index d269ba840be..b1ef9062d0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -181,6 +181,13 @@ pty = [ # without pulling in extra packages. ] honcho = ["honcho-ai==2.0.1"] +# Cloud memory providers — opt-in, lazy-installed via tools/lazy_deps.py +# (memory.supermemory / memory.mem0) at first use. Exact pins MUST match the +# LAZY_DEPS pins (enforced by tests/test_project_metadata.py). Deliberately +# excluded from [all] like honcho/hindsight so a quarantined upstream release +# can't break fresh installs. +supermemory = ["supermemory==3.50.0"] +mem0 = ["mem0ai==2.0.10"] # Image resize recovery for the vision tools. Pillow is now a CORE dependency # (see the main `dependencies` list) since the byte/pixel shrink paths are on # the default vision-embed path and the mid-session lazy install deadlocked the diff --git a/tests/plugins/memory/test_memory_lazy_install.py b/tests/plugins/memory/test_memory_lazy_install.py new file mode 100644 index 00000000000..2a7fc532273 --- /dev/null +++ b/tests/plugins/memory/test_memory_lazy_install.py @@ -0,0 +1,254 @@ +"""Regression tests: supermemory + mem0 memory providers must lazy-install +their SDKs like honcho/hindsight. + +Both providers ship a third-party SDK (``supermemory`` / ``mem0ai``) that is +NOT a core dependency. Before this fix they imported the SDK directly with no +``tools.lazy_deps.ensure()`` preflight and had no ``LAZY_DEPS`` allowlist +entry. On the published Docker image the agent venv is sealed +(``HERMES_DISABLE_LAZY_INSTALLS=1``) and lazy installs are redirected to a +writable durable target (``HERMES_LAZY_INSTALL_TARGET``). honcho/hindsight +route through ``ensure()`` and therefore install fine on a hosted instance; +supermemory/mem0 never called it, so the SDK was never installed there and +the provider silently reported itself unavailable. + +These tests pin the contract: + +1. Both features are in the ``LAZY_DEPS`` allowlist (without an entry, + ``ensure()`` raises ``FeatureUnavailable`` — the original silent-dark bug). +2. Each provider's SDK-import chokepoint actually calls ``ensure()``. +3. supermemory's ``is_available()`` no longer gates on the SDK being + importable (the chicken-and-egg trap that stopped the provider loading at + all on a sealed venv, so ``initialize()``/``ensure()`` never ran). +4. The real sealed-venv durable-target gate accepts the new features (the + exact hosted-Fly condition the user hit). + +The pip subprocess is never actually run — ``_venv_pip_install`` / +``_is_satisfied`` are stubbed so we exercise the real ``ensure()`` control +flow without touching PyPI. +""" + +from __future__ import annotations + +import os + +import pytest + +import tools.lazy_deps as ld + + +MEMORY_FEATURES = ("memory.supermemory", "memory.mem0") + + +# --------------------------------------------------------------------------- +# 1. Allowlist contract — the core regression. +# --------------------------------------------------------------------------- + + +class TestAllowlistEntries: + @pytest.mark.parametrize("feature", MEMORY_FEATURES) + def test_feature_is_allowlisted(self, feature): + # Without an allowlist entry, ensure() raises FeatureUnavailable with + # "not in LAZY_DEPS" — which is exactly why the SDK never installed on + # a hosted instance before this fix. + assert feature in ld.LAZY_DEPS, ( + f"{feature!r} missing from LAZY_DEPS — its SDK can never " + f"lazy-install on a sealed Docker venv." + ) + + @pytest.mark.parametrize("feature", MEMORY_FEATURES) + def test_feature_specs_pass_safety(self, feature): + for spec in ld.LAZY_DEPS[feature]: + assert ld._spec_is_safe(spec), f"{feature}: {spec!r} fails safety" + + def test_supermemory_spec_package(self): + specs = ld.LAZY_DEPS["memory.supermemory"] + assert any(ld._pkg_name_from_spec(s) == "supermemory" for s in specs) + + def test_mem0_spec_package(self): + # mem0's pip package is ``mem0ai`` (imports as ``mem0``). + specs = ld.LAZY_DEPS["memory.mem0"] + assert any(ld._pkg_name_from_spec(s) == "mem0ai" for s in specs) + + @pytest.mark.parametrize("feature", MEMORY_FEATURES) + def test_unknown_feature_would_raise_without_entry(self, feature, monkeypatch): + # Demonstrate the failure mode the allowlist entry prevents: a feature + # NOT in LAZY_DEPS raises rather than installing. + monkeypatch.setattr(ld, "_allow_lazy_installs", lambda: True) + with pytest.raises(ld.FeatureUnavailable, match="not in LAZY_DEPS"): + ld.ensure(feature + ".typo", prompt=False) + + +# --------------------------------------------------------------------------- +# 2. Import sites call ensure(). +# --------------------------------------------------------------------------- + + +class TestSupermemoryEnsureCalled: + def test_client_construction_calls_ensure(self, monkeypatch): + """_SupermemoryClient.__init__ must call ensure('memory.supermemory') + before importing the SDK.""" + from plugins.memory.supermemory import _SupermemoryClient + + calls = [] + monkeypatch.setattr( + ld, "ensure", + lambda feature, **kw: calls.append((feature, kw)), + ) + + # Stub the SDK so construction doesn't need the real package. The + # client does ``from supermemory import Supermemory`` right after + # ensure(); inject a fake module. + import sys + import types + + fake = types.ModuleType("supermemory") + fake.Supermemory = lambda **kw: object() + monkeypatch.setitem(sys.modules, "supermemory", fake) + + _SupermemoryClient(api_key="k", timeout=5.0, container_tag="hermes") + + assert ("memory.supermemory", {"prompt": False}) in calls, ( + "supermemory client did not call ensure('memory.supermemory', " + f"prompt=False); calls={calls}" + ) + + +class TestMem0EnsureCalled: + def test_create_backend_calls_ensure(self, monkeypatch): + """SupermemoryMemoryProvider-style mem0 provider must call + ensure('memory.mem0') in _create_backend before importing the SDK.""" + from plugins.memory.mem0 import Mem0MemoryProvider + + calls = [] + monkeypatch.setattr( + ld, "ensure", + lambda feature, **kw: calls.append((feature, kw)), + ) + + prov = Mem0MemoryProvider() + # Platform mode is the default; force a known mode and stub the backend + # import so we isolate the ensure() call. + prov._mode = "platform" + prov._api_key = "k" + + import sys + import types + + fake = types.ModuleType("mem0") + fake.MemoryClient = lambda **kw: object() + fake.Memory = object + monkeypatch.setitem(sys.modules, "mem0", fake) + # _backend imports ``from mem0 import MemoryClient`` lazily inside + # PlatformBackend.__init__, so the fake module satisfies it. + + prov._create_backend() + + assert ("memory.mem0", {"prompt": False}) in calls, ( + f"mem0 _create_backend did not call ensure('memory.mem0', " + f"prompt=False); calls={calls}" + ) + + +# --------------------------------------------------------------------------- +# 3. supermemory is_available() chicken-and-egg fix. +# --------------------------------------------------------------------------- + + +class TestSupermemoryIsAvailable: + def test_available_with_key_even_when_sdk_absent(self, monkeypatch): + """With the key set but the SDK not importable, is_available() must + still return True — otherwise the provider never loads on a sealed + venv and ensure() (which installs the SDK) never runs.""" + from plugins.memory.supermemory import SupermemoryMemoryProvider + import builtins + + monkeypatch.setenv("SUPERMEMORY_API_KEY", "sk-test") + + # Make any attempt to import the SDK fail, simulating the + # not-yet-installed sealed-venv state. + real_import = builtins.__import__ + + def _no_supermemory(name, *args, **kwargs): + if name == "supermemory" or name.startswith("supermemory."): + raise ImportError("No module named 'supermemory'") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _no_supermemory) + + prov = SupermemoryMemoryProvider() + assert prov.is_available() is True + + def test_unavailable_without_key(self, monkeypatch): + from plugins.memory.supermemory import SupermemoryMemoryProvider + + monkeypatch.delenv("SUPERMEMORY_API_KEY", raising=False) + prov = SupermemoryMemoryProvider() + assert prov.is_available() is False + + +# --------------------------------------------------------------------------- +# 4. Real sealed-venv durable-target gate accepts the new features. +# +# This is the exact hosted-Fly condition: HERMES_DISABLE_LAZY_INSTALLS=1 seals +# the venv, but HERMES_LAZY_INSTALL_TARGET redirects installs to a writable +# durable dir, so installs are still ALLOWED. We exercise the real +# _allow_lazy_installs() + ensure() flow end-to-end with only the pip +# subprocess stubbed. +# --------------------------------------------------------------------------- + + +class TestSealedVenvDurableTarget: + @pytest.mark.parametrize("feature", MEMORY_FEATURES) + def test_ensure_installs_into_durable_target_on_sealed_venv( + self, feature, monkeypatch, tmp_path + ): + # Sealed venv + durable target = the published Docker image config. + monkeypatch.setenv("HERMES_DISABLE_LAZY_INSTALLS", "1") + monkeypatch.setenv("HERMES_LAZY_INSTALL_TARGET", str(tmp_path / "lazy")) + # config.yaml kill-switch left at default (allow). + monkeypatch.setattr( + "hermes_cli.config.load_config", + lambda: {"security": {"allow_lazy_installs": True}}, + ) + + # Real gate must permit installs because a durable target is set. + assert ld._allow_lazy_installs() is True, ( + "sealed venv WITH a durable target must allow installs — this is " + "the path honcho/hindsight use on hosted Fly instances" + ) + + # Drive ensure(): missing first, satisfied after the (stubbed) install. + states = iter([False, True]) + monkeypatch.setattr(ld, "_is_satisfied", lambda spec: next(states)) + + captured = {} + + def fake_install(specs, **kw): + captured["specs"] = specs + captured["target_env"] = os.environ.get("HERMES_LAZY_INSTALL_TARGET") + return ld._InstallResult(True, "ok", "") + + monkeypatch.setattr(ld, "_venv_pip_install", fake_install) + + ld.ensure(feature, prompt=False) # must not raise + + assert captured.get("specs") == ld.LAZY_DEPS[feature] + assert captured.get("target_env"), ( + "install ran without the durable target env set" + ) + + @pytest.mark.parametrize("feature", MEMORY_FEATURES) + def test_sealed_venv_without_target_blocks(self, feature, monkeypatch): + # Sealed venv and NO durable target → installs blocked (can't mutate + # the sealed venv). Belt-and-suspenders: confirms the gate still + # protects the seal for these features. + monkeypatch.setenv("HERMES_DISABLE_LAZY_INSTALLS", "1") + monkeypatch.delenv("HERMES_LAZY_INSTALL_TARGET", raising=False) + monkeypatch.setattr( + "hermes_cli.config.load_config", + lambda: {"security": {"allow_lazy_installs": True}}, + ) + monkeypatch.setattr(ld, "_is_satisfied", lambda spec: False) + + with pytest.raises(ld.FeatureUnavailable, match="lazy installs disabled"): + ld.ensure(feature, prompt=False) diff --git a/tests/plugins/memory/test_supermemory_provider.py b/tests/plugins/memory/test_supermemory_provider.py index e9b3a0c8e13..c0dc890433f 100644 --- a/tests/plugins/memory/test_supermemory_provider.py +++ b/tests/plugins/memory/test_supermemory_provider.py @@ -71,19 +71,32 @@ def test_is_available_false_without_api_key(monkeypatch): assert p.is_available() is False -def test_is_available_false_when_import_missing(monkeypatch): +def test_is_available_true_when_import_missing_but_key_set(monkeypatch): + # Regression: is_available() must NOT gate on the supermemory SDK being + # importable. The SDK is lazy-installed at client construction (see + # _SupermemoryClient.__init__ -> tools.lazy_deps.ensure). Gating here is a + # chicken-and-egg trap: on a sealed Docker venv the package isn't present + # until ensure() runs, but ensure() only runs once the provider loads — + # which this gates. So with the key set and the SDK absent, the provider + # must still report available. Mirrors honcho/mem0 (config-presence only). monkeypatch.setenv("SUPERMEMORY_API_KEY", "test-key") import builtins real_import = builtins.__import__ def fake_import(name, *args, **kwargs): - if name == "supermemory": + if name == "supermemory" or name.startswith("supermemory."): raise ImportError("missing") return real_import(name, *args, **kwargs) monkeypatch.setattr(builtins, "__import__", fake_import) p = SupermemoryMemoryProvider() + assert p.is_available() is True + + +def test_is_available_false_without_key(monkeypatch): + monkeypatch.delenv("SUPERMEMORY_API_KEY", raising=False) + p = SupermemoryMemoryProvider() assert p.is_available() is False diff --git a/tests/test_project_metadata.py b/tests/test_project_metadata.py index 6c761cb2cdb..f2f4887d609 100644 --- a/tests/test_project_metadata.py +++ b/tests/test_project_metadata.py @@ -73,6 +73,7 @@ def test_lazy_installable_extras_excluded_from_all(): "modal", "daytona", "messaging", "slack", "matrix", "dingtalk", "feishu", "honcho", "hindsight", + "supermemory", "mem0", "mistral", # mistralai — Voxtral STT/TTS, lazy-installed (stt.mistral / tts.mistral) } all_extra_specs = optional_dependencies["all"] diff --git a/tools/lazy_deps.py b/tools/lazy_deps.py index 2e74f8391ca..0bf3424c094 100644 --- a/tools/lazy_deps.py +++ b/tools/lazy_deps.py @@ -137,6 +137,15 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = { # ─── Memory providers ────────────────────────────────────────────────── "memory.honcho": ("honcho-ai==2.0.1",), "memory.hindsight": ("hindsight-client==0.6.1",), + # supermemory + mem0 are opt-in cloud memory providers with their own + # SDKs. On the published Docker image the agent venv is sealed + # (HERMES_DISABLE_LAZY_INSTALLS=1) and lazy installs are redirected to the + # durable target — so, like honcho/hindsight, these MUST go through + # ensure() to be installable there. Without an allowlist entry + an + # ensure() call at the import site, the SDK never installs on a hosted + # instance and the provider silently reports itself unavailable. + "memory.supermemory": ("supermemory==3.50.0",), + "memory.mem0": ("mem0ai==2.0.10",), # ─── Messaging platforms (lazy-installable on demand) ────────────────── "platform.telegram": ("python-telegram-bot[webhooks]==22.6",),