diff --git a/agent/agent_init.py b/agent/agent_init.py index 0c700c279b98..e1d219c62c63 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -1354,6 +1354,27 @@ def init_agent( _agent_cfg = _load_agent_config() except Exception: _agent_cfg = {} + + # LM Studio can either be explicitly preloaded through LM Studio's + # management API (the historical Hermes behavior) or left to LM Studio's + # just-in-time / Auto-Evict chat-completions path. Keep the default + # explicit for backward compatibility; users with LM Studio Auto-Evict can + # opt into JIT via ``model.lmstudio_load_mode: jit``. + agent.lmstudio_load_mode = "explicit" + try: + _model_section = _agent_cfg.get("model", {}) + if isinstance(_model_section, dict): + _load_mode = str(_model_section.get("lmstudio_load_mode", "explicit") or "explicit").strip().lower() + if _load_mode in {"explicit", "jit"}: + agent.lmstudio_load_mode = _load_mode + else: + logger.warning( + "Invalid model.lmstudio_load_mode=%r; expected 'explicit' or 'jit'. Using explicit.", + _model_section.get("lmstudio_load_mode"), + ) + except Exception: + agent.lmstudio_load_mode = "explicit" + try: agent._tool_guardrails = ToolCallGuardrailController( ToolCallGuardrailConfig.from_mapping( diff --git a/run_agent.py b/run_agent.py index 7cb3839d504e..3ba625d9cca8 100644 --- a/run_agent.py +++ b/run_agent.py @@ -770,10 +770,13 @@ class AIAgent: def _ensure_lmstudio_runtime_loaded(self, config_context_length: Optional[int] = None) -> None: """ - Preload the LM Studio model with at least Hermes' minimum context. + Preload the LM Studio model unless configured to rely on LM Studio JIT loading. """ if (self.provider or "").strip().lower() != "lmstudio": return + if (getattr(self, "lmstudio_load_mode", "explicit") or "explicit").strip().lower() == "jit": + logger.debug("LM Studio explicit preload skipped: lmstudio_load_mode=jit") + return try: from agent.model_metadata import MINIMUM_CONTEXT_LENGTH from hermes_cli.models import ensure_lmstudio_model_loaded diff --git a/tests/run_agent/test_lmstudio_load_mode.py b/tests/run_agent/test_lmstudio_load_mode.py new file mode 100644 index 000000000000..50244cc5ea99 --- /dev/null +++ b/tests/run_agent/test_lmstudio_load_mode.py @@ -0,0 +1,63 @@ +from types import SimpleNamespace +from typing import Any, cast + +from run_agent import AIAgent + + +def _agent(load_mode="explicit"): + return SimpleNamespace( + provider="lmstudio", + model="test/model", + base_url="http://127.0.0.1:1234/v1", + api_key="", + lmstudio_load_mode=load_mode, + _config_context_length=None, + context_compressor=None, + api_mode="chat_completions", + ) + + +def test_lmstudio_jit_load_mode_skips_explicit_preload(monkeypatch): + calls = [] + + def fake_ensure(*args, **kwargs): + calls.append((args, kwargs)) + return 64000 + + monkeypatch.setattr("hermes_cli.models.ensure_lmstudio_model_loaded", fake_ensure) + + AIAgent._ensure_lmstudio_runtime_loaded(cast(Any, _agent("jit"))) + + assert calls == [] + + +def test_lmstudio_explicit_load_mode_preserves_preload(monkeypatch): + calls = [] + + def fake_ensure(*args, **kwargs): + calls.append((args, kwargs)) + return 64000 + + monkeypatch.setattr("hermes_cli.models.ensure_lmstudio_model_loaded", fake_ensure) + + AIAgent._ensure_lmstudio_runtime_loaded(cast(Any, _agent("explicit"))) + + assert len(calls) == 1 + assert calls[0][0][:3] == ("test/model", "http://127.0.0.1:1234/v1", "") + assert calls[0][0][3] == 64000 + + +def test_missing_lmstudio_load_mode_defaults_to_explicit(monkeypatch): + calls = [] + agent = _agent() + delattr(agent, "lmstudio_load_mode") + + def fake_ensure(*args, **kwargs): + calls.append((args, kwargs)) + return 64000 + + monkeypatch.setattr("hermes_cli.models.ensure_lmstudio_model_loaded", fake_ensure) + + AIAgent._ensure_lmstudio_runtime_loaded(cast(Any, agent)) + + assert len(calls) == 1 diff --git a/website/docs/integrations/providers.md b/website/docs/integrations/providers.md index d07d27e22b34..e9e94d086305 100644 --- a/website/docs/integrations/providers.md +++ b/website/docs/integrations/providers.md @@ -844,7 +844,7 @@ hermes model # If LM Studio server auth is enabled, enter LM_API_KEY when prompted ``` -Hermes will automatically load a LM Studio model with 64K context length +By default, Hermes explicitly asks LM Studio to load the selected model with 64K context length before the first request. To change context length in LM Studio: @@ -860,6 +860,18 @@ You can use the CLI to estimate if the model will fit: `lms load model-name --co To set persistent per-model defaults: My Models tab → gear icon on the model → set context size. ::: +If you use LM Studio's Just-In-Time loading / Auto-Evict feature and want LM Studio to manage model loading and eviction from normal chat requests, skip Hermes' explicit preload step: + +```bash +hermes config set model.lmstudio_load_mode jit +``` + +Set it back to the default explicit preload behavior with: + +```bash +hermes config set model.lmstudio_load_mode explicit +``` + **Tool calling:** Supported since LM Studio 0.3.6. Models with native tool-calling training (Qwen 2.5, Llama 3.x, Mistral, Hermes) are auto-detected and shown with a tool badge. Other models use a generic fallback that may be less reliable. ---