mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
fix(agent): size the Ollama context window from /api/ps, not the trained max
Ollama sizes a model's real context window by free VRAM at load time, often far below the GGUF trained max that /api/show reports, and its OpenAI-compatible endpoint has no options passthrough — per-request num_ctx and keep_alive are silently dropped, so the window cannot be controlled from the client. The compressor was being sized to the trained max (e.g. 262K for a model actually running at 32K). - Add query_ollama_loaded_context() reading the effective window from /api/ps (60s cache, never persisted — transient load state). - Reconcile after each successful response via sync_ollama_loaded_context(): resize the compressor to the loaded window and warn when it is below the tool-use minimum. Selection surfaces keep showing the trained max; explicit model.context_length still wins. No-op for non-Ollama providers. - Refresh model.ollama_keep_alive through the native API (rate-limited /api/generate ping) since /v1 drops it. - Remove the inert num_ctx/keep_alive request-body plumbing; warn that model.ollama_num_ctx has no effect and point at OLLAMA_CONTEXT_LENGTH / Modelfile num_ctx. Keep detection for the pre-flight window check. - Disable thinking on Ollama thinking models via reasoning_effort 'none' — the only switch its /v1 handler parses (think is dropped).
This commit is contained in:
parent
0e4598b271
commit
1908dd09fc
13 changed files with 540 additions and 96 deletions
|
|
@ -1993,23 +1993,22 @@ def init_agent(
|
|||
agent.session_cost_status = "unknown"
|
||||
agent.session_cost_source = "none"
|
||||
|
||||
# ── Ollama num_ctx injection ──
|
||||
# Ollama defaults to 2048 context regardless of the model's capabilities.
|
||||
# When running against an Ollama server, detect the model's max context
|
||||
# and pass num_ctx on every chat request so the full window is used.
|
||||
# User override: set model.ollama_num_ctx in config.yaml to cap VRAM use.
|
||||
# If model.context_length is set, it caps num_ctx so the user's VRAM
|
||||
# budget is respected even when GGUF metadata advertises a larger window.
|
||||
# ── Ollama context-window detection ──
|
||||
# Detect the model's own window (a Modelfile num_ctx pin, else the GGUF
|
||||
# trained max) via /api/show. Pre-flight sanity signal ONLY, consumed by
|
||||
# _ollama_context_limit_error: Ollama's OpenAI-compatible /v1 endpoint
|
||||
# ignores per-request num_ctx, so this value is never sent on the wire.
|
||||
# The window the server actually loads is VRAM-sized and reconciled
|
||||
# post-load from /api/ps (sync_ollama_loaded_context).
|
||||
agent._ollama_num_ctx: int | None = None
|
||||
_ollama_num_ctx_override = None
|
||||
if isinstance(_model_cfg, dict):
|
||||
_ollama_num_ctx_override = _model_cfg.get("ollama_num_ctx")
|
||||
if _ollama_num_ctx_override is not None:
|
||||
try:
|
||||
agent._ollama_num_ctx = int(_ollama_num_ctx_override)
|
||||
except (TypeError, ValueError):
|
||||
_ra().logger.debug("Invalid ollama_num_ctx config value: %r", _ollama_num_ctx_override)
|
||||
if agent._ollama_num_ctx is None and agent.base_url and is_local_endpoint(agent.base_url):
|
||||
if isinstance(_model_cfg, dict) and _model_cfg.get("ollama_num_ctx") is not None:
|
||||
_ra().logger.warning(
|
||||
"model.ollama_num_ctx has no effect and is ignored: Ollama's "
|
||||
"OpenAI-compatible endpoint drops per-request num_ctx. Set the "
|
||||
"window on the server instead — OLLAMA_CONTEXT_LENGTH env var, "
|
||||
"or PARAMETER num_ctx in a Modelfile."
|
||||
)
|
||||
if agent.base_url and is_local_endpoint(agent.base_url):
|
||||
try:
|
||||
# ``agent.api_key`` may be a callable (Entra token provider).
|
||||
# Ollama detection makes a manual HTTP request and expects a
|
||||
|
|
@ -2021,32 +2020,14 @@ def init_agent(
|
|||
agent._ollama_num_ctx = _detected
|
||||
except Exception as exc:
|
||||
_ra().logger.debug("Ollama num_ctx detection failed: %s", exc)
|
||||
# Cap auto-detected ollama_num_ctx to the user's explicit context_length.
|
||||
# Without this, GGUF metadata can advertise 256K+ which Ollama honours
|
||||
# by allocating that much VRAM — blowing up small GPUs even though the
|
||||
# user explicitly set a smaller context_length in config.yaml.
|
||||
if (
|
||||
agent._ollama_num_ctx
|
||||
and _config_context_length
|
||||
and _ollama_num_ctx_override is None # don't override explicit ollama_num_ctx
|
||||
and agent._ollama_num_ctx > _config_context_length
|
||||
):
|
||||
_ra().logger.info(
|
||||
"Ollama num_ctx capped: %d -> %d (model.context_length override)",
|
||||
agent._ollama_num_ctx, _config_context_length,
|
||||
)
|
||||
agent._ollama_num_ctx = _config_context_length
|
||||
if agent._ollama_num_ctx and not agent.quiet_mode:
|
||||
_ra().logger.info(
|
||||
"Ollama num_ctx: will request %d tokens (model max from /api/show)",
|
||||
agent._ollama_num_ctx,
|
||||
)
|
||||
|
||||
# ── Ollama keep_alive ──
|
||||
# How long the server keeps the model resident after a request (Ollama
|
||||
# default: 5 minutes). Set model.ollama_keep_alive in config.yaml to a Go
|
||||
# duration string ("30m", "2h") or seconds; -1 keeps it loaded until the
|
||||
# server exits. Sent per-request alongside num_ctx.
|
||||
# server exits. The OpenAI-compatible endpoint drops keep_alive from
|
||||
# request bodies, so it is applied via the native API after each response
|
||||
# (sync_ollama_loaded_context).
|
||||
agent._ollama_keep_alive = None
|
||||
if isinstance(_model_cfg, dict):
|
||||
_keep_alive_raw = _model_cfg.get("ollama_keep_alive")
|
||||
|
|
|
|||
|
|
@ -2788,6 +2788,124 @@ def intent_ack_continuation_mode(agent) -> str:
|
|||
return "codex_only" if agent.api_mode == "codex_responses" else "off"
|
||||
|
||||
|
||||
def sync_ollama_loaded_context(agent) -> None:
|
||||
"""Reconcile runtime state with the model Ollama actually has loaded.
|
||||
|
||||
Two jobs, both only possible once the model is resident (called after
|
||||
each successful response):
|
||||
|
||||
1. Resize the compressor to the effective context window. Selection
|
||||
shows the trained max (a model property); the server sizes the real
|
||||
window by free VRAM at load time, and the OpenAI-compat /v1 endpoint
|
||||
cannot change it (its request struct has no options field — verified
|
||||
against the Ollama source; a window pinned via the native API is
|
||||
reverted by the next /v1 request). ``/api/ps`` reports the window in
|
||||
effect. Runs BEFORE update_from_response so the calibration reset in
|
||||
update_model is immediately repopulated with this response's usage.
|
||||
An explicit model.context_length override wins unconditionally.
|
||||
|
||||
2. Apply model.ollama_keep_alive through the native API — /v1 drops it
|
||||
for the same reason. One cheap empty /api/generate refreshes the
|
||||
residency timer to the configured duration (idempotent; the model is
|
||||
already loaded, so no work happens server-side).
|
||||
"""
|
||||
if (getattr(agent, "provider", "") or "").strip().lower() != "ollama":
|
||||
return
|
||||
compressor = getattr(agent, "context_compressor", None)
|
||||
if compressor is None:
|
||||
return
|
||||
|
||||
api_key = agent.api_key if isinstance(agent.api_key, str) else ""
|
||||
|
||||
keep_alive = getattr(agent, "_ollama_keep_alive", None)
|
||||
if keep_alive is not None:
|
||||
_refresh_ollama_keep_alive(agent.model, agent.base_url, api_key or "", keep_alive)
|
||||
|
||||
if getattr(agent, "_config_context_length", None):
|
||||
return
|
||||
|
||||
try:
|
||||
from agent.model_metadata import query_ollama_loaded_context
|
||||
|
||||
# Fresh probe (no cache): this is a status question, the server is
|
||||
# local, and a cached miss from before the load would blind the
|
||||
# sync for the TTL.
|
||||
loaded = query_ollama_loaded_context(
|
||||
agent.model, agent.base_url, api_key=api_key or "", use_cache=False
|
||||
)
|
||||
except Exception:
|
||||
return
|
||||
if not loaded or loaded == getattr(compressor, "context_length", 0):
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Ollama loaded context: %s runs at %d tokens (was sized %d); "
|
||||
"resizing compressor to the effective window",
|
||||
agent.model, loaded, getattr(compressor, "context_length", 0),
|
||||
)
|
||||
try:
|
||||
from agent.model_metadata import MINIMUM_CONTEXT_LENGTH
|
||||
|
||||
if loaded < MINIMUM_CONTEXT_LENGTH:
|
||||
logger.warning(
|
||||
"Ollama loaded %s with a %d-token window — below the %d "
|
||||
"minimum Hermes needs for reliable agent work. Raise the "
|
||||
"server's window (OLLAMA_CONTEXT_LENGTH env var or a "
|
||||
"Modelfile num_ctx) or expect aggressive compression.",
|
||||
agent.model, loaded, MINIMUM_CONTEXT_LENGTH,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
compressor.update_model(
|
||||
model=agent.model,
|
||||
context_length=loaded,
|
||||
base_url=agent.base_url,
|
||||
api_key=agent.api_key,
|
||||
provider=agent.provider,
|
||||
api_mode=agent.api_mode,
|
||||
)
|
||||
|
||||
|
||||
def _refresh_ollama_keep_alive(model: str, base_url: str, api_key: str, keep_alive) -> None:
|
||||
"""Refresh the residency timer via native /api/generate (rate-limited).
|
||||
|
||||
Fire-and-forget: an empty prompt against an already-loaded model is a
|
||||
no-op server-side except for resetting the keep_alive expiry. At most
|
||||
once per 60s per (model, base_url) so busy turn loops don't spam it.
|
||||
"""
|
||||
import threading
|
||||
import time as _time
|
||||
|
||||
key = (model, base_url.rstrip("/"))
|
||||
now = _time.monotonic()
|
||||
last = _OLLAMA_KEEP_ALIVE_SENT.get(key, 0.0)
|
||||
if (now - last) < 60.0:
|
||||
return
|
||||
_OLLAMA_KEEP_ALIVE_SENT[key] = now
|
||||
|
||||
def _send() -> None:
|
||||
try:
|
||||
import httpx
|
||||
|
||||
from agent.model_metadata import _auth_headers, _localhost_to_ipv4
|
||||
|
||||
server_url = _localhost_to_ipv4(base_url.rstrip("/"))
|
||||
if server_url.endswith("/v1"):
|
||||
server_url = server_url[:-3]
|
||||
with httpx.Client(timeout=5.0, headers=_auth_headers(api_key)) as client:
|
||||
client.post(
|
||||
f"{server_url}/api/generate",
|
||||
json={"model": model, "keep_alive": keep_alive},
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
threading.Thread(target=_send, daemon=True, name="ollama-keep-alive").start()
|
||||
|
||||
|
||||
_OLLAMA_KEEP_ALIVE_SENT: dict = {}
|
||||
|
||||
|
||||
def intent_ack_continuation_enabled(agent) -> bool:
|
||||
"""Whether intent-ack continuation should fire at all for this turn.
|
||||
|
||||
|
|
@ -3274,6 +3392,7 @@ __all__ = [
|
|||
"cleanup_dead_connections",
|
||||
"extract_api_error_context",
|
||||
"apply_pending_steer_to_tool_results",
|
||||
"sync_ollama_loaded_context",
|
||||
"_iter_pool_sockets",
|
||||
"force_close_tcp_sockets",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -976,8 +976,6 @@ def build_api_kwargs(agent, api_messages: list) -> dict:
|
|||
request_overrides=agent.request_overrides,
|
||||
session_id=getattr(agent, "session_id", None),
|
||||
provider_profile=_profile,
|
||||
ollama_num_ctx=agent._ollama_num_ctx,
|
||||
ollama_keep_alive=getattr(agent, "_ollama_keep_alive", None),
|
||||
ollama_supports_thinking=(
|
||||
agent._ollama_supports_thinking_cached()
|
||||
if (agent.provider or "").strip().lower() == "ollama"
|
||||
|
|
@ -1023,7 +1021,6 @@ def build_api_kwargs(agent, api_messages: list) -> dict:
|
|||
is_tokenhub=_is_tokenhub,
|
||||
is_lmstudio=_is_lmstudio,
|
||||
is_custom_provider=agent.provider == "custom",
|
||||
ollama_num_ctx=agent._ollama_num_ctx,
|
||||
provider_preferences=_prefs or None,
|
||||
openrouter_min_coding_score=agent.openrouter_min_coding_score,
|
||||
qwen_prepare_fn=agent._qwen_prepare_chat_messages if _is_qwen else None,
|
||||
|
|
|
|||
|
|
@ -143,12 +143,12 @@ def _ollama_context_limit_error(agent: Any, request_tokens: int) -> Optional[str
|
|||
f"Ollama loaded `{model}` with only {runtime_ctx:,} tokens of runtime "
|
||||
f"context, but Hermes needs at least {MINIMUM_CONTEXT_LENGTH:,} tokens "
|
||||
"for reliable tool use.\n\n"
|
||||
"Increase the Ollama context for this model and restart/reload the "
|
||||
"model before trying again. A known-good starting point is 65,536 "
|
||||
"tokens. In Hermes config, set `model.ollama_num_ctx: 65536` "
|
||||
"(and `model.context_length: 65536` if you also override the displayed "
|
||||
"model context). If you manage the model through an Ollama Modelfile, "
|
||||
"set `PARAMETER num_ctx 65536` there instead."
|
||||
"Increase the Ollama context for this model and reload it before "
|
||||
"trying again. A known-good starting point is 65,536 tokens: set "
|
||||
"`OLLAMA_CONTEXT_LENGTH=65536` in the Ollama server's environment "
|
||||
"and restart it, or set `PARAMETER num_ctx 65536` in the model's "
|
||||
"Modelfile. (Ollama's OpenAI-compatible API ignores per-request "
|
||||
"num_ctx, so this cannot be fixed from the client side.)"
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -2118,6 +2118,15 @@ def run_conversation(
|
|||
"cache_write_tokens": canonical_usage.cache_write_tokens,
|
||||
"reasoning_tokens": canonical_usage.reasoning_tokens,
|
||||
}
|
||||
# Reconcile with the window Ollama actually loaded before
|
||||
# the usage update: the /v1 endpoint cannot pin num_ctx,
|
||||
# so the served window (VRAM-tiered) is discoverable only
|
||||
# now that the model is resident. No-op for other providers.
|
||||
try:
|
||||
from agent.agent_runtime_helpers import sync_ollama_loaded_context
|
||||
sync_ollama_loaded_context(agent)
|
||||
except Exception as _ollama_sync_exc: # pragma: no cover - defensive
|
||||
logger.debug("Ollama loaded-context sync failed: %s", _ollama_sync_exc)
|
||||
agent.context_compressor.update_from_response(usage_dict)
|
||||
elif getattr(
|
||||
agent.context_compressor,
|
||||
|
|
|
|||
|
|
@ -1594,6 +1594,79 @@ def query_ollama_supports_thinking(model: str, base_url: str, api_key: str = "")
|
|||
return None
|
||||
|
||||
|
||||
def query_ollama_loaded_context(
|
||||
model: str, base_url: str, api_key: str = "", use_cache: bool = True
|
||||
) -> Optional[int]:
|
||||
"""Context window the Ollama server actually loaded ``model`` with.
|
||||
|
||||
``/api/show`` reports the GGUF trained maximum, but Ollama sizes the
|
||||
real window by free VRAM at load time, and the OpenAI-compat ``/v1``
|
||||
endpoint has no options field — requests cannot change it (a window
|
||||
pinned via the native API is reverted by the next /v1 request).
|
||||
``/api/ps`` is the only source that reports the window in effect.
|
||||
|
||||
Returns None when the server is unreachable, not Ollama, or the model
|
||||
is not currently loaded. Results are cached briefly (the window only
|
||||
changes on reload, and every /v1 request re-normalizes it to the
|
||||
server default, so it is stable minute-to-minute). Never persisted to
|
||||
the on-disk context cache: this is transient load state, not model
|
||||
metadata.
|
||||
"""
|
||||
import time as _time
|
||||
|
||||
import httpx
|
||||
|
||||
bare_model = _strip_provider_prefix(model)
|
||||
if not bare_model or not base_url:
|
||||
return None
|
||||
|
||||
cache_key = ("ollama_ps", bare_model, base_url.rstrip("/"))
|
||||
now = _time.monotonic()
|
||||
if use_cache:
|
||||
cached = _OLLAMA_PS_CACHE.get(cache_key)
|
||||
if cached is not None and (now - cached[1]) < _OLLAMA_PS_TTL_SECONDS:
|
||||
return cached[0]
|
||||
|
||||
server_url = _localhost_to_ipv4(base_url.rstrip("/"))
|
||||
if server_url.endswith("/v1"):
|
||||
server_url = server_url[:-3]
|
||||
|
||||
result: Optional[int] = None
|
||||
try:
|
||||
with httpx.Client(timeout=3.0, headers=_auth_headers(api_key)) as client:
|
||||
resp = client.get(f"{server_url}/api/ps")
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
models = data.get("models")
|
||||
if isinstance(models, list):
|
||||
base = bare_model.split(":", 1)[0]
|
||||
for entry in models:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
name = str(entry.get("name") or entry.get("model") or "")
|
||||
# Exact tag match; a tagless config value ("gemma4")
|
||||
# matches any loaded tag of the same base model.
|
||||
if name == bare_model or (
|
||||
":" not in bare_model
|
||||
and name.split(":", 1)[0] == base
|
||||
):
|
||||
ctx = entry.get("context_length")
|
||||
if isinstance(ctx, (int, float)) and ctx > 0:
|
||||
result = int(ctx)
|
||||
break
|
||||
except Exception:
|
||||
result = None
|
||||
|
||||
_OLLAMA_PS_CACHE[cache_key] = (result, now)
|
||||
return result
|
||||
|
||||
|
||||
# Loaded-window probes are re-checked frequently enough to notice an
|
||||
# eviction/reload but never on the request hot path more than once a minute.
|
||||
_OLLAMA_PS_CACHE: dict = {}
|
||||
_OLLAMA_PS_TTL_SECONDS = 60.0
|
||||
|
||||
|
||||
def _query_ollama_api_show(model: str, base_url: str, api_key: str = "") -> Optional[int]:
|
||||
"""Query an Ollama server's native ``/api/show`` for context length.
|
||||
|
||||
|
|
@ -2280,6 +2353,13 @@ def get_model_context_length(
|
|||
# 2b. Ollama native /api/show — any URL might be an Ollama server
|
||||
# (local, cloud, or custom hosting). Non-Ollama servers return
|
||||
# 404/405 quickly. Fall through on failure.
|
||||
#
|
||||
# Deliberately the trained max, NOT the /api/ps loaded window:
|
||||
# selection-time surfaces (picker, /api/model/info, init) show
|
||||
# the model's own property. The loaded window is transient state
|
||||
# reconciled post-load by sync_ollama_loaded_context() — feeding
|
||||
# it here would also fail agent init's minimum-context check for
|
||||
# a model currently resident with a small VRAM-tiered window.
|
||||
ctx = _query_ollama_api_show(model, base_url, api_key=api_key)
|
||||
if ctx is not None:
|
||||
if not _skip_persistent_context_cache(base_url, provider):
|
||||
|
|
|
|||
|
|
@ -307,8 +307,6 @@ class ChatCompletionsTransport(ProviderTransport):
|
|||
is_tokenhub: bool
|
||||
is_lmstudio: bool
|
||||
is_custom_provider: bool
|
||||
ollama_num_ctx: int | None
|
||||
ollama_keep_alive: int | str | None
|
||||
ollama_supports_thinking: bool | None
|
||||
# Provider routing
|
||||
provider_preferences: dict | None
|
||||
|
|
@ -587,8 +585,6 @@ class ChatCompletionsTransport(ProviderTransport):
|
|||
qwen_session_metadata=params.get("qwen_session_metadata"),
|
||||
model=model,
|
||||
base_url=params.get("base_url"),
|
||||
ollama_num_ctx=params.get("ollama_num_ctx"),
|
||||
ollama_keep_alive=params.get("ollama_keep_alive"),
|
||||
ollama_supports_thinking=params.get("ollama_supports_thinking"),
|
||||
session_id=params.get("session_id"),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,12 +3,16 @@
|
|||
Covers any endpoint registered as provider="custom", plus the first-class
|
||||
"ollama" provider (routed here by alias), and OpenAI-compatible reasoning
|
||||
endpoints (GLM-5.2 on Volcengine ARK, vLLM, llama.cpp). Key quirks:
|
||||
- ollama_num_ctx → extra_body.options.num_ctx (local context window)
|
||||
- ollama_keep_alive → extra_body.keep_alive (model residence time)
|
||||
- reasoning_config disabled → extra_body.think = False
|
||||
- reasoning_config disabled → reasoning_effort "none" on Ollama thinking
|
||||
models (its /v1 disable switch), extra_body.think = False elsewhere
|
||||
- reasoning_config enabled + effort → top-level reasoning_effort
|
||||
(the native OpenAI-compatible format GLM/ARK expect; unset omits it
|
||||
so the endpoint's server default applies)
|
||||
|
||||
Ollama's OpenAI-compatible endpoint has no options passthrough: num_ctx
|
||||
and keep_alive in the request body are silently dropped, so this profile
|
||||
does not emit them. The context window is server-controlled (reconciled
|
||||
post-load from /api/ps); keep_alive is refreshed via the native API.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
|
@ -24,30 +28,22 @@ class CustomProfile(ProviderProfile):
|
|||
self,
|
||||
*,
|
||||
reasoning_config: dict | None = None,
|
||||
ollama_num_ctx: int | None = None,
|
||||
ollama_keep_alive: int | str | None = None,
|
||||
ollama_supports_thinking: bool | None = None,
|
||||
**ctx: Any,
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
extra_body: dict[str, Any] = {}
|
||||
top_level: dict[str, Any] = {}
|
||||
|
||||
# Ollama context window
|
||||
if ollama_num_ctx:
|
||||
options = extra_body.get("options", {})
|
||||
options["num_ctx"] = ollama_num_ctx
|
||||
extra_body["options"] = options
|
||||
|
||||
# Ollama model residence time after the request (default 5m). Go
|
||||
# duration string or seconds; -1 = keep loaded until server exit.
|
||||
# Ignored by non-Ollama OpenAI-compatible servers.
|
||||
if ollama_keep_alive is not None:
|
||||
extra_body["keep_alive"] = ollama_keep_alive
|
||||
|
||||
# Reasoning / thinking control for custom OpenAI-compatible endpoints
|
||||
# (GLM-5.2 on Volcengine ARK, vLLM, Ollama, llama.cpp, …).
|
||||
#
|
||||
# - disabled → extra_body.think = False (Ollama's thinking-off flag)
|
||||
# - disabled + Ollama thinking model → TOP-LEVEL reasoning_effort
|
||||
# "none". Ollama's /v1 handler parses only reasoning_effort and
|
||||
# maps "none" to think=false internally; an extra_body ``think``
|
||||
# is an unknown field Go silently drops (verified live: think
|
||||
# had no effect, effort "none" suppressed reasoning).
|
||||
# - disabled elsewhere → extra_body.think = False (legacy shape for
|
||||
# non-Ollama endpoints that do parse it, e.g. ARK).
|
||||
# - enabled + effort set → TOP-LEVEL reasoning_effort string, the
|
||||
# format GLM-5.2/ARK and other OpenAI-compatible reasoning APIs
|
||||
# expect (GLM documents "high" and "max"; "max" is its default).
|
||||
|
|
@ -74,7 +70,10 @@ class CustomProfile(ProviderProfile):
|
|||
# unknown/non-Ollama and changes nothing.
|
||||
pass
|
||||
elif _effort == "none" or _enabled is False:
|
||||
extra_body["think"] = False
|
||||
if ollama_supports_thinking is True:
|
||||
top_level["reasoning_effort"] = "none"
|
||||
else:
|
||||
extra_body["think"] = False
|
||||
elif _effort:
|
||||
_aliases = {"xhigh": "max", "minimal": "low"}
|
||||
top_level["reasoning_effort"] = _aliases.get(_effort, _effort)
|
||||
|
|
|
|||
207
tests/agent/test_ollama_loaded_context.py
Normal file
207
tests/agent/test_ollama_loaded_context.py
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
"""Ollama loaded-context reconciliation.
|
||||
|
||||
Ollama sizes a model's real context window by free VRAM at load time —
|
||||
often far below the GGUF trained max that /api/show reports — and the
|
||||
OpenAI-compatible /v1 endpoint cannot change it (its request struct has no
|
||||
options field, so per-request num_ctx is silently dropped). ``/api/ps`` is
|
||||
the only source that reports the window in effect, and only after load.
|
||||
|
||||
Contract under test:
|
||||
- query_ollama_loaded_context() reads /api/ps for the model's effective
|
||||
window; None when unreachable / not loaded / malformed.
|
||||
- Selection-time resolution (get_model_context_length) keeps the trained
|
||||
max — the loaded window is transient state, not model metadata.
|
||||
- sync_ollama_loaded_context() resizes the compressor post-response and
|
||||
refreshes keep_alive via the native API (both impossible over /v1).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import agent.model_metadata as mm
|
||||
from agent.agent_runtime_helpers import sync_ollama_loaded_context
|
||||
from agent.model_metadata import query_ollama_loaded_context
|
||||
|
||||
|
||||
BASE_URL = "http://127.0.0.1:11434/v1"
|
||||
|
||||
|
||||
def _ps_response(models):
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
resp.json.return_value = {"models": models}
|
||||
return resp
|
||||
|
||||
|
||||
def _client_returning(resp):
|
||||
client = MagicMock()
|
||||
client.__enter__ = MagicMock(return_value=client)
|
||||
client.__exit__ = MagicMock(return_value=False)
|
||||
client.get.return_value = resp
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_ps_cache():
|
||||
mm._OLLAMA_PS_CACHE.clear()
|
||||
yield
|
||||
mm._OLLAMA_PS_CACHE.clear()
|
||||
|
||||
|
||||
class TestQueryOllamaLoadedContext:
|
||||
def test_returns_loaded_window_for_exact_tag(self):
|
||||
resp = _ps_response(
|
||||
[{"name": "gemma4:31b", "context_length": 32768, "size_vram": 20282741882}]
|
||||
)
|
||||
with patch("httpx.Client", return_value=_client_returning(resp)):
|
||||
assert query_ollama_loaded_context("gemma4:31b", BASE_URL) == 32768
|
||||
|
||||
def test_tagless_config_matches_any_loaded_tag(self):
|
||||
resp = _ps_response([{"name": "gemma4:latest", "context_length": 16384}])
|
||||
with patch("httpx.Client", return_value=_client_returning(resp)):
|
||||
assert query_ollama_loaded_context("gemma4", BASE_URL) == 16384
|
||||
|
||||
def test_different_tag_does_not_match(self):
|
||||
resp = _ps_response([{"name": "gemma4:31b", "context_length": 32768}])
|
||||
with patch("httpx.Client", return_value=_client_returning(resp)):
|
||||
assert query_ollama_loaded_context("gemma4:9b", BASE_URL) is None
|
||||
|
||||
def test_not_loaded_returns_none(self):
|
||||
with patch("httpx.Client", return_value=_client_returning(_ps_response([]))):
|
||||
assert query_ollama_loaded_context("gemma4:31b", BASE_URL) is None
|
||||
|
||||
def test_unreachable_returns_none(self):
|
||||
with patch("httpx.Client", side_effect=ConnectionError("refused")):
|
||||
assert query_ollama_loaded_context("gemma4:31b", BASE_URL) is None
|
||||
|
||||
def test_non_200_returns_none(self):
|
||||
resp = MagicMock()
|
||||
resp.status_code = 404
|
||||
with patch("httpx.Client", return_value=_client_returning(resp)):
|
||||
assert query_ollama_loaded_context("gemma4:31b", BASE_URL) is None
|
||||
|
||||
def test_cached_result_skips_second_probe(self):
|
||||
resp = _ps_response([{"name": "gemma4:31b", "context_length": 32768}])
|
||||
client = _client_returning(resp)
|
||||
with patch("httpx.Client", return_value=client) as factory:
|
||||
assert query_ollama_loaded_context("gemma4:31b", BASE_URL) == 32768
|
||||
assert query_ollama_loaded_context("gemma4:31b", BASE_URL) == 32768
|
||||
assert factory.call_count == 1
|
||||
|
||||
def test_use_cache_false_reprobes(self):
|
||||
resp = _ps_response([{"name": "gemma4:31b", "context_length": 32768}])
|
||||
client = _client_returning(resp)
|
||||
with patch("httpx.Client", return_value=client) as factory:
|
||||
query_ollama_loaded_context("gemma4:31b", BASE_URL)
|
||||
query_ollama_loaded_context("gemma4:31b", BASE_URL, use_cache=False)
|
||||
assert factory.call_count == 2
|
||||
|
||||
|
||||
class TestSelectionResolutionUnaffected:
|
||||
def test_get_model_context_length_ignores_loaded_window(self):
|
||||
"""Selection surfaces show the trained max, not the transient loaded
|
||||
window — feeding /api/ps into resolution would also fail agent init's
|
||||
minimum-context check for a model resident with a small window."""
|
||||
with (
|
||||
patch.object(mm, "query_ollama_loaded_context") as ps_probe,
|
||||
patch.object(mm, "_query_ollama_api_show", return_value=262144),
|
||||
patch.object(mm, "_skip_persistent_context_cache", return_value=True),
|
||||
):
|
||||
ctx = mm.get_model_context_length(
|
||||
"gemma4:31b", base_url=BASE_URL, provider="ollama"
|
||||
)
|
||||
assert ctx == 262144
|
||||
ps_probe.assert_not_called()
|
||||
|
||||
|
||||
def _ollama_agent(context_length=262144, config_ctx=None, keep_alive=None):
|
||||
agent = MagicMock()
|
||||
agent.provider = "ollama"
|
||||
agent.model = "gemma4:31b"
|
||||
agent.base_url = BASE_URL
|
||||
agent.api_key = "dummy-test-key"
|
||||
agent.api_mode = "chat_completions"
|
||||
agent._config_context_length = config_ctx
|
||||
agent._ollama_keep_alive = keep_alive
|
||||
agent.context_compressor.context_length = context_length
|
||||
return agent
|
||||
|
||||
|
||||
class TestSyncOllamaLoadedContext:
|
||||
def test_resizes_compressor_to_loaded_window(self):
|
||||
agent = _ollama_agent()
|
||||
with patch(
|
||||
"agent.model_metadata.query_ollama_loaded_context", return_value=32768
|
||||
):
|
||||
sync_ollama_loaded_context(agent)
|
||||
agent.context_compressor.update_model.assert_called_once()
|
||||
assert (
|
||||
agent.context_compressor.update_model.call_args.kwargs["context_length"]
|
||||
== 32768
|
||||
)
|
||||
|
||||
def test_noop_when_window_already_matches(self):
|
||||
agent = _ollama_agent(context_length=32768)
|
||||
with patch(
|
||||
"agent.model_metadata.query_ollama_loaded_context", return_value=32768
|
||||
):
|
||||
sync_ollama_loaded_context(agent)
|
||||
agent.context_compressor.update_model.assert_not_called()
|
||||
|
||||
def test_noop_for_non_ollama_provider(self):
|
||||
agent = _ollama_agent()
|
||||
agent.provider = "anthropic"
|
||||
with patch(
|
||||
"agent.model_metadata.query_ollama_loaded_context", return_value=32768
|
||||
) as probe:
|
||||
sync_ollama_loaded_context(agent)
|
||||
probe.assert_not_called()
|
||||
agent.context_compressor.update_model.assert_not_called()
|
||||
|
||||
def test_explicit_config_context_length_wins(self):
|
||||
agent = _ollama_agent(config_ctx=65536)
|
||||
with patch(
|
||||
"agent.model_metadata.query_ollama_loaded_context", return_value=32768
|
||||
) as probe:
|
||||
sync_ollama_loaded_context(agent)
|
||||
probe.assert_not_called()
|
||||
agent.context_compressor.update_model.assert_not_called()
|
||||
|
||||
def test_probe_failure_leaves_compressor_alone(self):
|
||||
agent = _ollama_agent()
|
||||
with patch(
|
||||
"agent.model_metadata.query_ollama_loaded_context", return_value=None
|
||||
):
|
||||
sync_ollama_loaded_context(agent)
|
||||
agent.context_compressor.update_model.assert_not_called()
|
||||
|
||||
def test_keep_alive_refreshed_via_native_api(self):
|
||||
agent = _ollama_agent(keep_alive="30m")
|
||||
with (
|
||||
patch(
|
||||
"agent.model_metadata.query_ollama_loaded_context", return_value=None
|
||||
),
|
||||
patch(
|
||||
"agent.agent_runtime_helpers._refresh_ollama_keep_alive"
|
||||
) as refresh,
|
||||
):
|
||||
sync_ollama_loaded_context(agent)
|
||||
refresh.assert_called_once_with(
|
||||
"gemma4:31b", BASE_URL, "dummy-test-key", "30m"
|
||||
)
|
||||
|
||||
def test_no_keep_alive_config_no_refresh(self):
|
||||
agent = _ollama_agent()
|
||||
with (
|
||||
patch(
|
||||
"agent.model_metadata.query_ollama_loaded_context", return_value=None
|
||||
),
|
||||
patch(
|
||||
"agent.agent_runtime_helpers._refresh_ollama_keep_alive"
|
||||
) as refresh,
|
||||
):
|
||||
sync_ollama_loaded_context(agent)
|
||||
refresh.assert_not_called()
|
||||
|
|
@ -410,16 +410,18 @@ class TestChatCompletionsBuildKwargs:
|
|||
# Nous rejects enabled=false; reasoning omitted entirely
|
||||
assert "reasoning" not in kw.get("extra_body", {})
|
||||
|
||||
def test_ollama_num_ctx(self, transport):
|
||||
def test_ollama_num_ctx_not_emitted(self, transport):
|
||||
"""Ollama /v1 has no options passthrough — emitting num_ctx would be
|
||||
silently dropped, so the transport must not send it at all."""
|
||||
from providers import get_provider_profile
|
||||
profile = get_provider_profile("custom")
|
||||
msgs = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="llama3", messages=msgs,
|
||||
provider_profile=profile,
|
||||
ollama_num_ctx=32768,
|
||||
)
|
||||
assert kw["extra_body"]["options"]["num_ctx"] == 32768
|
||||
assert "options" not in kw.get("extra_body", {})
|
||||
assert "keep_alive" not in kw.get("extra_body", {})
|
||||
|
||||
def test_custom_think_false(self, transport):
|
||||
from providers import get_provider_profile
|
||||
|
|
|
|||
|
|
@ -7,12 +7,15 @@ nothing when reasoning was *enabled*, so a configured ``reasoning_effort``
|
|||
was silently dropped for every custom endpoint.
|
||||
|
||||
These tests pin the wire-shape contract:
|
||||
- disabled → extra_body.think = False
|
||||
- disabled → extra_body.think = False (non-Ollama endpoints);
|
||||
reasoning_effort "none" when the Ollama server
|
||||
confirmed the model thinks (its /v1 disable switch)
|
||||
- enabled + effort → top-level reasoning_effort (native OpenAI-compat
|
||||
format GLM/ARK expect); OpenAI-only levels
|
||||
(xhigh/minimal) map to the nearest accepted level
|
||||
- enabled + no effort → nothing emitted (endpoint's server default applies)
|
||||
- ollama_num_ctx → extra_body.options.num_ctx, orthogonal to reasoning
|
||||
- num_ctx/keep_alive → never emitted (Ollama /v1 silently drops them; the
|
||||
window is reconciled post-load from /api/ps)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -118,21 +121,58 @@ class TestCustomReasoningWireShape:
|
|||
assert eb.get("think") is not True
|
||||
|
||||
|
||||
class TestCustomReasoningWithNumCtx:
|
||||
"""Ollama num_ctx and reasoning are independent and compose."""
|
||||
class TestCustomOllamaThinkingDisable:
|
||||
"""Confirmed-thinking Ollama models disable via reasoning_effort 'none'."""
|
||||
|
||||
def test_num_ctx_alone(self, custom_profile):
|
||||
eb, tl = custom_profile.build_api_kwargs_extras(
|
||||
reasoning_config=None, ollama_num_ctx=8192, model="qwen3"
|
||||
)
|
||||
assert eb == {"options": {"num_ctx": 8192}}
|
||||
assert tl == {}
|
||||
def test_disabled_with_thinking_support_sends_effort_none(self, custom_profile):
|
||||
"""ollama_supports_thinking=True → reasoning_effort 'none' top-level.
|
||||
|
||||
def test_num_ctx_with_effort(self, custom_profile):
|
||||
Ollama's /v1 handler only parses reasoning_effort ('none' maps to
|
||||
think=false internally); an extra_body think flag is an unknown field
|
||||
Go silently drops. Verified live: think=False had no effect, effort
|
||||
'none' suppressed reasoning.
|
||||
"""
|
||||
eb, tl = custom_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
ollama_num_ctx=8192,
|
||||
reasoning_config={"enabled": False},
|
||||
ollama_supports_thinking=True,
|
||||
model="qwen3",
|
||||
)
|
||||
assert eb == {"options": {"num_ctx": 8192}}
|
||||
assert tl == {"reasoning_effort": "none"}
|
||||
assert eb == {}
|
||||
|
||||
def test_effort_none_with_thinking_support_sends_effort_none(self, custom_profile):
|
||||
eb, tl = custom_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "none"},
|
||||
ollama_supports_thinking=True,
|
||||
model="qwen3",
|
||||
)
|
||||
assert tl == {"reasoning_effort": "none"}
|
||||
assert eb == {}
|
||||
|
||||
def test_no_thinking_support_emits_nothing(self, custom_profile):
|
||||
"""ollama_supports_thinking=False → emit no reasoning fields at all.
|
||||
|
||||
Ollama 400s on reasoning_effort (any value) for non-thinking models;
|
||||
a session's effort dial carried over from a thinking model must not
|
||||
brick the chat.
|
||||
"""
|
||||
eb, tl = custom_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
ollama_supports_thinking=False,
|
||||
model="hermes3:8b",
|
||||
)
|
||||
assert eb == {}
|
||||
assert tl == {}
|
||||
|
||||
|
||||
class TestCustomOllamaNoOptionsPassthrough:
|
||||
"""Ollama /v1 silently drops options/keep_alive — never emit them."""
|
||||
|
||||
def test_request_body_has_no_ollama_options(self, custom_profile):
|
||||
eb, tl = custom_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "high"},
|
||||
model="qwen3",
|
||||
)
|
||||
assert "options" not in eb
|
||||
assert "keep_alive" not in eb
|
||||
assert tl == {"reasoning_effort": "high"}
|
||||
|
|
|
|||
|
|
@ -32,7 +32,6 @@ class TestNvidiaProfileWiring:
|
|||
reasoning_config=None,
|
||||
request_overrides=None,
|
||||
session_id="test",
|
||||
ollama_num_ctx=None,
|
||||
)
|
||||
# NVIDIA profile sets default_max_tokens=16384
|
||||
assert kwargs.get("max_tokens") == 16384
|
||||
|
|
@ -56,7 +55,6 @@ class TestNvidiaProfileWiring:
|
|||
reasoning_config=None,
|
||||
request_overrides=None,
|
||||
session_id="test",
|
||||
ollama_num_ctx=None,
|
||||
)
|
||||
assert kwargs["model"] == "nvidia/test-model"
|
||||
|
||||
|
|
@ -74,7 +72,6 @@ class TestNvidiaProfileWiring:
|
|||
reasoning_config=None,
|
||||
request_overrides=None,
|
||||
session_id="test",
|
||||
ollama_num_ctx=None,
|
||||
)
|
||||
assert kwargs["messages"] == msgs
|
||||
|
||||
|
|
@ -93,7 +90,6 @@ class TestDeepSeekProfileWiring:
|
|||
reasoning_config=None,
|
||||
request_overrides=None,
|
||||
session_id="test",
|
||||
ollama_num_ctx=None,
|
||||
)
|
||||
# DeepSeek has no default_max_tokens
|
||||
assert kwargs["model"] == "deepseek-chat"
|
||||
|
|
@ -113,6 +109,5 @@ class TestDeepSeekProfileWiring:
|
|||
reasoning_config=None,
|
||||
request_overrides=None,
|
||||
session_id="test",
|
||||
ollama_num_ctx=None,
|
||||
)
|
||||
assert kwargs["messages"] == msgs
|
||||
|
|
|
|||
|
|
@ -285,17 +285,22 @@ class TestQwenParity:
|
|||
|
||||
|
||||
class TestCustomOllamaParity:
|
||||
"""Custom/Ollama: num_ctx, thinking controls — now tested via profile."""
|
||||
"""Custom/Ollama: request-body contract — now tested via profile.
|
||||
|
||||
def test_ollama_num_ctx(self, transport):
|
||||
Ollama's OpenAI-compatible endpoint has no options passthrough, so the
|
||||
transport must NOT emit num_ctx/keep_alive (they'd be silently dropped;
|
||||
the window is reconciled post-load from /api/ps instead).
|
||||
"""
|
||||
|
||||
def test_no_ollama_options_in_request_body(self, transport):
|
||||
kw = transport.build_kwargs(
|
||||
model="llama3.1",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("custom"),
|
||||
ollama_num_ctx=131072,
|
||||
)
|
||||
assert kw["extra_body"]["options"]["num_ctx"] == 131072
|
||||
assert "options" not in kw.get("extra_body", {})
|
||||
assert "keep_alive" not in kw.get("extra_body", {})
|
||||
|
||||
def test_think_false_when_disabled(self, transport):
|
||||
kw = transport.build_kwargs(
|
||||
|
|
@ -306,3 +311,17 @@ class TestCustomOllamaParity:
|
|||
reasoning_config={"enabled": False, "effort": "none"},
|
||||
)
|
||||
assert kw["extra_body"]["think"] is False
|
||||
|
||||
def test_effort_none_on_ollama_thinking_model(self, transport):
|
||||
"""A confirmed-thinking Ollama model disables via reasoning_effort
|
||||
'none' — the only switch its /v1 handler parses (think is dropped)."""
|
||||
kw = transport.build_kwargs(
|
||||
model="qwen3:72b",
|
||||
messages=_simple_messages(),
|
||||
tools=None,
|
||||
provider_profile=get_provider_profile("custom"),
|
||||
reasoning_config={"enabled": False, "effort": "none"},
|
||||
ollama_supports_thinking=True,
|
||||
)
|
||||
assert kw["reasoning_effort"] == "none"
|
||||
assert "think" not in kw.get("extra_body", {})
|
||||
|
|
|
|||
|
|
@ -4003,7 +4003,7 @@ class TestRunConversation:
|
|||
assert result["api_calls"] == 0
|
||||
assert result["turn_exit_reason"] == "ollama_runtime_context_too_small"
|
||||
assert "Ollama loaded `qwen3.5:9b` with only 4,096 tokens" in result["final_response"]
|
||||
assert "model.ollama_num_ctx: 65536" in result["final_response"]
|
||||
assert "OLLAMA_CONTEXT_LENGTH=65536" in result["final_response"]
|
||||
assert not agent.client.chat.completions.create.called
|
||||
assert "Ollama runtime context too small for Hermes tool use" in caplog.text
|
||||
assert "runtime_context=4096" in caplog.text
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue