mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-08 13:12:08 +00:00
fix: route gateway images by session model override
This commit is contained in:
parent
eb3cf9750e
commit
7702071c01
3 changed files with 190 additions and 10 deletions
|
|
@ -8187,7 +8187,10 @@ class GatewayRunner:
|
|||
if image_paths:
|
||||
# Decide routing: native (attach pixels) vs text (vision_analyze
|
||||
# pre-run + prepend description). See agent/image_routing.py.
|
||||
_img_mode = self._decide_image_input_mode()
|
||||
_img_mode = self._decide_image_input_mode(
|
||||
source=source,
|
||||
session_key=session_key,
|
||||
)
|
||||
if _img_mode == "native":
|
||||
# Defer attachment to the run_conversation call site.
|
||||
pending_native = getattr(self, "_pending_native_image_paths_by_session", None)
|
||||
|
|
@ -15085,25 +15088,62 @@ class GatewayRunner:
|
|||
ctx = copy_context()
|
||||
return await loop.run_in_executor(None, ctx.run, func, *args)
|
||||
|
||||
def _decide_image_input_mode(self) -> str:
|
||||
"""Resolve the image-input routing for the currently active model.
|
||||
def _decide_image_input_mode(
|
||||
self,
|
||||
*,
|
||||
source: Optional[SessionSource] = None,
|
||||
session_key: Optional[str] = None,
|
||||
user_config: Optional[dict] = None,
|
||||
provider: Optional[str] = None,
|
||||
model: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Resolve image-input routing for the effective model this turn.
|
||||
|
||||
Returns ``"native"`` (attach pixels on the user turn) or ``"text"``
|
||||
(pre-analyze with vision_analyze and prepend the description). See
|
||||
agent/image_routing.py for the full decision table.
|
||||
|
||||
The active provider/model are read from config.yaml so the decision
|
||||
tracks ``/model`` switches automatically on the next message.
|
||||
Gateway sessions can have /model overrides that live outside
|
||||
config.yaml. Image preprocessing runs before AIAgent sets the
|
||||
auxiliary_client runtime globals, so resolve the same per-session
|
||||
runtime bundle the upcoming agent turn will use instead of consulting
|
||||
only the persisted default model.
|
||||
"""
|
||||
try:
|
||||
from agent.image_routing import decide_image_input_mode
|
||||
from agent.auxiliary_client import _read_main_model, _read_main_provider
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
cfg = load_config()
|
||||
provider = _read_main_provider()
|
||||
model = _read_main_model()
|
||||
return decide_image_input_mode(provider, model, cfg)
|
||||
cfg = user_config if isinstance(user_config, dict) else load_config()
|
||||
resolved_provider = (provider or "").strip()
|
||||
resolved_model = (model or "").strip()
|
||||
|
||||
needs_session_runtime = not resolved_provider or not resolved_model
|
||||
has_session_identity = source is not None or session_key
|
||||
if needs_session_runtime and has_session_identity:
|
||||
try:
|
||||
turn_model, runtime_kwargs = self._resolve_session_agent_runtime(
|
||||
source=source,
|
||||
session_key=session_key,
|
||||
user_config=cfg,
|
||||
)
|
||||
if not resolved_model and isinstance(turn_model, str):
|
||||
resolved_model = turn_model.strip()
|
||||
runtime_provider = runtime_kwargs.get("provider") if isinstance(runtime_kwargs, dict) else None
|
||||
if not resolved_provider and isinstance(runtime_provider, str):
|
||||
resolved_provider = runtime_provider.strip()
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"image_routing: session runtime resolution failed, falling back to config — %s",
|
||||
exc,
|
||||
)
|
||||
|
||||
if not resolved_provider:
|
||||
resolved_provider = _read_main_provider()
|
||||
if not resolved_model:
|
||||
resolved_model = _read_main_model()
|
||||
|
||||
return decide_image_input_mode(resolved_provider, resolved_model, cfg)
|
||||
except Exception as exc:
|
||||
logger.debug("image_routing: decision failed, falling back to text — %s", exc)
|
||||
return "text"
|
||||
|
|
|
|||
140
tests/gateway/test_image_input_routing_runtime.py
Normal file
140
tests/gateway/test_image_input_routing_runtime.py
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
import pytest
|
||||
|
||||
from gateway.config import GatewayConfig, Platform, PlatformConfig
|
||||
from gateway.platforms.base import MessageEvent, MessageType
|
||||
from gateway.run import GatewayRunner
|
||||
from gateway.session import SessionSource
|
||||
|
||||
|
||||
def _make_runner() -> GatewayRunner:
|
||||
runner = object.__new__(GatewayRunner)
|
||||
runner.config = GatewayConfig(
|
||||
platforms={Platform.TELEGRAM: PlatformConfig(enabled=True, token="fake")}
|
||||
)
|
||||
runner.adapters = {}
|
||||
runner._pending_native_image_paths_by_session = {}
|
||||
runner._session_model_overrides = {}
|
||||
runner._session_reasoning_overrides = {}
|
||||
return runner
|
||||
|
||||
|
||||
def _source() -> SessionSource:
|
||||
return SessionSource(
|
||||
platform=Platform.TELEGRAM,
|
||||
chat_id="273403055",
|
||||
chat_type="dm",
|
||||
user_id="42",
|
||||
user_name="Maxim",
|
||||
)
|
||||
|
||||
|
||||
def _image_event(text: str = "look") -> MessageEvent:
|
||||
return MessageEvent(
|
||||
text=text,
|
||||
message_type=MessageType.PHOTO,
|
||||
source=_source(),
|
||||
media_urls=["/tmp/cashback.png"],
|
||||
media_types=["image/png"],
|
||||
)
|
||||
|
||||
|
||||
def _auto_config() -> dict:
|
||||
return {
|
||||
"agent": {"image_input_mode": "auto"},
|
||||
"auxiliary": {"vision": {"provider": "auto", "model": "", "base_url": ""}},
|
||||
"model": {"provider": "xiaomi", "default": "mimo-v2.5-pro"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepare_image_routing_uses_session_vision_model_override(monkeypatch):
|
||||
"""Telegram /model overrides must affect native-vs-text image routing.
|
||||
|
||||
Regression: _prepare_inbound_message_text used config.yaml's default model
|
||||
before the per-session model override was installed on auxiliary_client's
|
||||
runtime globals. A Telegram session switched to a vision model still had
|
||||
screenshots pre-analyzed as text when config.default was text-only.
|
||||
"""
|
||||
runner = _make_runner()
|
||||
source = _source()
|
||||
event = _image_event()
|
||||
cfg = _auto_config()
|
||||
|
||||
monkeypatch.setattr("gateway.run._load_gateway_config", lambda: cfg)
|
||||
monkeypatch.setattr("hermes_cli.config.load_config", lambda: cfg)
|
||||
monkeypatch.setattr("agent.auxiliary_client._read_main_provider", lambda: "xiaomi")
|
||||
monkeypatch.setattr("agent.auxiliary_client._read_main_model", lambda: "mimo-v2.5-pro")
|
||||
monkeypatch.setattr(
|
||||
runner,
|
||||
"_resolve_session_agent_runtime",
|
||||
lambda **_: ("gpt-5.5", {"provider": "openai-codex"}),
|
||||
)
|
||||
|
||||
def fake_supports(provider, model, config):
|
||||
return provider == "openai-codex" and model == "gpt-5.5"
|
||||
|
||||
monkeypatch.setattr("agent.image_routing._lookup_supports_vision", fake_supports)
|
||||
|
||||
async def fail_enrich(*_args, **_kwargs):
|
||||
pytest.fail("vision-capable session override should use native image routing")
|
||||
|
||||
monkeypatch.setattr(runner, "_enrich_message_with_vision", fail_enrich)
|
||||
|
||||
result = await runner._prepare_inbound_message_text(
|
||||
event=event,
|
||||
source=source,
|
||||
history=[],
|
||||
)
|
||||
|
||||
session_key = runner._session_key_for_source(source)
|
||||
assert result == "look"
|
||||
assert runner._pending_native_image_paths_by_session[session_key] == [
|
||||
"/tmp/cashback.png"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepare_image_routing_falls_back_to_text_for_text_only_session_override(monkeypatch):
|
||||
"""A text-only session override should get vision_analyze text fallback.
|
||||
|
||||
Regression mirror case: if config.default is a vision model but the current
|
||||
Telegram session is switched to a text-only provider (for example Mimo),
|
||||
auto routing must not attach pixels natively to the text-only model.
|
||||
"""
|
||||
runner = _make_runner()
|
||||
source = _source()
|
||||
event = _image_event()
|
||||
cfg = _auto_config()
|
||||
cfg["model"] = {"provider": "openai-codex", "default": "gpt-5.5"}
|
||||
|
||||
monkeypatch.setattr("gateway.run._load_gateway_config", lambda: cfg)
|
||||
monkeypatch.setattr("hermes_cli.config.load_config", lambda: cfg)
|
||||
monkeypatch.setattr("agent.auxiliary_client._read_main_provider", lambda: "openai-codex")
|
||||
monkeypatch.setattr("agent.auxiliary_client._read_main_model", lambda: "gpt-5.5")
|
||||
monkeypatch.setattr(
|
||||
runner,
|
||||
"_resolve_session_agent_runtime",
|
||||
lambda **_: ("mimo-v2.5-pro", {"provider": "xiaomi"}),
|
||||
)
|
||||
|
||||
def fake_supports(provider, model, config):
|
||||
return provider == "openai-codex" and model == "gpt-5.5"
|
||||
|
||||
monkeypatch.setattr("agent.image_routing._lookup_supports_vision", fake_supports)
|
||||
|
||||
async def fake_enrich(user_text, image_paths):
|
||||
assert user_text == "look"
|
||||
assert image_paths == ["/tmp/cashback.png"]
|
||||
return "[vision summary]\n\nlook"
|
||||
|
||||
monkeypatch.setattr(runner, "_enrich_message_with_vision", fake_enrich)
|
||||
|
||||
result = await runner._prepare_inbound_message_text(
|
||||
event=event,
|
||||
source=source,
|
||||
history=[],
|
||||
)
|
||||
|
||||
session_key = runner._session_key_for_source(source)
|
||||
assert result == "[vision summary]\n\nlook"
|
||||
assert runner._pending_native_image_paths_by_session.get(session_key) is None
|
||||
|
|
@ -14,7 +14,7 @@ def _make_runner() -> GatewayRunner:
|
|||
runner.adapters = {}
|
||||
runner._model = "openai/gpt-4.1-mini"
|
||||
runner._base_url = None
|
||||
runner._decide_image_input_mode = lambda: "native"
|
||||
runner._decide_image_input_mode = lambda **_: "native"
|
||||
return runner
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue