diff --git a/gateway/run.py b/gateway/run.py index e187ff8a3ec..646a4856928 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -10035,7 +10035,10 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew if image_paths: # Decide routing: native (attach pixels) vs text (vision_analyze # pre-run + prepend description). See agent/image_routing.py. - _img_mode = self._decide_image_input_mode() + _img_mode = self._decide_image_input_mode( + source=source, + session_key=session_key, + ) if _img_mode == "native": # Defer attachment to the run_conversation call site. pending_native = getattr(self, "_pending_native_image_paths_by_session", None) @@ -14381,25 +14384,62 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew except TypeError: executor.shutdown(wait=False) - def _decide_image_input_mode(self) -> str: - """Resolve the image-input routing for the currently active model. + def _decide_image_input_mode( + self, + *, + source: Optional[SessionSource] = None, + session_key: Optional[str] = None, + user_config: Optional[dict] = None, + provider: Optional[str] = None, + model: Optional[str] = None, + ) -> str: + """Resolve image-input routing for the effective model this turn. Returns ``"native"`` (attach pixels on the user turn) or ``"text"`` (pre-analyze with vision_analyze and prepend the description). See agent/image_routing.py for the full decision table. - The active provider/model are read from config.yaml so the decision - tracks ``/model`` switches automatically on the next message. + Gateway sessions can have /model overrides that live outside + config.yaml. Image preprocessing runs before AIAgent sets the + auxiliary_client runtime globals, so resolve the same per-session + runtime bundle the upcoming agent turn will use instead of consulting + only the persisted default model. """ try: from agent.image_routing import decide_image_input_mode from agent.auxiliary_client import _read_main_model, _read_main_provider from hermes_cli.config import load_config - cfg = load_config() - provider = _read_main_provider() - model = _read_main_model() - return decide_image_input_mode(provider, model, cfg) + cfg = user_config if isinstance(user_config, dict) else load_config() + resolved_provider = (provider or "").strip() + resolved_model = (model or "").strip() + + needs_session_runtime = not resolved_provider or not resolved_model + has_session_identity = source is not None or session_key + if needs_session_runtime and has_session_identity: + try: + turn_model, runtime_kwargs = self._resolve_session_agent_runtime( + source=source, + session_key=session_key, + user_config=cfg, + ) + if not resolved_model and isinstance(turn_model, str): + resolved_model = turn_model.strip() + runtime_provider = runtime_kwargs.get("provider") if isinstance(runtime_kwargs, dict) else None + if not resolved_provider and isinstance(runtime_provider, str): + resolved_provider = runtime_provider.strip() + except Exception as exc: + logger.debug( + "image_routing: session runtime resolution failed, falling back to config — %s", + exc, + ) + + if not resolved_provider: + resolved_provider = _read_main_provider() + if not resolved_model: + resolved_model = _read_main_model() + + return decide_image_input_mode(resolved_provider, resolved_model, cfg) except Exception as exc: logger.debug("image_routing: decision failed, falling back to text — %s", exc) return "text" diff --git a/tests/gateway/test_image_input_routing_runtime.py b/tests/gateway/test_image_input_routing_runtime.py new file mode 100644 index 00000000000..5bf34d39015 --- /dev/null +++ b/tests/gateway/test_image_input_routing_runtime.py @@ -0,0 +1,140 @@ +import pytest + +from gateway.config import GatewayConfig, Platform, PlatformConfig +from gateway.platforms.base import MessageEvent, MessageType +from gateway.run import GatewayRunner +from gateway.session import SessionSource + + +def _make_runner() -> GatewayRunner: + runner = object.__new__(GatewayRunner) + runner.config = GatewayConfig( + platforms={Platform.TELEGRAM: PlatformConfig(enabled=True, token="fake")} + ) + runner.adapters = {} + runner._pending_native_image_paths_by_session = {} + runner._session_model_overrides = {} + runner._session_reasoning_overrides = {} + return runner + + +def _source() -> SessionSource: + return SessionSource( + platform=Platform.TELEGRAM, + chat_id="273403055", + chat_type="dm", + user_id="42", + user_name="Maxim", + ) + + +def _image_event(text: str = "look") -> MessageEvent: + return MessageEvent( + text=text, + message_type=MessageType.PHOTO, + source=_source(), + media_urls=["/tmp/cashback.png"], + media_types=["image/png"], + ) + + +def _auto_config() -> dict: + return { + "agent": {"image_input_mode": "auto"}, + "auxiliary": {"vision": {"provider": "auto", "model": "", "base_url": ""}}, + "model": {"provider": "xiaomi", "default": "mimo-v2.5-pro"}, + } + + +@pytest.mark.asyncio +async def test_prepare_image_routing_uses_session_vision_model_override(monkeypatch): + """Telegram /model overrides must affect native-vs-text image routing. + + Regression: _prepare_inbound_message_text used config.yaml's default model + before the per-session model override was installed on auxiliary_client's + runtime globals. A Telegram session switched to a vision model still had + screenshots pre-analyzed as text when config.default was text-only. + """ + runner = _make_runner() + source = _source() + event = _image_event() + cfg = _auto_config() + + monkeypatch.setattr("gateway.run._load_gateway_config", lambda: cfg) + monkeypatch.setattr("hermes_cli.config.load_config", lambda: cfg) + monkeypatch.setattr("agent.auxiliary_client._read_main_provider", lambda: "xiaomi") + monkeypatch.setattr("agent.auxiliary_client._read_main_model", lambda: "mimo-v2.5-pro") + monkeypatch.setattr( + runner, + "_resolve_session_agent_runtime", + lambda **_: ("gpt-5.5", {"provider": "openai-codex"}), + ) + + def fake_supports(provider, model, config): + return provider == "openai-codex" and model == "gpt-5.5" + + monkeypatch.setattr("agent.image_routing._lookup_supports_vision", fake_supports) + + async def fail_enrich(*_args, **_kwargs): + pytest.fail("vision-capable session override should use native image routing") + + monkeypatch.setattr(runner, "_enrich_message_with_vision", fail_enrich) + + result = await runner._prepare_inbound_message_text( + event=event, + source=source, + history=[], + ) + + session_key = runner._session_key_for_source(source) + assert result == "look" + assert runner._pending_native_image_paths_by_session[session_key] == [ + "/tmp/cashback.png" + ] + + +@pytest.mark.asyncio +async def test_prepare_image_routing_falls_back_to_text_for_text_only_session_override(monkeypatch): + """A text-only session override should get vision_analyze text fallback. + + Regression mirror case: if config.default is a vision model but the current + Telegram session is switched to a text-only provider (for example Mimo), + auto routing must not attach pixels natively to the text-only model. + """ + runner = _make_runner() + source = _source() + event = _image_event() + cfg = _auto_config() + cfg["model"] = {"provider": "openai-codex", "default": "gpt-5.5"} + + monkeypatch.setattr("gateway.run._load_gateway_config", lambda: cfg) + monkeypatch.setattr("hermes_cli.config.load_config", lambda: cfg) + monkeypatch.setattr("agent.auxiliary_client._read_main_provider", lambda: "openai-codex") + monkeypatch.setattr("agent.auxiliary_client._read_main_model", lambda: "gpt-5.5") + monkeypatch.setattr( + runner, + "_resolve_session_agent_runtime", + lambda **_: ("mimo-v2.5-pro", {"provider": "xiaomi"}), + ) + + def fake_supports(provider, model, config): + return provider == "openai-codex" and model == "gpt-5.5" + + monkeypatch.setattr("agent.image_routing._lookup_supports_vision", fake_supports) + + async def fake_enrich(user_text, image_paths): + assert user_text == "look" + assert image_paths == ["/tmp/cashback.png"] + return "[vision summary]\n\nlook" + + monkeypatch.setattr(runner, "_enrich_message_with_vision", fake_enrich) + + result = await runner._prepare_inbound_message_text( + event=event, + source=source, + history=[], + ) + + session_key = runner._session_key_for_source(source) + assert result == "[vision summary]\n\nlook" + assert runner._pending_native_image_paths_by_session.get(session_key) is None diff --git a/tests/gateway/test_native_image_buffer_isolation.py b/tests/gateway/test_native_image_buffer_isolation.py index 55f96730a59..dbaa4350a4d 100644 --- a/tests/gateway/test_native_image_buffer_isolation.py +++ b/tests/gateway/test_native_image_buffer_isolation.py @@ -14,7 +14,7 @@ def _make_runner() -> GatewayRunner: runner.adapters = {} runner._model = "openai/gpt-4.1-mini" runner._base_url = None - runner._decide_image_input_mode = lambda: "native" + runner._decide_image_input_mode = lambda **_: "native" return runner