hermes-agent/tools/computer_use/vision_routing.py

"""Vision-routing decisions for ``computer_use`` capture results.

Background
----------
``computer_use(action='capture', mode='som'|'vision')`` returns a
``_multimodal`` envelope containing the captured screenshot. That envelope
is delivered back to the **active session model** as the tool result. When
the active main model has no vision capability (e.g. text-only or
text+code-only models), or when the active provider rejects multimodal
content inside tool-result messages, the screenshot trips a 404 / 400 at
the provider boundary and the agent loop reports a hard tool failure.

Issue #24015 reports this regression for the ``cua-driver`` backend:
configuring ``auxiliary.vision`` (a dedicated vision-capable model) in
``config.yaml`` was silently ignored — the screenshot was still routed at
the *main* model and failed with HTTP 404 ``No endpoints found that
support image input`` even though a perfectly good vision backend was
sitting in config waiting to be used.

This module centralises the small policy decision: should a captured
screenshot be returned as multimodal content (main model handles vision
natively) or pre-analysed via the auxiliary vision pipeline so the main
model only ever sees text?

Behaviour (mirrors ``vision_analyze`` for consistency)
------------------------------------------------------
* If the user explicitly configured ``auxiliary.vision`` (any of
  ``provider``, ``model``, or ``base_url`` non-empty / not ``"auto"``),
  the screenshot is routed through the aux vision pipeline. Users who
  pay for a dedicated vision model usually want it used.
* Otherwise, if the user explicitly declared the active model vision-capable
  via ``model.supports_vision`` / provider model config, return ``False``.
  This is the escape hatch for custom/local OpenAI-compatible VLM routes that
  are absent from models.dev and provider allowlists.
* Otherwise, if the active main model+provider can carry an image inside
  a tool-result message AND the model reports ``supports_vision=True``
  in models.dev metadata, return ``False`` (use the multimodal path).
* In every other case (non-vision main model, provider that does not
  accept multimodal tool results, lookup failure), route through aux
  vision so the main model receives a text description it can act on.

The decision intentionally fails *closed* (i.e. towards aux routing) when
metadata is missing or ambiguous: returning a screenshot to a model that
cannot read it is a hard tool failure, while routing it through aux costs
one extra LLM call and yields a usable description.
"""

from __future__ import annotations

import logging
from typing import Any, Dict, Optional

logger = logging.getLogger(__name__)


def _explicit_aux_vision_override(cfg: Optional[Dict[str, Any]]) -> bool:
    """True when ``auxiliary.vision`` carries a non-default user override.

    Mirrors ``agent.image_routing._explicit_aux_vision_override`` so the
    capture path and the user-attached-image path agree on what counts as
    an explicit user request for the aux vision pipeline. ``provider:
    "auto"``, blank values, or a missing block all count as *not*
    explicit.
    """
    if not isinstance(cfg, dict):
        return False
    aux = cfg.get("auxiliary") or {}
    if not isinstance(aux, dict):
        return False
    vision = aux.get("vision") or {}
    if not isinstance(vision, dict):
        return False

    provider = str(vision.get("provider") or "").strip().lower()
    model = str(vision.get("model") or "").strip()
    base_url = str(vision.get("base_url") or "").strip()

    if provider in ("", "auto") and not model and not base_url:
        return False
    return True


def _lookup_user_declared_supports_vision(
    provider: str,
    model: str,
    cfg: Optional[Dict[str, Any]],
) -> Optional[bool]:
    """Return config-declared ``supports_vision`` for the active route."""
    try:
        from agent.image_routing import _supports_vision_override
    except Exception as exc:  # pragma: no cover - defensive
        logger.debug(
            "computer_use vision_routing: config override lookup import failed: %s",
            exc,
        )
        return None
    try:
        return _supports_vision_override(cfg, provider, model)
    except Exception as exc:  # pragma: no cover - defensive
        logger.debug(
            "computer_use vision_routing: config override lookup failed: %s",
            exc,
        )
        return None


def _lookup_supports_vision(
    provider: str,
    model: str,
    cfg: Optional[Dict[str, Any]] = None,
) -> Optional[bool]:
    """Return config/models.dev ``supports_vision`` for *(provider, model)*."""
    if not provider or not model:
        return None
    try:
        from agent.image_routing import _lookup_supports_vision as _lookup_image_supports
    except Exception:
        _lookup_image_supports = None
    if _lookup_image_supports is not None:
        try:
            return _lookup_image_supports(provider, model, cfg)
        except Exception as exc:  # pragma: no cover - defensive
            logger.debug(
                "computer_use vision_routing: image-routing caps lookup failed "
                "for %s:%s — %s",
                provider, model, exc,
            )
            return None
    try:
        from agent.models_dev import get_model_capabilities
        caps = get_model_capabilities(provider, model)
    except Exception as exc:  # pragma: no cover - defensive
        logger.debug(
            "computer_use vision_routing: caps lookup failed for %s:%s — %s",
            provider, model, exc,
        )
        return None
    if caps is None:
        return None
    return bool(getattr(caps, "supports_vision", False))


def _provider_accepts_multimodal_tool_result(provider: str, model: str) -> Optional[bool]:
    """Return whether *provider*+*model* carries images inside tool-result messages.

    Reuses ``tools.vision_tools._supports_media_in_tool_results`` so the
    capture-routing decision stays in lockstep with the
    ``vision_analyze`` native fast path. Returns None on import failure
    so callers fall back to aux routing rather than guessing.
    """
    if not provider:
        return None
    try:
        from tools.vision_tools import _supports_media_in_tool_results
    except Exception as exc:  # pragma: no cover - defensive
        logger.debug(
            "computer_use vision_routing: tool-result support lookup failed: %s",
            exc,
        )
        return None
    return bool(_supports_media_in_tool_results(provider, model))


def should_route_capture_to_aux_vision(
    provider: str,
    model: str,
    cfg: Optional[Dict[str, Any]],
) -> bool:
    """Return True iff the captured screenshot should be pre-analysed via aux vision.

    Args:
      provider: active inference provider id (e.g. ``"openrouter"``,
        ``"anthropic"``, ``"openai-codex"``). Lower-case canonical id.
      model:    active main model slug as it would be sent to the provider.
      cfg:      loaded ``config.yaml`` dict (or None).

    Returns:
      ``True`` when the caller should hand the screenshot to the aux vision
      pipeline (and surface a text-only tool result). ``False`` when the
      caller should keep the existing multimodal envelope (main model
      handles vision natively).
    """
    if _explicit_aux_vision_override(cfg):
        return True

    user_declared = _lookup_user_declared_supports_vision(provider, model, cfg)
    if user_declared is True:
        return False
    if user_declared is False:
        return True

    accepts_tool_image = _provider_accepts_multimodal_tool_result(provider, model)
    if accepts_tool_image is None or accepts_tool_image is False:
        return True

    supports_vision = _lookup_supports_vision(provider, model, cfg)
    if supports_vision is True:
        return False
    return True


__all__ = [
    "should_route_capture_to_aux_vision",
]