From 80d1400ce6f972692979faf0aaa2c5fe888b39f5 Mon Sep 17 00:00:00 2001 From: alt-glitch Date: Mon, 15 Jun 2026 16:31:26 +0530 Subject: [PATCH] feat(image_generate): support image input for image-to-image / editing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit image_generate was text-to-image only. Add an optional `image_urls` param (string or list of: http(s) URL, data: URI, or local file path) that routes to the active FAL model's sibling edit endpoint when it has one. - Each edit-capable catalog model declares an `edit` block with the edit endpoint id, the native image key, single-vs-multi cardinality, max images, and the endpoint's OWN accepted-param whitelist. Edit and generate endpoints diverge (e.g. flux-2-pro/edit rejects num_inference_steps/guidance_scale), so edit payloads are built from the edit whitelist, not inherited from generate — every generated payload is a strict subset of the real FAL OpenAPI schema. - Edits never force an output size (FAL infers it from the input) and never chain the upscaler. - Local paths are encoded to base64 data URIs via the vision pipeline's size+dimension-aware encoder before submission (FAL can't read host files). - Models without an edit endpoint return a `note` in the result instead of silently generating from scratch; a configured non-FAL plugin provider is noted when image input forces the in-tree FAL edit path. Wired endpoints (verified against FAL OpenAPI): flux-2/klein/9b/edit, flux-2-pro/edit, nano-banana-pro/edit, gpt-image-1.5/edit, openai/gpt-image-2/edit, qwen-image-edit (single-image, singular image_url key). Text-to-image path is unchanged when image_urls is omitted. --- tests/tools/test_image_generation.py | 19 +- .../test_image_generation_image_input.py | 337 +++++++++++++++ tools/image_generation_tool.py | 403 +++++++++++++++++- 3 files changed, 736 insertions(+), 23 deletions(-) create mode 100644 tests/tools/test_image_generation_image_input.py diff --git a/tests/tools/test_image_generation.py b/tests/tools/test_image_generation.py index b24e6bc1fcc2..45a42647af17 100644 --- a/tests/tools/test_image_generation.py +++ b/tests/tools/test_image_generation.py @@ -363,11 +363,24 @@ class TestAspectRatioNormalization: class TestRegistryIntegration: - def test_schema_exposes_only_prompt_and_aspect_ratio_to_agent(self, image_tool): + def test_schema_stays_tight_no_model_selection(self, image_tool): """The agent-facing schema must stay tight — model selection is a - user-level config choice, not an agent-level arg.""" + user-level config choice, not an agent-level arg. The agent may pass + prompt, aspect_ratio, and image_urls (for image-to-image / edit), but + never model/provider/quality/step knobs.""" props = image_tool.IMAGE_GENERATE_SCHEMA["parameters"]["properties"] - assert set(props.keys()) == {"prompt", "aspect_ratio"} + assert set(props.keys()) == {"prompt", "aspect_ratio", "image_urls"} + # Invariant: model-selection / tuning params never leak into the schema. + forbidden = {"model", "provider", "quality", "num_inference_steps", + "guidance_scale", "num_images", "seed"} + assert not (set(props.keys()) & forbidden) + + def test_image_urls_is_a_string_array(self, image_tool): + spec = image_tool.IMAGE_GENERATE_SCHEMA["parameters"]["properties"]["image_urls"] + assert spec["type"] == "array" + assert spec["items"]["type"] == "string" + # Optional — never required, so text-to-image stays a one-arg call. + assert "image_urls" not in image_tool.IMAGE_GENERATE_SCHEMA["parameters"]["required"] def test_aspect_ratio_enum_is_three_values(self, image_tool): enum = image_tool.IMAGE_GENERATE_SCHEMA["parameters"]["properties"]["aspect_ratio"]["enum"] diff --git a/tests/tools/test_image_generation_image_input.py b/tests/tools/test_image_generation_image_input.py new file mode 100644 index 000000000000..875abaecba6f --- /dev/null +++ b/tests/tools/test_image_generation_image_input.py @@ -0,0 +1,337 @@ +"""Tests for image-to-image / edit input on tools/image_generation_tool.py. + +Covers the ``image_urls`` parameter added to ``image_generate``: + + * input-ref resolution (http(s) URL / data: URI / local path → data URI), + * edit-endpoint routing (models with an ``edit`` entry switch model + payload), + * the fall-through when a model has no edit endpoint, + * the end-to-end submit path (which model id and arguments reach FAL). + +These are behavior/invariant tests — they assert *relationships* (the image +key lands in the payload, the submitted model id is the edit endpoint, a model +without edit is untouched), not frozen catalog snapshots, so they survive +catalog churn. +""" + +from __future__ import annotations + +import base64 +import importlib +import json +import struct +import zlib +from pathlib import Path +from unittest.mock import patch + +import pytest + + +@pytest.fixture +def image_tool(): + import tools.image_generation_tool as mod + return importlib.reload(mod) + + +def _tiny_png_bytes() -> bytes: + """A minimal valid 1x1 PNG (so MIME sniffing + Pillow open succeed).""" + def chunk(typ: bytes, data: bytes) -> bytes: + return ( + struct.pack(">I", len(data)) + typ + data + + struct.pack(">I", zlib.crc32(typ + data) & 0xFFFFFFFF) + ) + sig = b"\x89PNG\r\n\x1a\n" + ihdr = chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0)) + raw = b"\x00\xff\x00\x00" # one filtered RGB pixel + idat = chunk(b"IDAT", zlib.compress(raw)) + iend = chunk(b"IEND", b"") + return sig + ihdr + idat + iend + + +# --------------------------------------------------------------------------- +# Catalog invariants for edit endpoints +# --------------------------------------------------------------------------- + +class TestEditCatalog: + def test_edit_entries_have_required_keys(self, image_tool): + for mid, meta in image_tool.FAL_MODELS.items(): + edit = meta.get("edit") + if edit is None: + continue + assert isinstance(edit, dict), f"{mid}.edit must be a dict" + for key in ("model", "image_key", "max_images"): + assert key in edit, f"{mid}.edit missing {key!r}" + assert isinstance(edit["model"], str) and edit["model"] + assert isinstance(edit["image_key"], str) and edit["image_key"] + + def test_synthesized_edit_meta_lets_image_key_through(self, image_tool): + """Every declared edit endpoint must whitelist its own image key, or + _build_fal_payload would silently strip the reference images.""" + for mid, meta in image_tool.FAL_MODELS.items(): + if not meta.get("edit"): + continue + target = image_tool._resolve_edit_target(mid, ["http://x/y.png"]) + assert target is not None, f"{mid} should resolve an edit target" + edit_model_id, edit_meta = target + assert edit_model_id == meta["edit"]["model"] + assert meta["edit"]["image_key"] in edit_meta["supports"] + # Edits infer output size from the input — no forced size mapping. + assert edit_meta["size_style"] == "none" + # Editing never chains the upscaler. + assert edit_meta["upscale"] is False + + def test_edit_supports_does_not_inherit_generate_only_keys(self, image_tool): + """Regression: edit endpoints must declare their OWN supports, not + inherit the generate whitelist. flux-2-pro/edit rejects + num_inference_steps/guidance_scale that flux-2-pro generate accepts — + if we inherited, those would 422 the edit call.""" + meta = image_tool.FAL_MODELS["fal-ai/flux-2-pro"] + _, edit_meta = image_tool._resolve_edit_target( + "fal-ai/flux-2-pro", ["http://x/y.png"] + ) + assert "num_inference_steps" not in edit_meta["supports"] + assert "guidance_scale" not in edit_meta["supports"] + # but the generate endpoint DOES accept them (proves divergence). + assert "num_inference_steps" in meta["supports"] + + def test_multi_flag_matches_image_key_plurality(self, image_tool): + """Single-image endpoints use the singular ``image_url`` key; multi- + image endpoints use ``image_urls``.""" + for mid, meta in image_tool.FAL_MODELS.items(): + edit = meta.get("edit") + if not edit: + continue + _, edit_meta = image_tool._resolve_edit_target(mid, ["http://x/y.png"]) + if edit_meta["_image_key"] == "image_url": + assert edit_meta["_multi"] is False, f"{mid} singular key must be _multi=False" + assert edit["max_images"] == 1 + else: + assert edit_meta["_image_key"] == "image_urls" + + +# --------------------------------------------------------------------------- +# Edit-target routing decision +# --------------------------------------------------------------------------- + +class TestResolveEditTarget: + def test_no_images_means_no_edit(self, image_tool): + assert image_tool._resolve_edit_target("fal-ai/nano-banana-pro", []) is None + + def test_model_without_edit_endpoint_returns_none(self, image_tool): + # z-image/turbo and ideogram/v3 deliberately have no edit endpoint. + assert image_tool._resolve_edit_target( + "fal-ai/z-image/turbo", ["http://x/y.png"] + ) is None + assert image_tool._resolve_edit_target( + "fal-ai/ideogram/v3", ["http://x/y.png"] + ) is None + + def test_unknown_model_returns_none(self, image_tool): + assert image_tool._resolve_edit_target( + "fal-ai/does-not-exist", ["http://x/y.png"] + ) is None + + +# --------------------------------------------------------------------------- +# Input image resolution +# --------------------------------------------------------------------------- + +class TestResolveInputImages: + def test_http_url_passthrough(self, image_tool): + assert image_tool._resolve_input_images("https://a/b.png") == ["https://a/b.png"] + + def test_data_uri_passthrough(self, image_tool): + uri = "data:image/png;base64,ZZ" + assert image_tool._resolve_input_images([uri]) == [uri] + + def test_none_is_empty(self, image_tool): + assert image_tool._resolve_input_images(None) == [] + + def test_missing_local_file_dropped(self, image_tool): + assert image_tool._resolve_input_images(["/no/such/file.png"]) == [] + + def test_max_images_truncates(self, image_tool): + urls = ["https://a/1.png", "https://a/2.png", "https://a/3.png"] + assert image_tool._resolve_input_images(urls, max_images=2) == urls[:2] + + def test_local_file_encoded_to_data_uri(self, image_tool, tmp_path): + p = tmp_path / "ref.png" + p.write_bytes(_tiny_png_bytes()) + out = image_tool._resolve_input_images([str(p)]) + assert len(out) == 1 + assert out[0].startswith("data:image/") + assert ";base64," in out[0] + # round-trips to non-empty bytes + b64 = out[0].split(";base64,", 1)[1] + assert base64.b64decode(b64) + + +# --------------------------------------------------------------------------- +# End-to-end submit path (mocked FAL) +# --------------------------------------------------------------------------- + +class _FakeHandler: + def __init__(self, result): + self._result = result + + def get(self): + return self._result + + +def _patch_backend(image_tool, monkeypatch, captured): + monkeypatch.setattr(image_tool, "fal_key_is_configured", lambda: True) + monkeypatch.setattr(image_tool, "_resolve_managed_fal_gateway", lambda: None) + + def fake_submit(model, arguments=None, **kw): + captured["model"] = model + captured["arguments"] = arguments + return _FakeHandler({"images": [{"url": "https://out/img.png", "width": 1, "height": 1}]}) + + monkeypatch.setattr(image_tool, "_submit_fal_request", fake_submit) + + +class TestEndToEndSubmit: + def test_text_to_image_untouched_without_images(self, image_tool, monkeypatch): + captured = {} + _patch_backend(image_tool, monkeypatch, captured) + monkeypatch.setattr( + image_tool, "_resolve_fal_model", + lambda: ("fal-ai/nano-banana-pro", image_tool.FAL_MODELS["fal-ai/nano-banana-pro"]), + ) + out = json.loads(image_tool.image_generate_tool(prompt="a cat")) + assert out["success"] is True + # No edit routing: the generate model id is submitted, no image key. + assert captured["model"] == "fal-ai/nano-banana-pro" + assert "image_urls" not in captured["arguments"] + + def test_edit_routing_switches_model_and_adds_images(self, image_tool, monkeypatch): + captured = {} + _patch_backend(image_tool, monkeypatch, captured) + monkeypatch.setattr( + image_tool, "_resolve_fal_model", + lambda: ("fal-ai/nano-banana-pro", image_tool.FAL_MODELS["fal-ai/nano-banana-pro"]), + ) + out = json.loads(image_tool.image_generate_tool( + prompt="make it night", + image_urls=["https://in/ref.png"], + )) + assert out["success"] is True + assert out.get("edited") is True + # Routed to the edit endpoint, with the reference image in the payload. + assert captured["model"] == "fal-ai/nano-banana-pro/edit" + assert captured["arguments"]["image_urls"] == ["https://in/ref.png"] + assert captured["arguments"]["prompt"] == "make it night" + + def test_edit_payload_omits_forced_image_size(self, image_tool, monkeypatch): + """Edits must not inject an explicit image_size/aspect_ratio — FAL + infers output dims from the input image.""" + captured = {} + _patch_backend(image_tool, monkeypatch, captured) + monkeypatch.setattr( + image_tool, "_resolve_fal_model", + lambda: ("fal-ai/flux-2-pro", image_tool.FAL_MODELS["fal-ai/flux-2-pro"]), + ) + image_tool.image_generate_tool(prompt="x", image_urls=["https://in/r.png"]) + # flux-2-pro/edit DOES accept image_size, but we don't force one. + assert "image_size" not in captured["arguments"] + + def test_flux2_pro_edit_drops_generate_only_params(self, image_tool, monkeypatch): + """num_inference_steps/guidance_scale must NOT reach flux-2-pro/edit + even when passed as overrides (they'd 422).""" + captured = {} + _patch_backend(image_tool, monkeypatch, captured) + monkeypatch.setattr( + image_tool, "_resolve_fal_model", + lambda: ("fal-ai/flux-2-pro", image_tool.FAL_MODELS["fal-ai/flux-2-pro"]), + ) + image_tool.image_generate_tool( + prompt="x", + image_urls=["https://in/r.png"], + num_inference_steps=50, + guidance_scale=4.5, + ) + assert "num_inference_steps" not in captured["arguments"] + assert "guidance_scale" not in captured["arguments"] + + def test_qwen_edit_uses_singular_image_url(self, image_tool, monkeypatch): + """Qwen's edit endpoint takes a single ``image_url`` string, not a list.""" + captured = {} + _patch_backend(image_tool, monkeypatch, captured) + monkeypatch.setattr( + image_tool, "_resolve_fal_model", + lambda: ("fal-ai/qwen-image", image_tool.FAL_MODELS["fal-ai/qwen-image"]), + ) + image_tool.image_generate_tool( + prompt="x", + image_urls=["https://in/a.png", "https://in/b.png"], + ) + assert captured["model"] == "fal-ai/qwen-image-edit" + # Singular string, first image only. + assert captured["arguments"]["image_url"] == "https://in/a.png" + assert "image_urls" not in captured["arguments"] + + def test_no_edit_model_surfaces_note_in_result(self, image_tool, monkeypatch): + captured = {} + _patch_backend(image_tool, monkeypatch, captured) + monkeypatch.setattr( + image_tool, "_resolve_fal_model", + lambda: ("fal-ai/z-image/turbo", image_tool.FAL_MODELS["fal-ai/z-image/turbo"]), + ) + out = json.loads(image_tool.image_generate_tool( + prompt="a dog", + image_urls=["https://in/ref.png"], + )) + assert out["success"] is True + assert "note" in out and "does not support image" in out["note"] + + def test_images_on_model_without_edit_fall_back_to_text(self, image_tool, monkeypatch): + captured = {} + _patch_backend(image_tool, monkeypatch, captured) + # z-image/turbo has no edit endpoint. + monkeypatch.setattr( + image_tool, "_resolve_fal_model", + lambda: ("fal-ai/z-image/turbo", image_tool.FAL_MODELS["fal-ai/z-image/turbo"]), + ) + out = json.loads(image_tool.image_generate_tool( + prompt="a dog", + image_urls=["https://in/ref.png"], + )) + assert out["success"] is True + # Stays on the generate model; images dropped (no edit endpoint). + assert captured["model"] == "fal-ai/z-image/turbo" + assert "image_urls" not in captured["arguments"] + + def test_edit_does_not_chain_upscaler(self, image_tool, monkeypatch): + """flux-2-pro has upscale=True for generate, but its edit endpoint must + not chain the upscaler (would double-submit).""" + captured = {} + _patch_backend(image_tool, monkeypatch, captured) + monkeypatch.setattr( + image_tool, "_resolve_fal_model", + lambda: ("fal-ai/flux-2-pro", image_tool.FAL_MODELS["fal-ai/flux-2-pro"]), + ) + # If the upscaler were chained, _submit_fal_request would be called a + # second time with the clarity-upscaler model and overwrite captured. + out = json.loads(image_tool.image_generate_tool( + prompt="add flames", + image_urls=["https://in/ref.png"], + )) + assert out["success"] is True + assert captured["model"] == "fal-ai/flux-2-pro/edit" # not the upscaler + + def test_local_path_reaches_fal_as_data_uri(self, image_tool, monkeypatch, tmp_path): + captured = {} + _patch_backend(image_tool, monkeypatch, captured) + monkeypatch.setattr( + image_tool, "_resolve_fal_model", + lambda: ("fal-ai/flux-2/klein/9b", image_tool.FAL_MODELS["fal-ai/flux-2/klein/9b"]), + ) + p = tmp_path / "ref.png" + p.write_bytes(_tiny_png_bytes()) + out = json.loads(image_tool.image_generate_tool( + prompt="stylize", + image_urls=[str(p)], + )) + assert out["success"] is True + assert captured["model"] == "fal-ai/flux-2/klein/9b/edit" + imgs = captured["arguments"]["image_urls"] + assert len(imgs) == 1 and imgs[0].startswith("data:image/") diff --git a/tools/image_generation_tool.py b/tools/image_generation_tool.py index d7eeb30d1750..1ab25c656424 100644 --- a/tools/image_generation_tool.py +++ b/tools/image_generation_tool.py @@ -26,7 +26,7 @@ import os import datetime import threading import uuid -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional # fal_client is imported lazily — see _load_fal_client(). Pulling it # eagerly added ~64 ms to every CLI cold start because @@ -116,6 +116,24 @@ FAL_MODELS: Dict[str, Dict[str, Any]] = { "output_format", "enable_safety_checker", }, "upscale": False, + # Image-to-image / edit: when the caller supplies input images we + # switch to the model's sibling edit endpoint. Each edit block declares + # the endpoint's OWN accepted-param whitelist (edit and generate + # endpoints diverge — e.g. flux-2-pro/edit rejects num_inference_steps), + # the native key the reference images go under, and whether the endpoint + # takes a list (``multi``) or a single image. Edits never inject an + # explicit image_size — FAL infers output size from the input. + "edit": { + "model": "fal-ai/flux-2/klein/9b/edit", + "image_key": "image_urls", + "multi": True, + "max_images": 4, + "supports": { + "prompt", "image_urls", "image_size", "num_inference_steps", + "num_images", "seed", "output_format", "enable_safety_checker", + "sync_mode", + }, + }, }, "fal-ai/flux-2-pro": { "display": "FLUX 2 Pro", @@ -143,6 +161,19 @@ FAL_MODELS: Dict[str, Dict[str, Any]] = { "safety_tolerance", "sync_mode", "seed", }, "upscale": True, # Backward-compat: current default behavior. + "edit": { + "model": "fal-ai/flux-2-pro/edit", + "image_key": "image_urls", + "multi": True, + "max_images": 9, # FLUX 2 Pro edit accepts up to 9 reference images. + # NOTE: flux-2-pro/edit rejects num_inference_steps / guidance_scale + # / num_images that the generate endpoint accepts. + "supports": { + "prompt", "image_urls", "image_size", "seed", + "safety_tolerance", "enable_safety_checker", "output_format", + "sync_mode", + }, + }, }, "fal-ai/z-image/turbo": { "display": "Z-Image Turbo", @@ -194,6 +225,20 @@ FAL_MODELS: Dict[str, Dict[str, Any]] = { "enable_web_search", "limit_generations", }, "upscale": False, + "edit": { + "model": "fal-ai/nano-banana-pro/edit", + "image_key": "image_urls", + "multi": True, + # Gemini 3 Pro Image edit accepts multiple reference images; cap + # conservatively to keep payloads (and cost) bounded. + "max_images": 6, + "supports": { + "prompt", "image_urls", "aspect_ratio", "resolution", + "num_images", "output_format", "safety_tolerance", "seed", + "sync_mode", "enable_web_search", "limit_generations", + "system_prompt", + }, + }, }, "fal-ai/gpt-image-1.5": { "display": "GPT Image 1.5", @@ -218,6 +263,17 @@ FAL_MODELS: Dict[str, Dict[str, Any]] = { "background", "sync_mode", }, "upscale": False, + "edit": { + "model": "fal-ai/gpt-image-1.5/edit", + "image_key": "image_urls", + "multi": True, + "max_images": 4, + "supports": { + "prompt", "image_urls", "image_size", "quality", "num_images", + "output_format", "background", "input_fidelity", + "mask_image_url", "sync_mode", + }, + }, }, "fal-ai/gpt-image-2": { "display": "GPT Image 2", @@ -250,6 +306,18 @@ FAL_MODELS: Dict[str, Dict[str, Any]] = { # through the shared FAL billing path. }, "upscale": False, + "edit": { + # GPT Image 2's edit endpoint lives under the ``openai/`` namespace + # (not ``fal-ai/``), unlike its text-to-image generate endpoint. + "model": "openai/gpt-image-2/edit", + "image_key": "image_urls", + "multi": True, + "max_images": 4, + "supports": { + "prompt", "image_urls", "image_size", "quality", "num_images", + "output_format", "mask_url", "sync_mode", + }, + }, }, "fal-ai/ideogram/v3": { "display": "Ideogram V3", @@ -317,6 +385,21 @@ FAL_MODELS: Dict[str, Dict[str, Any]] = { "num_images", "output_format", "acceleration", "seed", "sync_mode", }, "upscale": False, + "edit": { + # Qwen's edit endpoint takes a SINGLE image under the singular key + # ``image_url`` (not a list) — unlike the flux/gpt/nano edit + # endpoints which take an ``image_urls`` list. + "model": "fal-ai/qwen-image-edit", + "image_key": "image_url", + "multi": False, + "max_images": 1, + "supports": { + "prompt", "image_url", "image_size", "num_inference_steps", + "guidance_scale", "num_images", "acceleration", "seed", + "output_format", "negative_prompt", "enable_safety_checker", + "sync_mode", + }, + }, }, # Krea 2 — Krea's first foundation image model, day-0 partner launch on # fal (2026-05-27). Same model family as our direct ``plugins/image_gen/krea`` @@ -517,14 +600,20 @@ def _build_fal_payload( aspect_ratio: str = DEFAULT_ASPECT_RATIO, seed: Optional[int] = None, overrides: Optional[Dict[str, Any]] = None, + meta: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """Build a FAL request payload for `model_id` from unified inputs. Translates aspect_ratio into the model's native size spec (preset enum, aspect-ratio enum, or GPT literal string), merges model defaults, applies caller overrides, then filters to the model's ``supports`` whitelist. + + ``meta`` may be passed explicitly for synthesized endpoints that are not in + the static ``FAL_MODELS`` catalog (e.g. the edit endpoint produced by + :func:`_resolve_edit_target`). When omitted it's looked up by ``model_id``. """ - meta = FAL_MODELS[model_id] + if meta is None: + meta = FAL_MODELS[model_id] size_style = meta["size_style"] sizes = meta["sizes"] @@ -539,6 +628,10 @@ def _build_fal_payload( payload["image_size"] = sizes[aspect] elif size_style == "aspect_ratio": payload["aspect_ratio"] = sizes[aspect] + elif size_style == "none": + # Edit endpoints infer output size from the input image — don't inject + # an explicit size/aspect_ratio. + pass else: raise ValueError(f"Unknown size_style: {size_style!r}") @@ -554,6 +647,163 @@ def _build_fal_payload( return {k: v for k, v in payload.items() if k in supports} +def _resolve_edit_target( + model_id: str, input_images: List[str], +) -> Optional[tuple]: + """Decide whether to route this call to an image-edit endpoint. + + Returns ``(edit_model_id, edit_meta_dict)`` when the active model declares + an ``edit`` endpoint AND the caller supplied at least one input image, or + ``None`` to keep the normal text-to-image path. + + ``edit_meta_dict`` is a synthesized FAL_MODELS-style entry built from the + edit block's OWN declared ``supports`` whitelist — NOT inherited from the + generate endpoint, because generate and edit endpoints diverge (e.g. + flux-2-pro/edit rejects ``num_inference_steps``/``guidance_scale`` that the + generate endpoint accepts). Editing never injects an explicit output size + (``size_style="none"``) — FAL infers it from the input image — and never + chains the upscaler. + """ + if not input_images: + return None + meta = FAL_MODELS.get(model_id) + if not meta: + return None + edit = meta.get("edit") + if not isinstance(edit, dict) or not edit.get("model"): + return None + + image_key = edit.get("image_key", "image_urls") + supports = set(edit.get("supports") or set()) + # The image key must always be allowed through, even if a catalog author + # forgot to list it explicitly. + supports.add(image_key) + edit_meta: Dict[str, Any] = { + "display": meta.get("display", model_id) + " (edit)", + # Edits don't map our abstract aspect_ratio onto a forced size — the + # endpoint infers output dimensions from the reference image. + "size_style": "none", + "sizes": {}, + "defaults": dict(edit.get("defaults") or {}), + "supports": supports, + "upscale": False, + "_image_key": image_key, + "_multi": bool(edit.get("multi", True)), + "_max_images": edit.get("max_images"), + } + return edit["model"], edit_meta + + +# --------------------------------------------------------------------------- +# Image input (image-to-image / edit) resolution +# --------------------------------------------------------------------------- +# +# FAL edit/image-to-image endpoints accept each reference image as one of: +# - a public http(s) URL (FAL fetches it server-side), or +# - a base64 ``data:`` URI (FAL decodes it inline). +# +# A gateway-delivered image (Telegram/Discord/WhatsApp photo) lands on disk as +# a local cache file (see gateway adapters' ``cache_image_from_url``), so we +# must encode local paths to a data URI before submission — FAL cannot read the +# host filesystem. We reuse the vision tool's encoder, which already sniffs the +# MIME type and applies a size-aware downscale so we never blow past provider +# payload ceilings. +# +# Per-image hard cap mirrors the vision pipeline (20 MB); above that no major +# backend accepts the image and we drop it with a warning rather than fail the +# whole generation. + +_MAX_INPUT_IMAGE_BYTES = 20 * 1024 * 1024 + + +def _resolve_input_image_ref(ref: str) -> Optional[str]: + """Resolve one user-supplied image reference to a FAL-submittable string. + + Accepts three shapes (the same contract as ``video_generate``'s + ``image_url`` and ``vision_analyze``'s ``image_url``): + + * ``http(s)://…`` — passed through unchanged (FAL fetches it). + * ``data:image/…`` — passed through unchanged (already inline base64). + * absolute / ``~``-relative local path — read from disk and encoded to a + base64 ``data:`` URI (size-aware downscale via the vision encoder). + + Returns ``None`` (and logs) when the ref is empty, points at a missing + file, or can't be encoded — the caller skips it. + """ + if not isinstance(ref, str): + return None + ref = ref.strip() + if not ref: + return None + + lower = ref.lower() + if lower.startswith(("http://", "https://", "data:")): + return ref + + # Treat everything else as a local filesystem path. Expand ``~`` so + # gateway-cached paths and user-typed home-relative paths both resolve. + candidate = os.path.expanduser(ref) + if not os.path.isfile(candidate): + logger.warning( + "image input ref is not an http(s)/data URL and no file exists at " + "%r — skipping", ref, + ) + return None + + try: + from pathlib import Path as _Path + from tools.vision_tools import ( + _EMBED_MAX_DIMENSION, + _resize_image_for_vision, + ) + + # _resize_image_for_vision returns a data: URI, downscaling if the + # encoded payload would exceed the byte cap OR the pixel-dimension cap. + # Passing max_dimension matters: some edit endpoints (e.g. + # nano-banana-pro/edit) reject by pixel count before decode, so a + # small-byte-but-huge-pixel image would 422 without the dimension cap. + return _resize_image_for_vision( + _Path(candidate), + max_base64_bytes=_MAX_INPUT_IMAGE_BYTES, + max_dimension=_EMBED_MAX_DIMENSION, + ) + except Exception as exc: # noqa: BLE001 - one bad image must not kill the call + logger.warning("Could not encode local image input %r: %s", ref, exc) + return None + + +def _resolve_input_images( + image_urls: Optional[Any], max_images: Optional[int] = None, +) -> List[str]: + """Normalize the ``image_urls`` arg into a list of submittable refs. + + Accepts a single string or a list of strings; resolves each via + :func:`_resolve_input_image_ref`; drops any that fail to resolve. When + ``max_images`` is set, the resolved list is truncated to that many (FAL + edit endpoints cap reference counts — e.g. FLUX 2 Pro allows up to 9). + """ + if image_urls is None: + return [] + if isinstance(image_urls, str): + image_urls = [image_urls] + if not isinstance(image_urls, (list, tuple)): + return [] + + resolved: List[str] = [] + for ref in image_urls: + out = _resolve_input_image_ref(ref) + if out: + resolved.append(out) + if max_images is not None and max_images > 0: + if len(resolved) > max_images: + logger.info( + "Got %d input images but the active model caps at %d — using " + "the first %d", len(resolved), max_images, max_images, + ) + resolved = resolved[:max_images] + return resolved + + # --------------------------------------------------------------------------- # Upscaler # --------------------------------------------------------------------------- @@ -729,13 +979,21 @@ def image_generate_tool( num_images: Optional[int] = None, output_format: Optional[str] = None, seed: Optional[int] = None, + image_urls: Optional[Any] = None, ) -> str: """Generate an image from a text prompt using the configured FAL model. - The agent-facing schema exposes only ``prompt`` and ``aspect_ratio``; the - remaining kwargs are overrides for direct Python callers and are filtered - per-model via the ``supports`` whitelist (unsupported overrides are - silently dropped so legacy callers don't break when switching models). + The agent-facing schema exposes ``prompt``, ``aspect_ratio``, and + ``image_urls``; the remaining kwargs are overrides for direct Python + callers and are filtered per-model via the ``supports`` whitelist + (unsupported overrides are silently dropped so legacy callers don't break + when switching models). + + ``image_urls`` (a string or list of strings — http(s) URL, ``data:`` URI, + or local file path) turns this into an image-to-image / edit call when the + active model declares an ``edit`` endpoint. Models without an edit endpoint + ignore the images and fall back to text-to-image (logged), so the call + never hard-fails just because the user attached a reference. Returns a JSON string with ``{"success": bool, "image": url | None, "error": str, "error_type": str}``. @@ -752,6 +1010,10 @@ def image_generate_tool( "num_images": num_images, "output_format": output_format, "seed": seed, + "image_urls_count": ( + len(image_urls) if isinstance(image_urls, (list, tuple)) + else (1 if isinstance(image_urls, str) and image_urls.strip() else 0) + ), }, "error": None, "success": False, @@ -786,16 +1048,59 @@ def image_generate_tool( if output_format is not None: overrides["output_format"] = output_format + # ---- Image-to-image / edit routing ------------------------------- + # Resolve user-supplied reference images (url / data-uri / local path) + # to FAL-submittable strings. If the active model has an edit endpoint + # and we have at least one usable image, switch model + payload to it. + # Otherwise we keep text-to-image (warning if images were supplied but + # the model can't use them). + submit_model_id = model_id + submit_meta = meta + edit_target = None + edit_note = None + if image_urls is not None: + edit = meta.get("edit") if isinstance(meta, dict) else None + max_imgs = edit.get("max_images") if isinstance(edit, dict) else None + resolved_images = _resolve_input_images(image_urls, max_images=max_imgs) + if resolved_images: + edit_target = _resolve_edit_target(model_id, resolved_images) + if edit_target is not None: + submit_model_id, submit_meta = edit_target + image_key = submit_meta["_image_key"] + if submit_meta.get("_multi", True): + overrides[image_key] = resolved_images + else: + # Single-image endpoint (e.g. qwen): the payload key is + # a bare string, not a list. + overrides[image_key] = resolved_images[0] + else: + edit_note = ( + f"The active model '{meta.get('display', model_id)}' " + f"does not support image input/editing; generated from " + f"the prompt alone. To edit an image, switch to an " + f"edit-capable model via `hermes tools` → Image " + f"Generation (e.g. Nano Banana Pro, FLUX 2 Pro, GPT " + f"Image)." + ) + logger.warning( + "image_urls supplied but model '%s' has no edit " + "endpoint — ignoring images and generating from text", + model_id, + ) + arguments = _build_fal_payload( - model_id, prompt, aspect_lc, seed=seed, overrides=overrides, + submit_model_id, prompt, aspect_lc, seed=seed, overrides=overrides, + meta=(submit_meta if edit_target is not None else None), ) logger.info( - "Generating image with %s (%s) — prompt: %s", - meta.get("display", model_id), model_id, prompt[:80], + "%s image with %s (%s) — prompt: %s", + "Editing" if edit_target is not None else "Generating", + submit_meta.get("display", submit_model_id), submit_model_id, + prompt[:80], ) - handler = _submit_fal_request(model_id, arguments=arguments) + handler = _submit_fal_request(submit_model_id, arguments=arguments) result = handler.get() generation_time = (datetime.datetime.now() - start_time).total_seconds() @@ -807,7 +1112,7 @@ def image_generate_tool( if not images: raise ValueError("No images were generated") - should_upscale = bool(meta.get("upscale", False)) + should_upscale = bool(submit_meta.get("upscale", False)) formatted_images = [] for img in images: @@ -842,6 +1147,13 @@ def image_generate_tool( "success": True, "image": formatted_images[0]["url"] if formatted_images else None, } + if edit_note: + # Surface the "images were ignored" reason in the tool RESULT (not + # just a server log) so the agent can react (switch model / tell + # the user) instead of silently returning a from-scratch image. + response_data["note"] = edit_note + elif edit_target is not None: + response_data["edited"] = True debug_call_data["success"] = True debug_call_data["images_generated"] = len(formatted_images) @@ -1002,10 +1314,11 @@ from tools.registry import registry, tool_error IMAGE_GENERATE_SCHEMA = { "name": "image_generate", "description": ( - "Generate high-quality images from text prompts. The underlying " - "backend (FAL, OpenAI, etc.) and model are user-configured and not " - "selectable by the agent. Returns either a URL or an absolute file " - "path in the `image` field; display it with markdown " + "Generate high-quality images from text prompts, or edit/transform " + "existing images when `image_urls` is supplied (image-to-image). The " + "underlying backend (FAL, OpenAI, etc.) and model are user-configured " + "and not selectable by the agent. Returns either a URL or an absolute " + "file path in the `image` field; display it with markdown " "![description](url-or-path) and the gateway will deliver it. When " "the active terminal backend has a different filesystem, successful " "local-file results may also include `agent_visible_image` for " @@ -1016,7 +1329,7 @@ IMAGE_GENERATE_SCHEMA = { "properties": { "prompt": { "type": "string", - "description": "The text prompt describing the desired image. Be detailed and descriptive.", + "description": "The text prompt describing the desired image. Be detailed and descriptive. When editing (image_urls set), describe the change you want.", }, "aspect_ratio": { "type": "string", @@ -1024,6 +1337,21 @@ IMAGE_GENERATE_SCHEMA = { "description": "The aspect ratio of the generated image. 'landscape' is 16:9 wide, 'portrait' is 16:9 tall, 'square' is 1:1.", "default": DEFAULT_ASPECT_RATIO, }, + "image_urls": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Optional list of input images to edit or use as " + "references (image-to-image). Each item is an http(s) URL, " + "a data: URI, or a local file path (e.g. an image a user " + "sent in chat). When supplied, the call routes to the " + "active model's edit endpoint if it has one; models " + "without edit support ignore the images and generate from " + "the prompt alone. Reference multiple images in the prompt " + "by index (e.g. 'put the person from image 1 in the scene " + "from image 2')." + ), + }, }, "required": ["prompt"], }, @@ -1153,18 +1481,53 @@ def _handle_image_generate(args, **kw): if not prompt: return tool_error("prompt is required for image generation") aspect_ratio = args.get("aspect_ratio", DEFAULT_ASPECT_RATIO) + image_urls = args.get("image_urls") task_id = kw.get("task_id") # Route to a plugin-registered provider if one is active (and it's - # not the in-tree FAL path). - dispatched = _dispatch_to_plugin_provider(prompt, aspect_ratio) - if dispatched is not None: - return _postprocess_image_generate_result(dispatched, task_id=task_id) + # not the in-tree FAL path). Plugin providers don't yet accept input + # images through this tool, so when the caller supplied image_urls we + # skip plugin dispatch and use the in-tree FAL edit path rather than + # silently dropping the images. + has_input_images = bool( + image_urls if isinstance(image_urls, (list, tuple)) else + (isinstance(image_urls, str) and image_urls.strip()) + ) + if not has_input_images: + dispatched = _dispatch_to_plugin_provider(prompt, aspect_ratio) + if dispatched is not None: + return _postprocess_image_generate_result(dispatched, task_id=task_id) + + # If a non-FAL plugin provider is configured but the caller supplied input + # images, we fall back to the in-tree FAL edit path (plugin providers don't + # accept image input through this tool yet). Note the backend switch in the + # result so it isn't a silent surprise. + reroute_note = None + if has_input_images: + configured = _read_configured_image_provider() + if configured and configured.lower() != "fal": + reroute_note = ( + f"Image input was supplied, so the call used the built-in FAL " + f"edit path instead of the configured image_gen.provider=" + f"'{configured}' (that provider does not accept image input " + f"through this tool)." + ) raw = image_generate_tool( prompt=prompt, aspect_ratio=aspect_ratio, + image_urls=image_urls, ) + if reroute_note: + try: + payload = json.loads(raw) + if isinstance(payload, dict): + # Don't clobber an existing edit_note; append. + existing = payload.get("note") + payload["note"] = (existing + " " + reroute_note) if existing else reroute_note + raw = json.dumps(payload, ensure_ascii=False) + except Exception: + pass return _postprocess_image_generate_result(raw, task_id=task_id)