mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-10-20 05:15:31 +00:00
feat(image_generate): support image input for image-to-image / editing
image_generate was text-to-image only. Add an optional `image_urls` param (string or list of: http(s) URL, data: URI, or local file path) that routes to the active FAL model's sibling edit endpoint when it has one. - Each edit-capable catalog model declares an `edit` block with the edit endpoint id, the native image key, single-vs-multi cardinality, max images, and the endpoint's OWN accepted-param whitelist. Edit and generate endpoints diverge (e.g. flux-2-pro/edit rejects num_inference_steps/guidance_scale), so edit payloads are built from the edit whitelist, not inherited from generate — every generated payload is a strict subset of the real FAL OpenAPI schema. - Edits never force an output size (FAL infers it from the input) and never chain the upscaler. - Local paths are encoded to base64 data URIs via the vision pipeline's size+dimension-aware encoder before submission (FAL can't read host files). - Models without an edit endpoint return a `note` in the result instead of silently generating from scratch; a configured non-FAL plugin provider is noted when image input forces the in-tree FAL edit path. Wired endpoints (verified against FAL OpenAPI): flux-2/klein/9b/edit, flux-2-pro/edit, nano-banana-pro/edit, gpt-image-1.5/edit, openai/gpt-image-2/edit, qwen-image-edit (single-image, singular image_url key). Text-to-image path is unchanged when image_urls is omitted.
This commit is contained in:
parent
6cb88a0874
commit
80d1400ce6
3 changed files with 736 additions and 23 deletions
|
|
@ -363,11 +363,24 @@ class TestAspectRatioNormalization:
|
|||
|
||||
class TestRegistryIntegration:
|
||||
|
||||
def test_schema_exposes_only_prompt_and_aspect_ratio_to_agent(self, image_tool):
|
||||
def test_schema_stays_tight_no_model_selection(self, image_tool):
|
||||
"""The agent-facing schema must stay tight — model selection is a
|
||||
user-level config choice, not an agent-level arg."""
|
||||
user-level config choice, not an agent-level arg. The agent may pass
|
||||
prompt, aspect_ratio, and image_urls (for image-to-image / edit), but
|
||||
never model/provider/quality/step knobs."""
|
||||
props = image_tool.IMAGE_GENERATE_SCHEMA["parameters"]["properties"]
|
||||
assert set(props.keys()) == {"prompt", "aspect_ratio"}
|
||||
assert set(props.keys()) == {"prompt", "aspect_ratio", "image_urls"}
|
||||
# Invariant: model-selection / tuning params never leak into the schema.
|
||||
forbidden = {"model", "provider", "quality", "num_inference_steps",
|
||||
"guidance_scale", "num_images", "seed"}
|
||||
assert not (set(props.keys()) & forbidden)
|
||||
|
||||
def test_image_urls_is_a_string_array(self, image_tool):
|
||||
spec = image_tool.IMAGE_GENERATE_SCHEMA["parameters"]["properties"]["image_urls"]
|
||||
assert spec["type"] == "array"
|
||||
assert spec["items"]["type"] == "string"
|
||||
# Optional — never required, so text-to-image stays a one-arg call.
|
||||
assert "image_urls" not in image_tool.IMAGE_GENERATE_SCHEMA["parameters"]["required"]
|
||||
|
||||
def test_aspect_ratio_enum_is_three_values(self, image_tool):
|
||||
enum = image_tool.IMAGE_GENERATE_SCHEMA["parameters"]["properties"]["aspect_ratio"]["enum"]
|
||||
|
|
|
|||
337
tests/tools/test_image_generation_image_input.py
Normal file
337
tests/tools/test_image_generation_image_input.py
Normal file
|
|
@ -0,0 +1,337 @@
|
|||
"""Tests for image-to-image / edit input on tools/image_generation_tool.py.
|
||||
|
||||
Covers the ``image_urls`` parameter added to ``image_generate``:
|
||||
|
||||
* input-ref resolution (http(s) URL / data: URI / local path → data URI),
|
||||
* edit-endpoint routing (models with an ``edit`` entry switch model + payload),
|
||||
* the fall-through when a model has no edit endpoint,
|
||||
* the end-to-end submit path (which model id and arguments reach FAL).
|
||||
|
||||
These are behavior/invariant tests — they assert *relationships* (the image
|
||||
key lands in the payload, the submitted model id is the edit endpoint, a model
|
||||
without edit is untouched), not frozen catalog snapshots, so they survive
|
||||
catalog churn.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import importlib
|
||||
import json
|
||||
import struct
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def image_tool():
|
||||
import tools.image_generation_tool as mod
|
||||
return importlib.reload(mod)
|
||||
|
||||
|
||||
def _tiny_png_bytes() -> bytes:
|
||||
"""A minimal valid 1x1 PNG (so MIME sniffing + Pillow open succeed)."""
|
||||
def chunk(typ: bytes, data: bytes) -> bytes:
|
||||
return (
|
||||
struct.pack(">I", len(data)) + typ + data
|
||||
+ struct.pack(">I", zlib.crc32(typ + data) & 0xFFFFFFFF)
|
||||
)
|
||||
sig = b"\x89PNG\r\n\x1a\n"
|
||||
ihdr = chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0))
|
||||
raw = b"\x00\xff\x00\x00" # one filtered RGB pixel
|
||||
idat = chunk(b"IDAT", zlib.compress(raw))
|
||||
iend = chunk(b"IEND", b"")
|
||||
return sig + ihdr + idat + iend
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Catalog invariants for edit endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestEditCatalog:
|
||||
def test_edit_entries_have_required_keys(self, image_tool):
|
||||
for mid, meta in image_tool.FAL_MODELS.items():
|
||||
edit = meta.get("edit")
|
||||
if edit is None:
|
||||
continue
|
||||
assert isinstance(edit, dict), f"{mid}.edit must be a dict"
|
||||
for key in ("model", "image_key", "max_images"):
|
||||
assert key in edit, f"{mid}.edit missing {key!r}"
|
||||
assert isinstance(edit["model"], str) and edit["model"]
|
||||
assert isinstance(edit["image_key"], str) and edit["image_key"]
|
||||
|
||||
def test_synthesized_edit_meta_lets_image_key_through(self, image_tool):
|
||||
"""Every declared edit endpoint must whitelist its own image key, or
|
||||
_build_fal_payload would silently strip the reference images."""
|
||||
for mid, meta in image_tool.FAL_MODELS.items():
|
||||
if not meta.get("edit"):
|
||||
continue
|
||||
target = image_tool._resolve_edit_target(mid, ["http://x/y.png"])
|
||||
assert target is not None, f"{mid} should resolve an edit target"
|
||||
edit_model_id, edit_meta = target
|
||||
assert edit_model_id == meta["edit"]["model"]
|
||||
assert meta["edit"]["image_key"] in edit_meta["supports"]
|
||||
# Edits infer output size from the input — no forced size mapping.
|
||||
assert edit_meta["size_style"] == "none"
|
||||
# Editing never chains the upscaler.
|
||||
assert edit_meta["upscale"] is False
|
||||
|
||||
def test_edit_supports_does_not_inherit_generate_only_keys(self, image_tool):
|
||||
"""Regression: edit endpoints must declare their OWN supports, not
|
||||
inherit the generate whitelist. flux-2-pro/edit rejects
|
||||
num_inference_steps/guidance_scale that flux-2-pro generate accepts —
|
||||
if we inherited, those would 422 the edit call."""
|
||||
meta = image_tool.FAL_MODELS["fal-ai/flux-2-pro"]
|
||||
_, edit_meta = image_tool._resolve_edit_target(
|
||||
"fal-ai/flux-2-pro", ["http://x/y.png"]
|
||||
)
|
||||
assert "num_inference_steps" not in edit_meta["supports"]
|
||||
assert "guidance_scale" not in edit_meta["supports"]
|
||||
# but the generate endpoint DOES accept them (proves divergence).
|
||||
assert "num_inference_steps" in meta["supports"]
|
||||
|
||||
def test_multi_flag_matches_image_key_plurality(self, image_tool):
|
||||
"""Single-image endpoints use the singular ``image_url`` key; multi-
|
||||
image endpoints use ``image_urls``."""
|
||||
for mid, meta in image_tool.FAL_MODELS.items():
|
||||
edit = meta.get("edit")
|
||||
if not edit:
|
||||
continue
|
||||
_, edit_meta = image_tool._resolve_edit_target(mid, ["http://x/y.png"])
|
||||
if edit_meta["_image_key"] == "image_url":
|
||||
assert edit_meta["_multi"] is False, f"{mid} singular key must be _multi=False"
|
||||
assert edit["max_images"] == 1
|
||||
else:
|
||||
assert edit_meta["_image_key"] == "image_urls"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Edit-target routing decision
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestResolveEditTarget:
|
||||
def test_no_images_means_no_edit(self, image_tool):
|
||||
assert image_tool._resolve_edit_target("fal-ai/nano-banana-pro", []) is None
|
||||
|
||||
def test_model_without_edit_endpoint_returns_none(self, image_tool):
|
||||
# z-image/turbo and ideogram/v3 deliberately have no edit endpoint.
|
||||
assert image_tool._resolve_edit_target(
|
||||
"fal-ai/z-image/turbo", ["http://x/y.png"]
|
||||
) is None
|
||||
assert image_tool._resolve_edit_target(
|
||||
"fal-ai/ideogram/v3", ["http://x/y.png"]
|
||||
) is None
|
||||
|
||||
def test_unknown_model_returns_none(self, image_tool):
|
||||
assert image_tool._resolve_edit_target(
|
||||
"fal-ai/does-not-exist", ["http://x/y.png"]
|
||||
) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input image resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestResolveInputImages:
|
||||
def test_http_url_passthrough(self, image_tool):
|
||||
assert image_tool._resolve_input_images("https://a/b.png") == ["https://a/b.png"]
|
||||
|
||||
def test_data_uri_passthrough(self, image_tool):
|
||||
uri = "data:image/png;base64,ZZ"
|
||||
assert image_tool._resolve_input_images([uri]) == [uri]
|
||||
|
||||
def test_none_is_empty(self, image_tool):
|
||||
assert image_tool._resolve_input_images(None) == []
|
||||
|
||||
def test_missing_local_file_dropped(self, image_tool):
|
||||
assert image_tool._resolve_input_images(["/no/such/file.png"]) == []
|
||||
|
||||
def test_max_images_truncates(self, image_tool):
|
||||
urls = ["https://a/1.png", "https://a/2.png", "https://a/3.png"]
|
||||
assert image_tool._resolve_input_images(urls, max_images=2) == urls[:2]
|
||||
|
||||
def test_local_file_encoded_to_data_uri(self, image_tool, tmp_path):
|
||||
p = tmp_path / "ref.png"
|
||||
p.write_bytes(_tiny_png_bytes())
|
||||
out = image_tool._resolve_input_images([str(p)])
|
||||
assert len(out) == 1
|
||||
assert out[0].startswith("data:image/")
|
||||
assert ";base64," in out[0]
|
||||
# round-trips to non-empty bytes
|
||||
b64 = out[0].split(";base64,", 1)[1]
|
||||
assert base64.b64decode(b64)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# End-to-end submit path (mocked FAL)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class _FakeHandler:
|
||||
def __init__(self, result):
|
||||
self._result = result
|
||||
|
||||
def get(self):
|
||||
return self._result
|
||||
|
||||
|
||||
def _patch_backend(image_tool, monkeypatch, captured):
|
||||
monkeypatch.setattr(image_tool, "fal_key_is_configured", lambda: True)
|
||||
monkeypatch.setattr(image_tool, "_resolve_managed_fal_gateway", lambda: None)
|
||||
|
||||
def fake_submit(model, arguments=None, **kw):
|
||||
captured["model"] = model
|
||||
captured["arguments"] = arguments
|
||||
return _FakeHandler({"images": [{"url": "https://out/img.png", "width": 1, "height": 1}]})
|
||||
|
||||
monkeypatch.setattr(image_tool, "_submit_fal_request", fake_submit)
|
||||
|
||||
|
||||
class TestEndToEndSubmit:
|
||||
def test_text_to_image_untouched_without_images(self, image_tool, monkeypatch):
|
||||
captured = {}
|
||||
_patch_backend(image_tool, monkeypatch, captured)
|
||||
monkeypatch.setattr(
|
||||
image_tool, "_resolve_fal_model",
|
||||
lambda: ("fal-ai/nano-banana-pro", image_tool.FAL_MODELS["fal-ai/nano-banana-pro"]),
|
||||
)
|
||||
out = json.loads(image_tool.image_generate_tool(prompt="a cat"))
|
||||
assert out["success"] is True
|
||||
# No edit routing: the generate model id is submitted, no image key.
|
||||
assert captured["model"] == "fal-ai/nano-banana-pro"
|
||||
assert "image_urls" not in captured["arguments"]
|
||||
|
||||
def test_edit_routing_switches_model_and_adds_images(self, image_tool, monkeypatch):
|
||||
captured = {}
|
||||
_patch_backend(image_tool, monkeypatch, captured)
|
||||
monkeypatch.setattr(
|
||||
image_tool, "_resolve_fal_model",
|
||||
lambda: ("fal-ai/nano-banana-pro", image_tool.FAL_MODELS["fal-ai/nano-banana-pro"]),
|
||||
)
|
||||
out = json.loads(image_tool.image_generate_tool(
|
||||
prompt="make it night",
|
||||
image_urls=["https://in/ref.png"],
|
||||
))
|
||||
assert out["success"] is True
|
||||
assert out.get("edited") is True
|
||||
# Routed to the edit endpoint, with the reference image in the payload.
|
||||
assert captured["model"] == "fal-ai/nano-banana-pro/edit"
|
||||
assert captured["arguments"]["image_urls"] == ["https://in/ref.png"]
|
||||
assert captured["arguments"]["prompt"] == "make it night"
|
||||
|
||||
def test_edit_payload_omits_forced_image_size(self, image_tool, monkeypatch):
|
||||
"""Edits must not inject an explicit image_size/aspect_ratio — FAL
|
||||
infers output dims from the input image."""
|
||||
captured = {}
|
||||
_patch_backend(image_tool, monkeypatch, captured)
|
||||
monkeypatch.setattr(
|
||||
image_tool, "_resolve_fal_model",
|
||||
lambda: ("fal-ai/flux-2-pro", image_tool.FAL_MODELS["fal-ai/flux-2-pro"]),
|
||||
)
|
||||
image_tool.image_generate_tool(prompt="x", image_urls=["https://in/r.png"])
|
||||
# flux-2-pro/edit DOES accept image_size, but we don't force one.
|
||||
assert "image_size" not in captured["arguments"]
|
||||
|
||||
def test_flux2_pro_edit_drops_generate_only_params(self, image_tool, monkeypatch):
|
||||
"""num_inference_steps/guidance_scale must NOT reach flux-2-pro/edit
|
||||
even when passed as overrides (they'd 422)."""
|
||||
captured = {}
|
||||
_patch_backend(image_tool, monkeypatch, captured)
|
||||
monkeypatch.setattr(
|
||||
image_tool, "_resolve_fal_model",
|
||||
lambda: ("fal-ai/flux-2-pro", image_tool.FAL_MODELS["fal-ai/flux-2-pro"]),
|
||||
)
|
||||
image_tool.image_generate_tool(
|
||||
prompt="x",
|
||||
image_urls=["https://in/r.png"],
|
||||
num_inference_steps=50,
|
||||
guidance_scale=4.5,
|
||||
)
|
||||
assert "num_inference_steps" not in captured["arguments"]
|
||||
assert "guidance_scale" not in captured["arguments"]
|
||||
|
||||
def test_qwen_edit_uses_singular_image_url(self, image_tool, monkeypatch):
|
||||
"""Qwen's edit endpoint takes a single ``image_url`` string, not a list."""
|
||||
captured = {}
|
||||
_patch_backend(image_tool, monkeypatch, captured)
|
||||
monkeypatch.setattr(
|
||||
image_tool, "_resolve_fal_model",
|
||||
lambda: ("fal-ai/qwen-image", image_tool.FAL_MODELS["fal-ai/qwen-image"]),
|
||||
)
|
||||
image_tool.image_generate_tool(
|
||||
prompt="x",
|
||||
image_urls=["https://in/a.png", "https://in/b.png"],
|
||||
)
|
||||
assert captured["model"] == "fal-ai/qwen-image-edit"
|
||||
# Singular string, first image only.
|
||||
assert captured["arguments"]["image_url"] == "https://in/a.png"
|
||||
assert "image_urls" not in captured["arguments"]
|
||||
|
||||
def test_no_edit_model_surfaces_note_in_result(self, image_tool, monkeypatch):
|
||||
captured = {}
|
||||
_patch_backend(image_tool, monkeypatch, captured)
|
||||
monkeypatch.setattr(
|
||||
image_tool, "_resolve_fal_model",
|
||||
lambda: ("fal-ai/z-image/turbo", image_tool.FAL_MODELS["fal-ai/z-image/turbo"]),
|
||||
)
|
||||
out = json.loads(image_tool.image_generate_tool(
|
||||
prompt="a dog",
|
||||
image_urls=["https://in/ref.png"],
|
||||
))
|
||||
assert out["success"] is True
|
||||
assert "note" in out and "does not support image" in out["note"]
|
||||
|
||||
def test_images_on_model_without_edit_fall_back_to_text(self, image_tool, monkeypatch):
|
||||
captured = {}
|
||||
_patch_backend(image_tool, monkeypatch, captured)
|
||||
# z-image/turbo has no edit endpoint.
|
||||
monkeypatch.setattr(
|
||||
image_tool, "_resolve_fal_model",
|
||||
lambda: ("fal-ai/z-image/turbo", image_tool.FAL_MODELS["fal-ai/z-image/turbo"]),
|
||||
)
|
||||
out = json.loads(image_tool.image_generate_tool(
|
||||
prompt="a dog",
|
||||
image_urls=["https://in/ref.png"],
|
||||
))
|
||||
assert out["success"] is True
|
||||
# Stays on the generate model; images dropped (no edit endpoint).
|
||||
assert captured["model"] == "fal-ai/z-image/turbo"
|
||||
assert "image_urls" not in captured["arguments"]
|
||||
|
||||
def test_edit_does_not_chain_upscaler(self, image_tool, monkeypatch):
|
||||
"""flux-2-pro has upscale=True for generate, but its edit endpoint must
|
||||
not chain the upscaler (would double-submit)."""
|
||||
captured = {}
|
||||
_patch_backend(image_tool, monkeypatch, captured)
|
||||
monkeypatch.setattr(
|
||||
image_tool, "_resolve_fal_model",
|
||||
lambda: ("fal-ai/flux-2-pro", image_tool.FAL_MODELS["fal-ai/flux-2-pro"]),
|
||||
)
|
||||
# If the upscaler were chained, _submit_fal_request would be called a
|
||||
# second time with the clarity-upscaler model and overwrite captured.
|
||||
out = json.loads(image_tool.image_generate_tool(
|
||||
prompt="add flames",
|
||||
image_urls=["https://in/ref.png"],
|
||||
))
|
||||
assert out["success"] is True
|
||||
assert captured["model"] == "fal-ai/flux-2-pro/edit" # not the upscaler
|
||||
|
||||
def test_local_path_reaches_fal_as_data_uri(self, image_tool, monkeypatch, tmp_path):
|
||||
captured = {}
|
||||
_patch_backend(image_tool, monkeypatch, captured)
|
||||
monkeypatch.setattr(
|
||||
image_tool, "_resolve_fal_model",
|
||||
lambda: ("fal-ai/flux-2/klein/9b", image_tool.FAL_MODELS["fal-ai/flux-2/klein/9b"]),
|
||||
)
|
||||
p = tmp_path / "ref.png"
|
||||
p.write_bytes(_tiny_png_bytes())
|
||||
out = json.loads(image_tool.image_generate_tool(
|
||||
prompt="stylize",
|
||||
image_urls=[str(p)],
|
||||
))
|
||||
assert out["success"] is True
|
||||
assert captured["model"] == "fal-ai/flux-2/klein/9b/edit"
|
||||
imgs = captured["arguments"]["image_urls"]
|
||||
assert len(imgs) == 1 and imgs[0].startswith("data:image/")
|
||||
|
|
@ -26,7 +26,7 @@ import os
|
|||
import datetime
|
||||
import threading
|
||||
import uuid
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
# fal_client is imported lazily — see _load_fal_client(). Pulling it
|
||||
# eagerly added ~64 ms to every CLI cold start because
|
||||
|
|
@ -116,6 +116,24 @@ FAL_MODELS: Dict[str, Dict[str, Any]] = {
|
|||
"output_format", "enable_safety_checker",
|
||||
},
|
||||
"upscale": False,
|
||||
# Image-to-image / edit: when the caller supplies input images we
|
||||
# switch to the model's sibling edit endpoint. Each edit block declares
|
||||
# the endpoint's OWN accepted-param whitelist (edit and generate
|
||||
# endpoints diverge — e.g. flux-2-pro/edit rejects num_inference_steps),
|
||||
# the native key the reference images go under, and whether the endpoint
|
||||
# takes a list (``multi``) or a single image. Edits never inject an
|
||||
# explicit image_size — FAL infers output size from the input.
|
||||
"edit": {
|
||||
"model": "fal-ai/flux-2/klein/9b/edit",
|
||||
"image_key": "image_urls",
|
||||
"multi": True,
|
||||
"max_images": 4,
|
||||
"supports": {
|
||||
"prompt", "image_urls", "image_size", "num_inference_steps",
|
||||
"num_images", "seed", "output_format", "enable_safety_checker",
|
||||
"sync_mode",
|
||||
},
|
||||
},
|
||||
},
|
||||
"fal-ai/flux-2-pro": {
|
||||
"display": "FLUX 2 Pro",
|
||||
|
|
@ -143,6 +161,19 @@ FAL_MODELS: Dict[str, Dict[str, Any]] = {
|
|||
"safety_tolerance", "sync_mode", "seed",
|
||||
},
|
||||
"upscale": True, # Backward-compat: current default behavior.
|
||||
"edit": {
|
||||
"model": "fal-ai/flux-2-pro/edit",
|
||||
"image_key": "image_urls",
|
||||
"multi": True,
|
||||
"max_images": 9, # FLUX 2 Pro edit accepts up to 9 reference images.
|
||||
# NOTE: flux-2-pro/edit rejects num_inference_steps / guidance_scale
|
||||
# / num_images that the generate endpoint accepts.
|
||||
"supports": {
|
||||
"prompt", "image_urls", "image_size", "seed",
|
||||
"safety_tolerance", "enable_safety_checker", "output_format",
|
||||
"sync_mode",
|
||||
},
|
||||
},
|
||||
},
|
||||
"fal-ai/z-image/turbo": {
|
||||
"display": "Z-Image Turbo",
|
||||
|
|
@ -194,6 +225,20 @@ FAL_MODELS: Dict[str, Dict[str, Any]] = {
|
|||
"enable_web_search", "limit_generations",
|
||||
},
|
||||
"upscale": False,
|
||||
"edit": {
|
||||
"model": "fal-ai/nano-banana-pro/edit",
|
||||
"image_key": "image_urls",
|
||||
"multi": True,
|
||||
# Gemini 3 Pro Image edit accepts multiple reference images; cap
|
||||
# conservatively to keep payloads (and cost) bounded.
|
||||
"max_images": 6,
|
||||
"supports": {
|
||||
"prompt", "image_urls", "aspect_ratio", "resolution",
|
||||
"num_images", "output_format", "safety_tolerance", "seed",
|
||||
"sync_mode", "enable_web_search", "limit_generations",
|
||||
"system_prompt",
|
||||
},
|
||||
},
|
||||
},
|
||||
"fal-ai/gpt-image-1.5": {
|
||||
"display": "GPT Image 1.5",
|
||||
|
|
@ -218,6 +263,17 @@ FAL_MODELS: Dict[str, Dict[str, Any]] = {
|
|||
"background", "sync_mode",
|
||||
},
|
||||
"upscale": False,
|
||||
"edit": {
|
||||
"model": "fal-ai/gpt-image-1.5/edit",
|
||||
"image_key": "image_urls",
|
||||
"multi": True,
|
||||
"max_images": 4,
|
||||
"supports": {
|
||||
"prompt", "image_urls", "image_size", "quality", "num_images",
|
||||
"output_format", "background", "input_fidelity",
|
||||
"mask_image_url", "sync_mode",
|
||||
},
|
||||
},
|
||||
},
|
||||
"fal-ai/gpt-image-2": {
|
||||
"display": "GPT Image 2",
|
||||
|
|
@ -250,6 +306,18 @@ FAL_MODELS: Dict[str, Dict[str, Any]] = {
|
|||
# through the shared FAL billing path.
|
||||
},
|
||||
"upscale": False,
|
||||
"edit": {
|
||||
# GPT Image 2's edit endpoint lives under the ``openai/`` namespace
|
||||
# (not ``fal-ai/``), unlike its text-to-image generate endpoint.
|
||||
"model": "openai/gpt-image-2/edit",
|
||||
"image_key": "image_urls",
|
||||
"multi": True,
|
||||
"max_images": 4,
|
||||
"supports": {
|
||||
"prompt", "image_urls", "image_size", "quality", "num_images",
|
||||
"output_format", "mask_url", "sync_mode",
|
||||
},
|
||||
},
|
||||
},
|
||||
"fal-ai/ideogram/v3": {
|
||||
"display": "Ideogram V3",
|
||||
|
|
@ -317,6 +385,21 @@ FAL_MODELS: Dict[str, Dict[str, Any]] = {
|
|||
"num_images", "output_format", "acceleration", "seed", "sync_mode",
|
||||
},
|
||||
"upscale": False,
|
||||
"edit": {
|
||||
# Qwen's edit endpoint takes a SINGLE image under the singular key
|
||||
# ``image_url`` (not a list) — unlike the flux/gpt/nano edit
|
||||
# endpoints which take an ``image_urls`` list.
|
||||
"model": "fal-ai/qwen-image-edit",
|
||||
"image_key": "image_url",
|
||||
"multi": False,
|
||||
"max_images": 1,
|
||||
"supports": {
|
||||
"prompt", "image_url", "image_size", "num_inference_steps",
|
||||
"guidance_scale", "num_images", "acceleration", "seed",
|
||||
"output_format", "negative_prompt", "enable_safety_checker",
|
||||
"sync_mode",
|
||||
},
|
||||
},
|
||||
},
|
||||
# Krea 2 — Krea's first foundation image model, day-0 partner launch on
|
||||
# fal (2026-05-27). Same model family as our direct ``plugins/image_gen/krea``
|
||||
|
|
@ -517,14 +600,20 @@ def _build_fal_payload(
|
|||
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
|
||||
seed: Optional[int] = None,
|
||||
overrides: Optional[Dict[str, Any]] = None,
|
||||
meta: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build a FAL request payload for `model_id` from unified inputs.
|
||||
|
||||
Translates aspect_ratio into the model's native size spec (preset enum,
|
||||
aspect-ratio enum, or GPT literal string), merges model defaults, applies
|
||||
caller overrides, then filters to the model's ``supports`` whitelist.
|
||||
|
||||
``meta`` may be passed explicitly for synthesized endpoints that are not in
|
||||
the static ``FAL_MODELS`` catalog (e.g. the edit endpoint produced by
|
||||
:func:`_resolve_edit_target`). When omitted it's looked up by ``model_id``.
|
||||
"""
|
||||
meta = FAL_MODELS[model_id]
|
||||
if meta is None:
|
||||
meta = FAL_MODELS[model_id]
|
||||
size_style = meta["size_style"]
|
||||
sizes = meta["sizes"]
|
||||
|
||||
|
|
@ -539,6 +628,10 @@ def _build_fal_payload(
|
|||
payload["image_size"] = sizes[aspect]
|
||||
elif size_style == "aspect_ratio":
|
||||
payload["aspect_ratio"] = sizes[aspect]
|
||||
elif size_style == "none":
|
||||
# Edit endpoints infer output size from the input image — don't inject
|
||||
# an explicit size/aspect_ratio.
|
||||
pass
|
||||
else:
|
||||
raise ValueError(f"Unknown size_style: {size_style!r}")
|
||||
|
||||
|
|
@ -554,6 +647,163 @@ def _build_fal_payload(
|
|||
return {k: v for k, v in payload.items() if k in supports}
|
||||
|
||||
|
||||
def _resolve_edit_target(
|
||||
model_id: str, input_images: List[str],
|
||||
) -> Optional[tuple]:
|
||||
"""Decide whether to route this call to an image-edit endpoint.
|
||||
|
||||
Returns ``(edit_model_id, edit_meta_dict)`` when the active model declares
|
||||
an ``edit`` endpoint AND the caller supplied at least one input image, or
|
||||
``None`` to keep the normal text-to-image path.
|
||||
|
||||
``edit_meta_dict`` is a synthesized FAL_MODELS-style entry built from the
|
||||
edit block's OWN declared ``supports`` whitelist — NOT inherited from the
|
||||
generate endpoint, because generate and edit endpoints diverge (e.g.
|
||||
flux-2-pro/edit rejects ``num_inference_steps``/``guidance_scale`` that the
|
||||
generate endpoint accepts). Editing never injects an explicit output size
|
||||
(``size_style="none"``) — FAL infers it from the input image — and never
|
||||
chains the upscaler.
|
||||
"""
|
||||
if not input_images:
|
||||
return None
|
||||
meta = FAL_MODELS.get(model_id)
|
||||
if not meta:
|
||||
return None
|
||||
edit = meta.get("edit")
|
||||
if not isinstance(edit, dict) or not edit.get("model"):
|
||||
return None
|
||||
|
||||
image_key = edit.get("image_key", "image_urls")
|
||||
supports = set(edit.get("supports") or set())
|
||||
# The image key must always be allowed through, even if a catalog author
|
||||
# forgot to list it explicitly.
|
||||
supports.add(image_key)
|
||||
edit_meta: Dict[str, Any] = {
|
||||
"display": meta.get("display", model_id) + " (edit)",
|
||||
# Edits don't map our abstract aspect_ratio onto a forced size — the
|
||||
# endpoint infers output dimensions from the reference image.
|
||||
"size_style": "none",
|
||||
"sizes": {},
|
||||
"defaults": dict(edit.get("defaults") or {}),
|
||||
"supports": supports,
|
||||
"upscale": False,
|
||||
"_image_key": image_key,
|
||||
"_multi": bool(edit.get("multi", True)),
|
||||
"_max_images": edit.get("max_images"),
|
||||
}
|
||||
return edit["model"], edit_meta
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Image input (image-to-image / edit) resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# FAL edit/image-to-image endpoints accept each reference image as one of:
|
||||
# - a public http(s) URL (FAL fetches it server-side), or
|
||||
# - a base64 ``data:`` URI (FAL decodes it inline).
|
||||
#
|
||||
# A gateway-delivered image (Telegram/Discord/WhatsApp photo) lands on disk as
|
||||
# a local cache file (see gateway adapters' ``cache_image_from_url``), so we
|
||||
# must encode local paths to a data URI before submission — FAL cannot read the
|
||||
# host filesystem. We reuse the vision tool's encoder, which already sniffs the
|
||||
# MIME type and applies a size-aware downscale so we never blow past provider
|
||||
# payload ceilings.
|
||||
#
|
||||
# Per-image hard cap mirrors the vision pipeline (20 MB); above that no major
|
||||
# backend accepts the image and we drop it with a warning rather than fail the
|
||||
# whole generation.
|
||||
|
||||
_MAX_INPUT_IMAGE_BYTES = 20 * 1024 * 1024
|
||||
|
||||
|
||||
def _resolve_input_image_ref(ref: str) -> Optional[str]:
|
||||
"""Resolve one user-supplied image reference to a FAL-submittable string.
|
||||
|
||||
Accepts three shapes (the same contract as ``video_generate``'s
|
||||
``image_url`` and ``vision_analyze``'s ``image_url``):
|
||||
|
||||
* ``http(s)://…`` — passed through unchanged (FAL fetches it).
|
||||
* ``data:image/…`` — passed through unchanged (already inline base64).
|
||||
* absolute / ``~``-relative local path — read from disk and encoded to a
|
||||
base64 ``data:`` URI (size-aware downscale via the vision encoder).
|
||||
|
||||
Returns ``None`` (and logs) when the ref is empty, points at a missing
|
||||
file, or can't be encoded — the caller skips it.
|
||||
"""
|
||||
if not isinstance(ref, str):
|
||||
return None
|
||||
ref = ref.strip()
|
||||
if not ref:
|
||||
return None
|
||||
|
||||
lower = ref.lower()
|
||||
if lower.startswith(("http://", "https://", "data:")):
|
||||
return ref
|
||||
|
||||
# Treat everything else as a local filesystem path. Expand ``~`` so
|
||||
# gateway-cached paths and user-typed home-relative paths both resolve.
|
||||
candidate = os.path.expanduser(ref)
|
||||
if not os.path.isfile(candidate):
|
||||
logger.warning(
|
||||
"image input ref is not an http(s)/data URL and no file exists at "
|
||||
"%r — skipping", ref,
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
from pathlib import Path as _Path
|
||||
from tools.vision_tools import (
|
||||
_EMBED_MAX_DIMENSION,
|
||||
_resize_image_for_vision,
|
||||
)
|
||||
|
||||
# _resize_image_for_vision returns a data: URI, downscaling if the
|
||||
# encoded payload would exceed the byte cap OR the pixel-dimension cap.
|
||||
# Passing max_dimension matters: some edit endpoints (e.g.
|
||||
# nano-banana-pro/edit) reject by pixel count before decode, so a
|
||||
# small-byte-but-huge-pixel image would 422 without the dimension cap.
|
||||
return _resize_image_for_vision(
|
||||
_Path(candidate),
|
||||
max_base64_bytes=_MAX_INPUT_IMAGE_BYTES,
|
||||
max_dimension=_EMBED_MAX_DIMENSION,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - one bad image must not kill the call
|
||||
logger.warning("Could not encode local image input %r: %s", ref, exc)
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_input_images(
|
||||
image_urls: Optional[Any], max_images: Optional[int] = None,
|
||||
) -> List[str]:
|
||||
"""Normalize the ``image_urls`` arg into a list of submittable refs.
|
||||
|
||||
Accepts a single string or a list of strings; resolves each via
|
||||
:func:`_resolve_input_image_ref`; drops any that fail to resolve. When
|
||||
``max_images`` is set, the resolved list is truncated to that many (FAL
|
||||
edit endpoints cap reference counts — e.g. FLUX 2 Pro allows up to 9).
|
||||
"""
|
||||
if image_urls is None:
|
||||
return []
|
||||
if isinstance(image_urls, str):
|
||||
image_urls = [image_urls]
|
||||
if not isinstance(image_urls, (list, tuple)):
|
||||
return []
|
||||
|
||||
resolved: List[str] = []
|
||||
for ref in image_urls:
|
||||
out = _resolve_input_image_ref(ref)
|
||||
if out:
|
||||
resolved.append(out)
|
||||
if max_images is not None and max_images > 0:
|
||||
if len(resolved) > max_images:
|
||||
logger.info(
|
||||
"Got %d input images but the active model caps at %d — using "
|
||||
"the first %d", len(resolved), max_images, max_images,
|
||||
)
|
||||
resolved = resolved[:max_images]
|
||||
return resolved
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Upscaler
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -729,13 +979,21 @@ def image_generate_tool(
|
|||
num_images: Optional[int] = None,
|
||||
output_format: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
image_urls: Optional[Any] = None,
|
||||
) -> str:
|
||||
"""Generate an image from a text prompt using the configured FAL model.
|
||||
|
||||
The agent-facing schema exposes only ``prompt`` and ``aspect_ratio``; the
|
||||
remaining kwargs are overrides for direct Python callers and are filtered
|
||||
per-model via the ``supports`` whitelist (unsupported overrides are
|
||||
silently dropped so legacy callers don't break when switching models).
|
||||
The agent-facing schema exposes ``prompt``, ``aspect_ratio``, and
|
||||
``image_urls``; the remaining kwargs are overrides for direct Python
|
||||
callers and are filtered per-model via the ``supports`` whitelist
|
||||
(unsupported overrides are silently dropped so legacy callers don't break
|
||||
when switching models).
|
||||
|
||||
``image_urls`` (a string or list of strings — http(s) URL, ``data:`` URI,
|
||||
or local file path) turns this into an image-to-image / edit call when the
|
||||
active model declares an ``edit`` endpoint. Models without an edit endpoint
|
||||
ignore the images and fall back to text-to-image (logged), so the call
|
||||
never hard-fails just because the user attached a reference.
|
||||
|
||||
Returns a JSON string with ``{"success": bool, "image": url | None,
|
||||
"error": str, "error_type": str}``.
|
||||
|
|
@ -752,6 +1010,10 @@ def image_generate_tool(
|
|||
"num_images": num_images,
|
||||
"output_format": output_format,
|
||||
"seed": seed,
|
||||
"image_urls_count": (
|
||||
len(image_urls) if isinstance(image_urls, (list, tuple))
|
||||
else (1 if isinstance(image_urls, str) and image_urls.strip() else 0)
|
||||
),
|
||||
},
|
||||
"error": None,
|
||||
"success": False,
|
||||
|
|
@ -786,16 +1048,59 @@ def image_generate_tool(
|
|||
if output_format is not None:
|
||||
overrides["output_format"] = output_format
|
||||
|
||||
# ---- Image-to-image / edit routing -------------------------------
|
||||
# Resolve user-supplied reference images (url / data-uri / local path)
|
||||
# to FAL-submittable strings. If the active model has an edit endpoint
|
||||
# and we have at least one usable image, switch model + payload to it.
|
||||
# Otherwise we keep text-to-image (warning if images were supplied but
|
||||
# the model can't use them).
|
||||
submit_model_id = model_id
|
||||
submit_meta = meta
|
||||
edit_target = None
|
||||
edit_note = None
|
||||
if image_urls is not None:
|
||||
edit = meta.get("edit") if isinstance(meta, dict) else None
|
||||
max_imgs = edit.get("max_images") if isinstance(edit, dict) else None
|
||||
resolved_images = _resolve_input_images(image_urls, max_images=max_imgs)
|
||||
if resolved_images:
|
||||
edit_target = _resolve_edit_target(model_id, resolved_images)
|
||||
if edit_target is not None:
|
||||
submit_model_id, submit_meta = edit_target
|
||||
image_key = submit_meta["_image_key"]
|
||||
if submit_meta.get("_multi", True):
|
||||
overrides[image_key] = resolved_images
|
||||
else:
|
||||
# Single-image endpoint (e.g. qwen): the payload key is
|
||||
# a bare string, not a list.
|
||||
overrides[image_key] = resolved_images[0]
|
||||
else:
|
||||
edit_note = (
|
||||
f"The active model '{meta.get('display', model_id)}' "
|
||||
f"does not support image input/editing; generated from "
|
||||
f"the prompt alone. To edit an image, switch to an "
|
||||
f"edit-capable model via `hermes tools` → Image "
|
||||
f"Generation (e.g. Nano Banana Pro, FLUX 2 Pro, GPT "
|
||||
f"Image)."
|
||||
)
|
||||
logger.warning(
|
||||
"image_urls supplied but model '%s' has no edit "
|
||||
"endpoint — ignoring images and generating from text",
|
||||
model_id,
|
||||
)
|
||||
|
||||
arguments = _build_fal_payload(
|
||||
model_id, prompt, aspect_lc, seed=seed, overrides=overrides,
|
||||
submit_model_id, prompt, aspect_lc, seed=seed, overrides=overrides,
|
||||
meta=(submit_meta if edit_target is not None else None),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Generating image with %s (%s) — prompt: %s",
|
||||
meta.get("display", model_id), model_id, prompt[:80],
|
||||
"%s image with %s (%s) — prompt: %s",
|
||||
"Editing" if edit_target is not None else "Generating",
|
||||
submit_meta.get("display", submit_model_id), submit_model_id,
|
||||
prompt[:80],
|
||||
)
|
||||
|
||||
handler = _submit_fal_request(model_id, arguments=arguments)
|
||||
handler = _submit_fal_request(submit_model_id, arguments=arguments)
|
||||
result = handler.get()
|
||||
|
||||
generation_time = (datetime.datetime.now() - start_time).total_seconds()
|
||||
|
|
@ -807,7 +1112,7 @@ def image_generate_tool(
|
|||
if not images:
|
||||
raise ValueError("No images were generated")
|
||||
|
||||
should_upscale = bool(meta.get("upscale", False))
|
||||
should_upscale = bool(submit_meta.get("upscale", False))
|
||||
|
||||
formatted_images = []
|
||||
for img in images:
|
||||
|
|
@ -842,6 +1147,13 @@ def image_generate_tool(
|
|||
"success": True,
|
||||
"image": formatted_images[0]["url"] if formatted_images else None,
|
||||
}
|
||||
if edit_note:
|
||||
# Surface the "images were ignored" reason in the tool RESULT (not
|
||||
# just a server log) so the agent can react (switch model / tell
|
||||
# the user) instead of silently returning a from-scratch image.
|
||||
response_data["note"] = edit_note
|
||||
elif edit_target is not None:
|
||||
response_data["edited"] = True
|
||||
|
||||
debug_call_data["success"] = True
|
||||
debug_call_data["images_generated"] = len(formatted_images)
|
||||
|
|
@ -1002,10 +1314,11 @@ from tools.registry import registry, tool_error
|
|||
IMAGE_GENERATE_SCHEMA = {
|
||||
"name": "image_generate",
|
||||
"description": (
|
||||
"Generate high-quality images from text prompts. The underlying "
|
||||
"backend (FAL, OpenAI, etc.) and model are user-configured and not "
|
||||
"selectable by the agent. Returns either a URL or an absolute file "
|
||||
"path in the `image` field; display it with markdown "
|
||||
"Generate high-quality images from text prompts, or edit/transform "
|
||||
"existing images when `image_urls` is supplied (image-to-image). The "
|
||||
"underlying backend (FAL, OpenAI, etc.) and model are user-configured "
|
||||
"and not selectable by the agent. Returns either a URL or an absolute "
|
||||
"file path in the `image` field; display it with markdown "
|
||||
" and the gateway will deliver it. When "
|
||||
"the active terminal backend has a different filesystem, successful "
|
||||
"local-file results may also include `agent_visible_image` for "
|
||||
|
|
@ -1016,7 +1329,7 @@ IMAGE_GENERATE_SCHEMA = {
|
|||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string",
|
||||
"description": "The text prompt describing the desired image. Be detailed and descriptive.",
|
||||
"description": "The text prompt describing the desired image. Be detailed and descriptive. When editing (image_urls set), describe the change you want.",
|
||||
},
|
||||
"aspect_ratio": {
|
||||
"type": "string",
|
||||
|
|
@ -1024,6 +1337,21 @@ IMAGE_GENERATE_SCHEMA = {
|
|||
"description": "The aspect ratio of the generated image. 'landscape' is 16:9 wide, 'portrait' is 16:9 tall, 'square' is 1:1.",
|
||||
"default": DEFAULT_ASPECT_RATIO,
|
||||
},
|
||||
"image_urls": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": (
|
||||
"Optional list of input images to edit or use as "
|
||||
"references (image-to-image). Each item is an http(s) URL, "
|
||||
"a data: URI, or a local file path (e.g. an image a user "
|
||||
"sent in chat). When supplied, the call routes to the "
|
||||
"active model's edit endpoint if it has one; models "
|
||||
"without edit support ignore the images and generate from "
|
||||
"the prompt alone. Reference multiple images in the prompt "
|
||||
"by index (e.g. 'put the person from image 1 in the scene "
|
||||
"from image 2')."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["prompt"],
|
||||
},
|
||||
|
|
@ -1153,18 +1481,53 @@ def _handle_image_generate(args, **kw):
|
|||
if not prompt:
|
||||
return tool_error("prompt is required for image generation")
|
||||
aspect_ratio = args.get("aspect_ratio", DEFAULT_ASPECT_RATIO)
|
||||
image_urls = args.get("image_urls")
|
||||
task_id = kw.get("task_id")
|
||||
|
||||
# Route to a plugin-registered provider if one is active (and it's
|
||||
# not the in-tree FAL path).
|
||||
dispatched = _dispatch_to_plugin_provider(prompt, aspect_ratio)
|
||||
if dispatched is not None:
|
||||
return _postprocess_image_generate_result(dispatched, task_id=task_id)
|
||||
# not the in-tree FAL path). Plugin providers don't yet accept input
|
||||
# images through this tool, so when the caller supplied image_urls we
|
||||
# skip plugin dispatch and use the in-tree FAL edit path rather than
|
||||
# silently dropping the images.
|
||||
has_input_images = bool(
|
||||
image_urls if isinstance(image_urls, (list, tuple)) else
|
||||
(isinstance(image_urls, str) and image_urls.strip())
|
||||
)
|
||||
if not has_input_images:
|
||||
dispatched = _dispatch_to_plugin_provider(prompt, aspect_ratio)
|
||||
if dispatched is not None:
|
||||
return _postprocess_image_generate_result(dispatched, task_id=task_id)
|
||||
|
||||
# If a non-FAL plugin provider is configured but the caller supplied input
|
||||
# images, we fall back to the in-tree FAL edit path (plugin providers don't
|
||||
# accept image input through this tool yet). Note the backend switch in the
|
||||
# result so it isn't a silent surprise.
|
||||
reroute_note = None
|
||||
if has_input_images:
|
||||
configured = _read_configured_image_provider()
|
||||
if configured and configured.lower() != "fal":
|
||||
reroute_note = (
|
||||
f"Image input was supplied, so the call used the built-in FAL "
|
||||
f"edit path instead of the configured image_gen.provider="
|
||||
f"'{configured}' (that provider does not accept image input "
|
||||
f"through this tool)."
|
||||
)
|
||||
|
||||
raw = image_generate_tool(
|
||||
prompt=prompt,
|
||||
aspect_ratio=aspect_ratio,
|
||||
image_urls=image_urls,
|
||||
)
|
||||
if reroute_note:
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
if isinstance(payload, dict):
|
||||
# Don't clobber an existing edit_note; append.
|
||||
existing = payload.get("note")
|
||||
payload["note"] = (existing + " " + reroute_note) if existing else reroute_note
|
||||
raw = json.dumps(payload, ensure_ascii=False)
|
||||
except Exception:
|
||||
pass
|
||||
return _postprocess_image_generate_result(raw, task_id=task_id)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue