mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-10-20 05:15:31 +00:00
80d1400ce6
image_generate was text-to-image only. Add an optional `image_urls` param (string or list of: http(s) URL, data: URI, or local file path) that routes to the active FAL model's sibling edit endpoint when it has one. - Each edit-capable catalog model declares an `edit` block with the edit endpoint id, the native image key, single-vs-multi cardinality, max images, and the endpoint's OWN accepted-param whitelist. Edit and generate endpoints diverge (e.g. flux-2-pro/edit rejects num_inference_steps/guidance_scale), so edit payloads are built from the edit whitelist, not inherited from generate — every generated payload is a strict subset of the real FAL OpenAPI schema. - Edits never force an output size (FAL infers it from the input) and never chain the upscaler. - Local paths are encoded to base64 data URIs via the vision pipeline's size+dimension-aware encoder before submission (FAL can't read host files). - Models without an edit endpoint return a `note` in the result instead of silently generating from scratch; a configured non-FAL plugin provider is noted when image input forces the in-tree FAL edit path. Wired endpoints (verified against FAL OpenAPI): flux-2/klein/9b/edit, flux-2-pro/edit, nano-banana-pro/edit, gpt-image-1.5/edit, openai/gpt-image-2/edit, qwen-image-edit (single-image, singular image_url key). Text-to-image path is unchanged when image_urls is omitted.
1543 lines
59 KiB
Python
1543 lines
59 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Image Generation Tools Module
|
||
|
||
Provides image generation via FAL.ai. Multiple FAL models are supported and
|
||
selectable via ``hermes tools`` → Image Generation; the active model is
|
||
persisted to ``image_gen.model`` in ``config.yaml``.
|
||
|
||
Architecture:
|
||
- ``FAL_MODELS`` is a catalog of supported models with per-model metadata
|
||
(size-style family, defaults, ``supports`` whitelist, upscaler flag).
|
||
- ``_build_fal_payload()`` translates the agent's unified inputs (prompt +
|
||
aspect_ratio) into the model-specific payload and filters to the
|
||
``supports`` whitelist so models never receive rejected keys.
|
||
- Upscaling via FAL's Clarity Upscaler is gated per-model via the ``upscale``
|
||
flag — on for FLUX 2 Pro (backward-compat), off for all faster/newer models
|
||
where upscaling would either hurt latency or add marginal quality.
|
||
|
||
Pricing shown in UI strings is as-of the initial commit; we accept drift and
|
||
update when it's noticed.
|
||
"""
|
||
|
||
import json
|
||
import logging
|
||
import os
|
||
import datetime
|
||
import threading
|
||
import uuid
|
||
from typing import Any, Dict, List, Optional
|
||
|
||
# fal_client is imported lazily — see _load_fal_client(). Pulling it
|
||
# eagerly added ~64 ms to every CLI cold start because
|
||
# discover_builtin_tools() imports this module unconditionally during
|
||
# the registry walk, even when image generation is never used.
|
||
#
|
||
# Tests that monkeypatch this attribute (e.g.
|
||
# ``monkeypatch.setattr(image_tool, "fal_client", fake_fal_client)``)
|
||
# still work: _load_fal_client() short-circuits when the attribute is
|
||
# anything truthy, so a test-installed mock is not overwritten by a
|
||
# subsequent real import.
|
||
fal_client: Any = None
|
||
|
||
|
||
def _load_fal_client() -> Any:
|
||
"""Lazily import fal_client and rebind the module global on first use.
|
||
|
||
Idempotent. Returns the (now-loaded) ``fal_client`` module reference.
|
||
Skips the import if the global is already truthy — this preserves the
|
||
test pattern of monkeypatching the module global to install a mock.
|
||
"""
|
||
global fal_client
|
||
if fal_client is not None:
|
||
return fal_client
|
||
from tools.fal_common import import_fal_client
|
||
fal_client = import_fal_client()
|
||
return fal_client
|
||
|
||
|
||
from tools.debug_helpers import DebugSession
|
||
from tools.fal_common import (
|
||
_ManagedFalSyncClient,
|
||
_extract_http_status,
|
||
_normalize_fal_queue_url_format, # noqa: F401 — re-exported for tests
|
||
)
|
||
from tools.managed_tool_gateway import resolve_managed_tool_gateway
|
||
from tools.tool_backend_helpers import (
|
||
fal_key_is_configured,
|
||
managed_nous_tools_enabled,
|
||
nous_tool_gateway_unavailable_message,
|
||
prefers_gateway,
|
||
)
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# FAL model catalog
|
||
# ---------------------------------------------------------------------------
|
||
#
|
||
# Each entry declares how to translate our unified inputs into the model's
|
||
# native payload shape. Size specification falls into three families:
|
||
#
|
||
# "image_size_preset" — preset enum ("square_hd", "landscape_16_9", ...)
|
||
# used by the flux family, z-image, qwen, recraft,
|
||
# ideogram.
|
||
# "aspect_ratio" — aspect ratio enum ("16:9", "1:1", ...) used by
|
||
# nano-banana (Gemini).
|
||
# "gpt_literal" — literal dimension strings ("1024x1024", etc.)
|
||
# used by gpt-image-1.5.
|
||
#
|
||
# ``supports`` is a whitelist of keys allowed in the outgoing payload — any
|
||
# key outside this set is stripped before submission so models never receive
|
||
# rejected parameters (each FAL model rejects unknown keys differently).
|
||
#
|
||
# ``upscale`` controls whether to chain Clarity Upscaler after generation.
|
||
|
||
FAL_MODELS: Dict[str, Dict[str, Any]] = {
|
||
"fal-ai/flux-2/klein/9b": {
|
||
"display": "FLUX 2 Klein 9B",
|
||
"speed": "<1s",
|
||
"strengths": "Fast, crisp text",
|
||
"price": "$0.006/MP",
|
||
"size_style": "image_size_preset",
|
||
"sizes": {
|
||
"landscape": "landscape_16_9",
|
||
"square": "square_hd",
|
||
"portrait": "portrait_16_9",
|
||
},
|
||
"defaults": {
|
||
"num_inference_steps": 4,
|
||
"output_format": "png",
|
||
"enable_safety_checker": False,
|
||
},
|
||
"supports": {
|
||
"prompt", "image_size", "num_inference_steps", "seed",
|
||
"output_format", "enable_safety_checker",
|
||
},
|
||
"upscale": False,
|
||
# Image-to-image / edit: when the caller supplies input images we
|
||
# switch to the model's sibling edit endpoint. Each edit block declares
|
||
# the endpoint's OWN accepted-param whitelist (edit and generate
|
||
# endpoints diverge — e.g. flux-2-pro/edit rejects num_inference_steps),
|
||
# the native key the reference images go under, and whether the endpoint
|
||
# takes a list (``multi``) or a single image. Edits never inject an
|
||
# explicit image_size — FAL infers output size from the input.
|
||
"edit": {
|
||
"model": "fal-ai/flux-2/klein/9b/edit",
|
||
"image_key": "image_urls",
|
||
"multi": True,
|
||
"max_images": 4,
|
||
"supports": {
|
||
"prompt", "image_urls", "image_size", "num_inference_steps",
|
||
"num_images", "seed", "output_format", "enable_safety_checker",
|
||
"sync_mode",
|
||
},
|
||
},
|
||
},
|
||
"fal-ai/flux-2-pro": {
|
||
"display": "FLUX 2 Pro",
|
||
"speed": "~6s",
|
||
"strengths": "Studio photorealism",
|
||
"price": "$0.03/MP",
|
||
"size_style": "image_size_preset",
|
||
"sizes": {
|
||
"landscape": "landscape_16_9",
|
||
"square": "square_hd",
|
||
"portrait": "portrait_16_9",
|
||
},
|
||
"defaults": {
|
||
"num_inference_steps": 50,
|
||
"guidance_scale": 4.5,
|
||
"num_images": 1,
|
||
"output_format": "png",
|
||
"enable_safety_checker": False,
|
||
"safety_tolerance": "5",
|
||
"sync_mode": True,
|
||
},
|
||
"supports": {
|
||
"prompt", "image_size", "num_inference_steps", "guidance_scale",
|
||
"num_images", "output_format", "enable_safety_checker",
|
||
"safety_tolerance", "sync_mode", "seed",
|
||
},
|
||
"upscale": True, # Backward-compat: current default behavior.
|
||
"edit": {
|
||
"model": "fal-ai/flux-2-pro/edit",
|
||
"image_key": "image_urls",
|
||
"multi": True,
|
||
"max_images": 9, # FLUX 2 Pro edit accepts up to 9 reference images.
|
||
# NOTE: flux-2-pro/edit rejects num_inference_steps / guidance_scale
|
||
# / num_images that the generate endpoint accepts.
|
||
"supports": {
|
||
"prompt", "image_urls", "image_size", "seed",
|
||
"safety_tolerance", "enable_safety_checker", "output_format",
|
||
"sync_mode",
|
||
},
|
||
},
|
||
},
|
||
"fal-ai/z-image/turbo": {
|
||
"display": "Z-Image Turbo",
|
||
"speed": "~2s",
|
||
"strengths": "Bilingual EN/CN, 6B",
|
||
"price": "$0.005/MP",
|
||
"size_style": "image_size_preset",
|
||
"sizes": {
|
||
"landscape": "landscape_16_9",
|
||
"square": "square_hd",
|
||
"portrait": "portrait_16_9",
|
||
},
|
||
"defaults": {
|
||
"num_inference_steps": 8,
|
||
"num_images": 1,
|
||
"output_format": "png",
|
||
"enable_safety_checker": False,
|
||
"enable_prompt_expansion": False, # avoid the extra per-request charge
|
||
},
|
||
"supports": {
|
||
"prompt", "image_size", "num_inference_steps", "num_images",
|
||
"seed", "output_format", "enable_safety_checker",
|
||
"enable_prompt_expansion",
|
||
},
|
||
"upscale": False,
|
||
},
|
||
"fal-ai/nano-banana-pro": {
|
||
"display": "Nano Banana Pro (Gemini 3 Pro Image)",
|
||
"speed": "~8s",
|
||
"strengths": "Gemini 3 Pro, reasoning depth, text rendering",
|
||
"price": "$0.15/image (1K)",
|
||
"size_style": "aspect_ratio",
|
||
"sizes": {
|
||
"landscape": "16:9",
|
||
"square": "1:1",
|
||
"portrait": "9:16",
|
||
},
|
||
"defaults": {
|
||
"num_images": 1,
|
||
"output_format": "png",
|
||
"safety_tolerance": "5",
|
||
# "1K" is the cheapest tier; 4K doubles the per-image cost.
|
||
# Users on Nous Subscription should stay at 1K for predictable billing.
|
||
"resolution": "1K",
|
||
},
|
||
"supports": {
|
||
"prompt", "aspect_ratio", "num_images", "output_format",
|
||
"safety_tolerance", "seed", "sync_mode", "resolution",
|
||
"enable_web_search", "limit_generations",
|
||
},
|
||
"upscale": False,
|
||
"edit": {
|
||
"model": "fal-ai/nano-banana-pro/edit",
|
||
"image_key": "image_urls",
|
||
"multi": True,
|
||
# Gemini 3 Pro Image edit accepts multiple reference images; cap
|
||
# conservatively to keep payloads (and cost) bounded.
|
||
"max_images": 6,
|
||
"supports": {
|
||
"prompt", "image_urls", "aspect_ratio", "resolution",
|
||
"num_images", "output_format", "safety_tolerance", "seed",
|
||
"sync_mode", "enable_web_search", "limit_generations",
|
||
"system_prompt",
|
||
},
|
||
},
|
||
},
|
||
"fal-ai/gpt-image-1.5": {
|
||
"display": "GPT Image 1.5",
|
||
"speed": "~15s",
|
||
"strengths": "Prompt adherence",
|
||
"price": "$0.034/image",
|
||
"size_style": "gpt_literal",
|
||
"sizes": {
|
||
"landscape": "1536x1024",
|
||
"square": "1024x1024",
|
||
"portrait": "1024x1536",
|
||
},
|
||
"defaults": {
|
||
# Quality is pinned to medium to keep portal billing predictable
|
||
# across all users (low is too rough, high is 4-6x more expensive).
|
||
"quality": "medium",
|
||
"num_images": 1,
|
||
"output_format": "png",
|
||
},
|
||
"supports": {
|
||
"prompt", "image_size", "quality", "num_images", "output_format",
|
||
"background", "sync_mode",
|
||
},
|
||
"upscale": False,
|
||
"edit": {
|
||
"model": "fal-ai/gpt-image-1.5/edit",
|
||
"image_key": "image_urls",
|
||
"multi": True,
|
||
"max_images": 4,
|
||
"supports": {
|
||
"prompt", "image_urls", "image_size", "quality", "num_images",
|
||
"output_format", "background", "input_fidelity",
|
||
"mask_image_url", "sync_mode",
|
||
},
|
||
},
|
||
},
|
||
"fal-ai/gpt-image-2": {
|
||
"display": "GPT Image 2",
|
||
"speed": "~20s",
|
||
"strengths": "SOTA text rendering + CJK, world-aware photorealism",
|
||
"price": "$0.04–0.06/image",
|
||
# GPT Image 2 uses FAL's standard preset enum (unlike 1.5's literal
|
||
# dimensions). We map to the 4:3 variants — the 16:9 presets
|
||
# (1024x576) fall below GPT-Image-2's 655,360 min-pixel requirement
|
||
# and would be rejected. 4:3 keeps us above the minimum on all
|
||
# three aspect ratios.
|
||
"size_style": "image_size_preset",
|
||
"sizes": {
|
||
"landscape": "landscape_4_3", # 1024x768
|
||
"square": "square_hd", # 1024x1024
|
||
"portrait": "portrait_4_3", # 768x1024
|
||
},
|
||
"defaults": {
|
||
# Same quality pinning as gpt-image-1.5: medium keeps Nous
|
||
# Portal billing predictable. "high" is 3-4x the per-image
|
||
# cost at the same size; "low" is too rough for production use.
|
||
"quality": "medium",
|
||
"num_images": 1,
|
||
"output_format": "png",
|
||
},
|
||
"supports": {
|
||
"prompt", "image_size", "quality", "num_images", "output_format",
|
||
"sync_mode",
|
||
# openai_api_key (BYOK) intentionally omitted — all users go
|
||
# through the shared FAL billing path.
|
||
},
|
||
"upscale": False,
|
||
"edit": {
|
||
# GPT Image 2's edit endpoint lives under the ``openai/`` namespace
|
||
# (not ``fal-ai/``), unlike its text-to-image generate endpoint.
|
||
"model": "openai/gpt-image-2/edit",
|
||
"image_key": "image_urls",
|
||
"multi": True,
|
||
"max_images": 4,
|
||
"supports": {
|
||
"prompt", "image_urls", "image_size", "quality", "num_images",
|
||
"output_format", "mask_url", "sync_mode",
|
||
},
|
||
},
|
||
},
|
||
"fal-ai/ideogram/v3": {
|
||
"display": "Ideogram V3",
|
||
"speed": "~5s",
|
||
"strengths": "Best typography",
|
||
"price": "$0.03-0.09/image",
|
||
"size_style": "image_size_preset",
|
||
"sizes": {
|
||
"landscape": "landscape_16_9",
|
||
"square": "square_hd",
|
||
"portrait": "portrait_16_9",
|
||
},
|
||
"defaults": {
|
||
"rendering_speed": "BALANCED",
|
||
"expand_prompt": True,
|
||
"style": "AUTO",
|
||
},
|
||
"supports": {
|
||
"prompt", "image_size", "rendering_speed", "expand_prompt",
|
||
"style", "seed",
|
||
},
|
||
"upscale": False,
|
||
},
|
||
"fal-ai/recraft/v4/pro/text-to-image": {
|
||
"display": "Recraft V4 Pro",
|
||
"speed": "~8s",
|
||
"strengths": "Design, brand systems, production-ready",
|
||
"price": "$0.25/image",
|
||
"size_style": "image_size_preset",
|
||
"sizes": {
|
||
"landscape": "landscape_16_9",
|
||
"square": "square_hd",
|
||
"portrait": "portrait_16_9",
|
||
},
|
||
"defaults": {
|
||
# V4 Pro dropped V3's required `style` enum — defaults handle taste now.
|
||
"enable_safety_checker": False,
|
||
},
|
||
"supports": {
|
||
"prompt", "image_size", "enable_safety_checker",
|
||
"colors", "background_color",
|
||
},
|
||
"upscale": False,
|
||
},
|
||
"fal-ai/qwen-image": {
|
||
"display": "Qwen Image",
|
||
"speed": "~12s",
|
||
"strengths": "LLM-based, complex text",
|
||
"price": "$0.02/MP",
|
||
"size_style": "image_size_preset",
|
||
"sizes": {
|
||
"landscape": "landscape_16_9",
|
||
"square": "square_hd",
|
||
"portrait": "portrait_16_9",
|
||
},
|
||
"defaults": {
|
||
"num_inference_steps": 30,
|
||
"guidance_scale": 2.5,
|
||
"num_images": 1,
|
||
"output_format": "png",
|
||
"acceleration": "regular",
|
||
},
|
||
"supports": {
|
||
"prompt", "image_size", "num_inference_steps", "guidance_scale",
|
||
"num_images", "output_format", "acceleration", "seed", "sync_mode",
|
||
},
|
||
"upscale": False,
|
||
"edit": {
|
||
# Qwen's edit endpoint takes a SINGLE image under the singular key
|
||
# ``image_url`` (not a list) — unlike the flux/gpt/nano edit
|
||
# endpoints which take an ``image_urls`` list.
|
||
"model": "fal-ai/qwen-image-edit",
|
||
"image_key": "image_url",
|
||
"multi": False,
|
||
"max_images": 1,
|
||
"supports": {
|
||
"prompt", "image_url", "image_size", "num_inference_steps",
|
||
"guidance_scale", "num_images", "acceleration", "seed",
|
||
"output_format", "negative_prompt", "enable_safety_checker",
|
||
"sync_mode",
|
||
},
|
||
},
|
||
},
|
||
# Krea 2 — Krea's first foundation image model, day-0 partner launch on
|
||
# fal (2026-05-27). Same model family as our direct ``plugins/image_gen/krea``
|
||
# backend, exposed here for users who prefer to bill through their
|
||
# existing FAL key / Nous Portal subscription rather than register
|
||
# directly with Krea. Both variants share the same parameter schema —
|
||
# only model id, price, and recommended use case differ.
|
||
"fal-ai/krea/v2/medium/text-to-image": {
|
||
"display": "Krea 2 Medium",
|
||
"speed": "~15-25s",
|
||
"strengths": "Illustration, anime, painting, expressive/artistic styles",
|
||
"price": "$0.030 (text) / $0.035 (style refs)",
|
||
"size_style": "aspect_ratio",
|
||
# Krea natively accepts 1:1, 4:3, 3:2, 16:9, 2.35:1, 4:5, 2:3, 9:16 —
|
||
# we map our 3 abstract ratios to the closest match.
|
||
"sizes": {
|
||
"landscape": "16:9",
|
||
"square": "1:1",
|
||
"portrait": "9:16",
|
||
},
|
||
"defaults": {
|
||
"creativity": "medium",
|
||
},
|
||
"supports": {
|
||
"prompt", "aspect_ratio", "creativity", "seed",
|
||
"image_style_references",
|
||
},
|
||
"upscale": False,
|
||
},
|
||
"fal-ai/krea/v2/large/text-to-image": {
|
||
"display": "Krea 2 Large",
|
||
"speed": "~25-60s",
|
||
"strengths": "Photorealism, raw textured looks (motion blur, grain, film)",
|
||
"price": "$0.060 (text) / $0.065 (style refs)",
|
||
"size_style": "aspect_ratio",
|
||
"sizes": {
|
||
"landscape": "16:9",
|
||
"square": "1:1",
|
||
"portrait": "9:16",
|
||
},
|
||
"defaults": {
|
||
"creativity": "medium",
|
||
},
|
||
"supports": {
|
||
"prompt", "aspect_ratio", "creativity", "seed",
|
||
"image_style_references",
|
||
},
|
||
"upscale": False,
|
||
},
|
||
}
|
||
|
||
# Default model is the fastest reasonable option. Kept cheap and sub-1s.
|
||
DEFAULT_MODEL = "fal-ai/flux-2/klein/9b"
|
||
|
||
DEFAULT_ASPECT_RATIO = "landscape"
|
||
VALID_ASPECT_RATIOS = ("landscape", "square", "portrait")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Upscaler (Clarity Upscaler — unchanged from previous implementation)
|
||
# ---------------------------------------------------------------------------
|
||
UPSCALER_MODEL = "fal-ai/clarity-upscaler"
|
||
UPSCALER_FACTOR = 2
|
||
UPSCALER_SAFETY_CHECKER = False
|
||
UPSCALER_DEFAULT_PROMPT = "masterpiece, best quality, highres"
|
||
UPSCALER_NEGATIVE_PROMPT = "(worst quality, low quality, normal quality:2)"
|
||
UPSCALER_CREATIVITY = 0.35
|
||
UPSCALER_RESEMBLANCE = 0.6
|
||
UPSCALER_GUIDANCE_SCALE = 4
|
||
UPSCALER_NUM_INFERENCE_STEPS = 18
|
||
|
||
|
||
_debug = DebugSession("image_tools", env_var="IMAGE_TOOLS_DEBUG")
|
||
_managed_fal_client = None
|
||
_managed_fal_client_config = None
|
||
_managed_fal_client_lock = threading.Lock()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Managed FAL gateway (Nous Subscription)
|
||
# ---------------------------------------------------------------------------
|
||
def _resolve_managed_fal_gateway():
|
||
"""Return managed fal-queue gateway config when the user prefers the gateway
|
||
or direct FAL credentials are absent."""
|
||
if fal_key_is_configured() and not prefers_gateway("image_gen"):
|
||
return None
|
||
return resolve_managed_tool_gateway("fal-queue")
|
||
|
||
|
||
def _get_managed_fal_client(managed_gateway):
|
||
"""Reuse the managed FAL client so its internal httpx.Client is not leaked per call."""
|
||
global _managed_fal_client, _managed_fal_client_config
|
||
|
||
client_config = (
|
||
managed_gateway.gateway_origin.rstrip("/"),
|
||
managed_gateway.nous_user_token,
|
||
)
|
||
with _managed_fal_client_lock:
|
||
if _managed_fal_client is not None and _managed_fal_client_config == client_config:
|
||
return _managed_fal_client
|
||
|
||
# Resolve fal_client on the legacy module — preserves the test
|
||
# pattern of monkey-patching ``image_generation_tool.fal_client``.
|
||
_load_fal_client()
|
||
_managed_fal_client = _ManagedFalSyncClient(
|
||
fal_client,
|
||
key=managed_gateway.nous_user_token,
|
||
queue_run_origin=managed_gateway.gateway_origin,
|
||
)
|
||
_managed_fal_client_config = client_config
|
||
return _managed_fal_client
|
||
|
||
|
||
def _submit_fal_request(model: str, arguments: Dict[str, Any]):
|
||
"""Submit a FAL request using direct credentials or the managed queue gateway."""
|
||
# Trigger the lazy import on first call. Idempotent.
|
||
_load_fal_client()
|
||
request_headers = {"x-idempotency-key": str(uuid.uuid4())}
|
||
managed_gateway = _resolve_managed_fal_gateway()
|
||
if managed_gateway is None:
|
||
return fal_client.submit(model, arguments=arguments, headers=request_headers)
|
||
|
||
managed_client = _get_managed_fal_client(managed_gateway)
|
||
try:
|
||
return managed_client.submit(
|
||
model,
|
||
arguments=arguments,
|
||
headers=request_headers,
|
||
)
|
||
except Exception as exc:
|
||
# 4xx from the managed gateway typically means the portal doesn't
|
||
# currently proxy this model (allowlist miss, billing gate, etc.)
|
||
# — surface a clearer message with actionable remediation instead
|
||
# of a raw HTTP error from httpx.
|
||
status = _extract_http_status(exc)
|
||
if status is not None and 400 <= status < 500:
|
||
gateway_message = ""
|
||
if status in {401, 402, 403}:
|
||
gateway_message = (
|
||
"\n\n"
|
||
+ nous_tool_gateway_unavailable_message(
|
||
"managed FAL image generation",
|
||
force_fresh=True,
|
||
)
|
||
)
|
||
raise ValueError(
|
||
f"Nous Subscription gateway rejected model '{model}' "
|
||
f"(HTTP {status}). This model may not yet be enabled on "
|
||
f"the Nous Portal's FAL proxy. Either:\n"
|
||
f" • Set FAL_KEY in your environment to use FAL.ai directly, or\n"
|
||
f" • Pick a different model via `hermes tools` → Image Generation."
|
||
f"{gateway_message}"
|
||
) from exc
|
||
raise
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Model resolution + payload construction
|
||
# ---------------------------------------------------------------------------
|
||
def _resolve_fal_model() -> tuple:
|
||
"""Resolve the active FAL model from config.yaml (primary) or default.
|
||
|
||
Returns (model_id, metadata_dict). Falls back to DEFAULT_MODEL if the
|
||
configured model is unknown (logged as a warning).
|
||
"""
|
||
model_id = ""
|
||
try:
|
||
from hermes_cli.config import load_config
|
||
cfg = load_config()
|
||
img_cfg = cfg.get("image_gen") if isinstance(cfg, dict) else None
|
||
if isinstance(img_cfg, dict):
|
||
raw = img_cfg.get("model")
|
||
if isinstance(raw, str):
|
||
model_id = raw.strip()
|
||
except Exception as exc:
|
||
logger.debug("Could not load image_gen.model from config: %s", exc)
|
||
|
||
# Env var escape hatch (undocumented; backward-compat for tests/scripts).
|
||
if not model_id:
|
||
model_id = os.getenv("FAL_IMAGE_MODEL", "").strip()
|
||
|
||
if not model_id:
|
||
return DEFAULT_MODEL, FAL_MODELS[DEFAULT_MODEL]
|
||
|
||
if model_id not in FAL_MODELS:
|
||
logger.warning(
|
||
"Unknown FAL model '%s' in config; falling back to %s",
|
||
model_id, DEFAULT_MODEL,
|
||
)
|
||
return DEFAULT_MODEL, FAL_MODELS[DEFAULT_MODEL]
|
||
|
||
return model_id, FAL_MODELS[model_id]
|
||
|
||
|
||
def _build_fal_payload(
|
||
model_id: str,
|
||
prompt: str,
|
||
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
|
||
seed: Optional[int] = None,
|
||
overrides: Optional[Dict[str, Any]] = None,
|
||
meta: Optional[Dict[str, Any]] = None,
|
||
) -> Dict[str, Any]:
|
||
"""Build a FAL request payload for `model_id` from unified inputs.
|
||
|
||
Translates aspect_ratio into the model's native size spec (preset enum,
|
||
aspect-ratio enum, or GPT literal string), merges model defaults, applies
|
||
caller overrides, then filters to the model's ``supports`` whitelist.
|
||
|
||
``meta`` may be passed explicitly for synthesized endpoints that are not in
|
||
the static ``FAL_MODELS`` catalog (e.g. the edit endpoint produced by
|
||
:func:`_resolve_edit_target`). When omitted it's looked up by ``model_id``.
|
||
"""
|
||
if meta is None:
|
||
meta = FAL_MODELS[model_id]
|
||
size_style = meta["size_style"]
|
||
sizes = meta["sizes"]
|
||
|
||
aspect = (aspect_ratio or DEFAULT_ASPECT_RATIO).lower().strip()
|
||
if aspect not in sizes:
|
||
aspect = DEFAULT_ASPECT_RATIO
|
||
|
||
payload: Dict[str, Any] = dict(meta.get("defaults", {}))
|
||
payload["prompt"] = (prompt or "").strip()
|
||
|
||
if size_style in {"image_size_preset", "gpt_literal"}:
|
||
payload["image_size"] = sizes[aspect]
|
||
elif size_style == "aspect_ratio":
|
||
payload["aspect_ratio"] = sizes[aspect]
|
||
elif size_style == "none":
|
||
# Edit endpoints infer output size from the input image — don't inject
|
||
# an explicit size/aspect_ratio.
|
||
pass
|
||
else:
|
||
raise ValueError(f"Unknown size_style: {size_style!r}")
|
||
|
||
if seed is not None and isinstance(seed, int):
|
||
payload["seed"] = seed
|
||
|
||
if overrides:
|
||
for k, v in overrides.items():
|
||
if v is not None:
|
||
payload[k] = v
|
||
|
||
supports = meta["supports"]
|
||
return {k: v for k, v in payload.items() if k in supports}
|
||
|
||
|
||
def _resolve_edit_target(
|
||
model_id: str, input_images: List[str],
|
||
) -> Optional[tuple]:
|
||
"""Decide whether to route this call to an image-edit endpoint.
|
||
|
||
Returns ``(edit_model_id, edit_meta_dict)`` when the active model declares
|
||
an ``edit`` endpoint AND the caller supplied at least one input image, or
|
||
``None`` to keep the normal text-to-image path.
|
||
|
||
``edit_meta_dict`` is a synthesized FAL_MODELS-style entry built from the
|
||
edit block's OWN declared ``supports`` whitelist — NOT inherited from the
|
||
generate endpoint, because generate and edit endpoints diverge (e.g.
|
||
flux-2-pro/edit rejects ``num_inference_steps``/``guidance_scale`` that the
|
||
generate endpoint accepts). Editing never injects an explicit output size
|
||
(``size_style="none"``) — FAL infers it from the input image — and never
|
||
chains the upscaler.
|
||
"""
|
||
if not input_images:
|
||
return None
|
||
meta = FAL_MODELS.get(model_id)
|
||
if not meta:
|
||
return None
|
||
edit = meta.get("edit")
|
||
if not isinstance(edit, dict) or not edit.get("model"):
|
||
return None
|
||
|
||
image_key = edit.get("image_key", "image_urls")
|
||
supports = set(edit.get("supports") or set())
|
||
# The image key must always be allowed through, even if a catalog author
|
||
# forgot to list it explicitly.
|
||
supports.add(image_key)
|
||
edit_meta: Dict[str, Any] = {
|
||
"display": meta.get("display", model_id) + " (edit)",
|
||
# Edits don't map our abstract aspect_ratio onto a forced size — the
|
||
# endpoint infers output dimensions from the reference image.
|
||
"size_style": "none",
|
||
"sizes": {},
|
||
"defaults": dict(edit.get("defaults") or {}),
|
||
"supports": supports,
|
||
"upscale": False,
|
||
"_image_key": image_key,
|
||
"_multi": bool(edit.get("multi", True)),
|
||
"_max_images": edit.get("max_images"),
|
||
}
|
||
return edit["model"], edit_meta
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Image input (image-to-image / edit) resolution
|
||
# ---------------------------------------------------------------------------
|
||
#
|
||
# FAL edit/image-to-image endpoints accept each reference image as one of:
|
||
# - a public http(s) URL (FAL fetches it server-side), or
|
||
# - a base64 ``data:`` URI (FAL decodes it inline).
|
||
#
|
||
# A gateway-delivered image (Telegram/Discord/WhatsApp photo) lands on disk as
|
||
# a local cache file (see gateway adapters' ``cache_image_from_url``), so we
|
||
# must encode local paths to a data URI before submission — FAL cannot read the
|
||
# host filesystem. We reuse the vision tool's encoder, which already sniffs the
|
||
# MIME type and applies a size-aware downscale so we never blow past provider
|
||
# payload ceilings.
|
||
#
|
||
# Per-image hard cap mirrors the vision pipeline (20 MB); above that no major
|
||
# backend accepts the image and we drop it with a warning rather than fail the
|
||
# whole generation.
|
||
|
||
_MAX_INPUT_IMAGE_BYTES = 20 * 1024 * 1024
|
||
|
||
|
||
def _resolve_input_image_ref(ref: str) -> Optional[str]:
|
||
"""Resolve one user-supplied image reference to a FAL-submittable string.
|
||
|
||
Accepts three shapes (the same contract as ``video_generate``'s
|
||
``image_url`` and ``vision_analyze``'s ``image_url``):
|
||
|
||
* ``http(s)://…`` — passed through unchanged (FAL fetches it).
|
||
* ``data:image/…`` — passed through unchanged (already inline base64).
|
||
* absolute / ``~``-relative local path — read from disk and encoded to a
|
||
base64 ``data:`` URI (size-aware downscale via the vision encoder).
|
||
|
||
Returns ``None`` (and logs) when the ref is empty, points at a missing
|
||
file, or can't be encoded — the caller skips it.
|
||
"""
|
||
if not isinstance(ref, str):
|
||
return None
|
||
ref = ref.strip()
|
||
if not ref:
|
||
return None
|
||
|
||
lower = ref.lower()
|
||
if lower.startswith(("http://", "https://", "data:")):
|
||
return ref
|
||
|
||
# Treat everything else as a local filesystem path. Expand ``~`` so
|
||
# gateway-cached paths and user-typed home-relative paths both resolve.
|
||
candidate = os.path.expanduser(ref)
|
||
if not os.path.isfile(candidate):
|
||
logger.warning(
|
||
"image input ref is not an http(s)/data URL and no file exists at "
|
||
"%r — skipping", ref,
|
||
)
|
||
return None
|
||
|
||
try:
|
||
from pathlib import Path as _Path
|
||
from tools.vision_tools import (
|
||
_EMBED_MAX_DIMENSION,
|
||
_resize_image_for_vision,
|
||
)
|
||
|
||
# _resize_image_for_vision returns a data: URI, downscaling if the
|
||
# encoded payload would exceed the byte cap OR the pixel-dimension cap.
|
||
# Passing max_dimension matters: some edit endpoints (e.g.
|
||
# nano-banana-pro/edit) reject by pixel count before decode, so a
|
||
# small-byte-but-huge-pixel image would 422 without the dimension cap.
|
||
return _resize_image_for_vision(
|
||
_Path(candidate),
|
||
max_base64_bytes=_MAX_INPUT_IMAGE_BYTES,
|
||
max_dimension=_EMBED_MAX_DIMENSION,
|
||
)
|
||
except Exception as exc: # noqa: BLE001 - one bad image must not kill the call
|
||
logger.warning("Could not encode local image input %r: %s", ref, exc)
|
||
return None
|
||
|
||
|
||
def _resolve_input_images(
|
||
image_urls: Optional[Any], max_images: Optional[int] = None,
|
||
) -> List[str]:
|
||
"""Normalize the ``image_urls`` arg into a list of submittable refs.
|
||
|
||
Accepts a single string or a list of strings; resolves each via
|
||
:func:`_resolve_input_image_ref`; drops any that fail to resolve. When
|
||
``max_images`` is set, the resolved list is truncated to that many (FAL
|
||
edit endpoints cap reference counts — e.g. FLUX 2 Pro allows up to 9).
|
||
"""
|
||
if image_urls is None:
|
||
return []
|
||
if isinstance(image_urls, str):
|
||
image_urls = [image_urls]
|
||
if not isinstance(image_urls, (list, tuple)):
|
||
return []
|
||
|
||
resolved: List[str] = []
|
||
for ref in image_urls:
|
||
out = _resolve_input_image_ref(ref)
|
||
if out:
|
||
resolved.append(out)
|
||
if max_images is not None and max_images > 0:
|
||
if len(resolved) > max_images:
|
||
logger.info(
|
||
"Got %d input images but the active model caps at %d — using "
|
||
"the first %d", len(resolved), max_images, max_images,
|
||
)
|
||
resolved = resolved[:max_images]
|
||
return resolved
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Upscaler
|
||
# ---------------------------------------------------------------------------
|
||
def _upscale_image(image_url: str, original_prompt: str) -> Optional[Dict[str, Any]]:
|
||
"""Upscale an image using FAL.ai's Clarity Upscaler.
|
||
|
||
Returns upscaled image dict, or None on failure (caller falls back to
|
||
the original image).
|
||
"""
|
||
try:
|
||
logger.info("Upscaling image with Clarity Upscaler...")
|
||
|
||
upscaler_arguments = {
|
||
"image_url": image_url,
|
||
"prompt": f"{UPSCALER_DEFAULT_PROMPT}, {original_prompt}",
|
||
"upscale_factor": UPSCALER_FACTOR,
|
||
"negative_prompt": UPSCALER_NEGATIVE_PROMPT,
|
||
"creativity": UPSCALER_CREATIVITY,
|
||
"resemblance": UPSCALER_RESEMBLANCE,
|
||
"guidance_scale": UPSCALER_GUIDANCE_SCALE,
|
||
"num_inference_steps": UPSCALER_NUM_INFERENCE_STEPS,
|
||
"enable_safety_checker": UPSCALER_SAFETY_CHECKER,
|
||
}
|
||
|
||
handler = _submit_fal_request(UPSCALER_MODEL, arguments=upscaler_arguments)
|
||
result = handler.get()
|
||
|
||
if result and "image" in result:
|
||
upscaled_image = result["image"]
|
||
logger.info(
|
||
"Image upscaled successfully to %sx%s",
|
||
upscaled_image.get("width", "unknown"),
|
||
upscaled_image.get("height", "unknown"),
|
||
)
|
||
return {
|
||
"url": upscaled_image["url"],
|
||
"width": upscaled_image.get("width", 0),
|
||
"height": upscaled_image.get("height", 0),
|
||
"upscaled": True,
|
||
"upscale_factor": UPSCALER_FACTOR,
|
||
}
|
||
logger.error("Upscaler returned invalid response")
|
||
return None
|
||
|
||
except Exception as e:
|
||
logger.error("Error upscaling image: %s", e, exc_info=True)
|
||
return None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Tool entry point
|
||
# ---------------------------------------------------------------------------
|
||
def _looks_like_absolute_file_path(value: str) -> bool:
|
||
if not value or not isinstance(value, str):
|
||
return False
|
||
lower = value.lower()
|
||
if lower.startswith(("http://", "https://", "data:")):
|
||
return False
|
||
if os.path.isabs(value):
|
||
return True
|
||
return len(value) >= 3 and value[1] == ":" and value[2] in {"/", "\\"}
|
||
|
||
|
||
def _active_terminal_env(task_id: str | None):
|
||
try:
|
||
from tools.terminal_tool import get_active_env
|
||
|
||
return get_active_env(task_id or "default")
|
||
except Exception as exc: # noqa: BLE001 - artifact hinting must not break generation
|
||
logger.debug("Could not inspect active terminal environment: %s", exc)
|
||
return None
|
||
|
||
|
||
def _agent_cache_base_for_env(env: Any) -> str | None:
|
||
if env is not None:
|
||
# Forward-looking optional override: an environment may expose its own
|
||
# agent-visible cache root via this callable. No backend defines it yet
|
||
# — it's an extension hook, not a typo. The getattr/callable guards make
|
||
# it a safe no-op until a producer exists.
|
||
explicit = getattr(env, "agent_visible_cache_base", None)
|
||
if callable(explicit):
|
||
try:
|
||
value = explicit()
|
||
if value:
|
||
return str(value).rstrip("/")
|
||
except Exception as exc: # noqa: BLE001
|
||
logger.debug("active env agent_visible_cache_base failed: %s", exc)
|
||
|
||
remote_home = getattr(env, "_remote_home", None)
|
||
if remote_home:
|
||
return f"{str(remote_home).rstrip('/')}/.hermes"
|
||
|
||
env_name = env.__class__.__name__
|
||
if env_name in {"DockerEnvironment", "SingularityEnvironment", "ModalEnvironment"}:
|
||
return "/root/.hermes"
|
||
|
||
# If no environment has been created yet, only backends with deterministic
|
||
# Hermes cache roots can be translated without side effects. SSH can still
|
||
# use a shell-visible tilde path; its first environment sync will upload
|
||
# the cache file before the first command runs.
|
||
backend = (os.getenv("TERMINAL_ENV") or "local").strip().lower()
|
||
if backend in {"docker", "singularity", "modal"}:
|
||
return "/root/.hermes"
|
||
if backend == "ssh":
|
||
return "~/.hermes"
|
||
return None
|
||
|
||
|
||
def _agent_visible_cache_path(host_path: str, env: Any) -> str | None:
|
||
if not _looks_like_absolute_file_path(host_path):
|
||
return None
|
||
|
||
cache_base = _agent_cache_base_for_env(env)
|
||
if not cache_base:
|
||
return None
|
||
|
||
try:
|
||
from tools.credential_files import map_cache_path_to_container
|
||
|
||
return map_cache_path_to_container(host_path, container_base=cache_base)
|
||
except Exception as exc: # noqa: BLE001
|
||
logger.debug("Could not translate image cache path for backend: %s", exc)
|
||
return None
|
||
|
||
|
||
def _force_artifact_sync(env: Any) -> None:
|
||
sync_manager = getattr(env, "_sync_manager", None)
|
||
if sync_manager is None:
|
||
return
|
||
try:
|
||
sync_manager.sync(force=True)
|
||
except Exception as exc: # noqa: BLE001 - keep generation success; log for operators
|
||
logger.warning("Could not force-sync generated image artifact: %s", exc)
|
||
|
||
|
||
def _postprocess_image_generate_result(raw: str, task_id: str | None = None) -> str:
|
||
"""Annotate successful local image results with backend-visible paths.
|
||
|
||
``image`` remains the host/gateway-deliverable path. When the active
|
||
terminal backend has a different filesystem, ``agent_visible_image`` gives
|
||
the path the agent can use with terminal/file tools.
|
||
"""
|
||
try:
|
||
payload = json.loads(raw) if isinstance(raw, str) else raw
|
||
except Exception:
|
||
return raw
|
||
|
||
if not isinstance(payload, dict) or not payload.get("success"):
|
||
return raw
|
||
|
||
image = payload.get("image")
|
||
if not isinstance(image, str) or not _looks_like_absolute_file_path(image):
|
||
return raw
|
||
|
||
env = _active_terminal_env(task_id)
|
||
agent_path = _agent_visible_cache_path(image, env)
|
||
if not agent_path or agent_path == image:
|
||
return raw
|
||
|
||
if env is not None:
|
||
_force_artifact_sync(env)
|
||
|
||
payload.setdefault("host_image", image)
|
||
payload.setdefault("agent_visible_image", agent_path)
|
||
return json.dumps(payload, ensure_ascii=False)
|
||
|
||
|
||
def image_generate_tool(
|
||
prompt: str,
|
||
aspect_ratio: str = DEFAULT_ASPECT_RATIO,
|
||
num_inference_steps: Optional[int] = None,
|
||
guidance_scale: Optional[float] = None,
|
||
num_images: Optional[int] = None,
|
||
output_format: Optional[str] = None,
|
||
seed: Optional[int] = None,
|
||
image_urls: Optional[Any] = None,
|
||
) -> str:
|
||
"""Generate an image from a text prompt using the configured FAL model.
|
||
|
||
The agent-facing schema exposes ``prompt``, ``aspect_ratio``, and
|
||
``image_urls``; the remaining kwargs are overrides for direct Python
|
||
callers and are filtered per-model via the ``supports`` whitelist
|
||
(unsupported overrides are silently dropped so legacy callers don't break
|
||
when switching models).
|
||
|
||
``image_urls`` (a string or list of strings — http(s) URL, ``data:`` URI,
|
||
or local file path) turns this into an image-to-image / edit call when the
|
||
active model declares an ``edit`` endpoint. Models without an edit endpoint
|
||
ignore the images and fall back to text-to-image (logged), so the call
|
||
never hard-fails just because the user attached a reference.
|
||
|
||
Returns a JSON string with ``{"success": bool, "image": url | None,
|
||
"error": str, "error_type": str}``.
|
||
"""
|
||
model_id, meta = _resolve_fal_model()
|
||
|
||
debug_call_data = {
|
||
"model": model_id,
|
||
"parameters": {
|
||
"prompt": prompt,
|
||
"aspect_ratio": aspect_ratio,
|
||
"num_inference_steps": num_inference_steps,
|
||
"guidance_scale": guidance_scale,
|
||
"num_images": num_images,
|
||
"output_format": output_format,
|
||
"seed": seed,
|
||
"image_urls_count": (
|
||
len(image_urls) if isinstance(image_urls, (list, tuple))
|
||
else (1 if isinstance(image_urls, str) and image_urls.strip() else 0)
|
||
),
|
||
},
|
||
"error": None,
|
||
"success": False,
|
||
"images_generated": 0,
|
||
"generation_time": 0,
|
||
}
|
||
|
||
start_time = datetime.datetime.now()
|
||
|
||
try:
|
||
if not prompt or not isinstance(prompt, str) or len(prompt.strip()) == 0:
|
||
raise ValueError("Prompt is required and must be a non-empty string")
|
||
|
||
if not (fal_key_is_configured() or _resolve_managed_fal_gateway()):
|
||
raise ValueError(_build_no_backend_setup_message())
|
||
|
||
aspect_lc = (aspect_ratio or DEFAULT_ASPECT_RATIO).lower().strip()
|
||
if aspect_lc not in VALID_ASPECT_RATIOS:
|
||
logger.warning(
|
||
"Invalid aspect_ratio '%s', defaulting to '%s'",
|
||
aspect_ratio, DEFAULT_ASPECT_RATIO,
|
||
)
|
||
aspect_lc = DEFAULT_ASPECT_RATIO
|
||
|
||
overrides: Dict[str, Any] = {}
|
||
if num_inference_steps is not None:
|
||
overrides["num_inference_steps"] = num_inference_steps
|
||
if guidance_scale is not None:
|
||
overrides["guidance_scale"] = guidance_scale
|
||
if num_images is not None:
|
||
overrides["num_images"] = num_images
|
||
if output_format is not None:
|
||
overrides["output_format"] = output_format
|
||
|
||
# ---- Image-to-image / edit routing -------------------------------
|
||
# Resolve user-supplied reference images (url / data-uri / local path)
|
||
# to FAL-submittable strings. If the active model has an edit endpoint
|
||
# and we have at least one usable image, switch model + payload to it.
|
||
# Otherwise we keep text-to-image (warning if images were supplied but
|
||
# the model can't use them).
|
||
submit_model_id = model_id
|
||
submit_meta = meta
|
||
edit_target = None
|
||
edit_note = None
|
||
if image_urls is not None:
|
||
edit = meta.get("edit") if isinstance(meta, dict) else None
|
||
max_imgs = edit.get("max_images") if isinstance(edit, dict) else None
|
||
resolved_images = _resolve_input_images(image_urls, max_images=max_imgs)
|
||
if resolved_images:
|
||
edit_target = _resolve_edit_target(model_id, resolved_images)
|
||
if edit_target is not None:
|
||
submit_model_id, submit_meta = edit_target
|
||
image_key = submit_meta["_image_key"]
|
||
if submit_meta.get("_multi", True):
|
||
overrides[image_key] = resolved_images
|
||
else:
|
||
# Single-image endpoint (e.g. qwen): the payload key is
|
||
# a bare string, not a list.
|
||
overrides[image_key] = resolved_images[0]
|
||
else:
|
||
edit_note = (
|
||
f"The active model '{meta.get('display', model_id)}' "
|
||
f"does not support image input/editing; generated from "
|
||
f"the prompt alone. To edit an image, switch to an "
|
||
f"edit-capable model via `hermes tools` → Image "
|
||
f"Generation (e.g. Nano Banana Pro, FLUX 2 Pro, GPT "
|
||
f"Image)."
|
||
)
|
||
logger.warning(
|
||
"image_urls supplied but model '%s' has no edit "
|
||
"endpoint — ignoring images and generating from text",
|
||
model_id,
|
||
)
|
||
|
||
arguments = _build_fal_payload(
|
||
submit_model_id, prompt, aspect_lc, seed=seed, overrides=overrides,
|
||
meta=(submit_meta if edit_target is not None else None),
|
||
)
|
||
|
||
logger.info(
|
||
"%s image with %s (%s) — prompt: %s",
|
||
"Editing" if edit_target is not None else "Generating",
|
||
submit_meta.get("display", submit_model_id), submit_model_id,
|
||
prompt[:80],
|
||
)
|
||
|
||
handler = _submit_fal_request(submit_model_id, arguments=arguments)
|
||
result = handler.get()
|
||
|
||
generation_time = (datetime.datetime.now() - start_time).total_seconds()
|
||
|
||
if not result or "images" not in result:
|
||
raise ValueError("Invalid response from FAL.ai API — no images returned")
|
||
|
||
images = result.get("images", [])
|
||
if not images:
|
||
raise ValueError("No images were generated")
|
||
|
||
should_upscale = bool(submit_meta.get("upscale", False))
|
||
|
||
formatted_images = []
|
||
for img in images:
|
||
if not (isinstance(img, dict) and "url" in img):
|
||
continue
|
||
original_image = {
|
||
"url": img["url"],
|
||
"width": img.get("width", 0),
|
||
"height": img.get("height", 0),
|
||
}
|
||
|
||
if should_upscale:
|
||
upscaled_image = _upscale_image(img["url"], prompt.strip())
|
||
if upscaled_image:
|
||
formatted_images.append(upscaled_image)
|
||
continue
|
||
logger.warning("Using original image as fallback (upscale failed)")
|
||
|
||
original_image["upscaled"] = False
|
||
formatted_images.append(original_image)
|
||
|
||
if not formatted_images:
|
||
raise ValueError("No valid image URLs returned from API")
|
||
|
||
upscaled_count = sum(1 for img in formatted_images if img.get("upscaled"))
|
||
logger.info(
|
||
"Generated %s image(s) in %.1fs (%s upscaled) via %s",
|
||
len(formatted_images), generation_time, upscaled_count, model_id,
|
||
)
|
||
|
||
response_data = {
|
||
"success": True,
|
||
"image": formatted_images[0]["url"] if formatted_images else None,
|
||
}
|
||
if edit_note:
|
||
# Surface the "images were ignored" reason in the tool RESULT (not
|
||
# just a server log) so the agent can react (switch model / tell
|
||
# the user) instead of silently returning a from-scratch image.
|
||
response_data["note"] = edit_note
|
||
elif edit_target is not None:
|
||
response_data["edited"] = True
|
||
|
||
debug_call_data["success"] = True
|
||
debug_call_data["images_generated"] = len(formatted_images)
|
||
debug_call_data["generation_time"] = generation_time
|
||
_debug.log_call("image_generate_tool", debug_call_data)
|
||
_debug.save()
|
||
|
||
return json.dumps(response_data, indent=2, ensure_ascii=False)
|
||
|
||
except Exception as e:
|
||
generation_time = (datetime.datetime.now() - start_time).total_seconds()
|
||
error_msg = f"Error generating image: {str(e)}"
|
||
logger.error("%s", error_msg, exc_info=True)
|
||
|
||
response_data = {
|
||
"success": False,
|
||
"image": None,
|
||
"error": str(e),
|
||
"error_type": type(e).__name__,
|
||
}
|
||
|
||
debug_call_data["error"] = error_msg
|
||
debug_call_data["generation_time"] = generation_time
|
||
_debug.log_call("image_generate_tool", debug_call_data)
|
||
_debug.save()
|
||
|
||
return json.dumps(response_data, indent=2, ensure_ascii=False)
|
||
|
||
|
||
def check_fal_api_key() -> bool:
|
||
"""True if the FAL.ai API key (direct or managed gateway) is available."""
|
||
return bool(fal_key_is_configured() or _resolve_managed_fal_gateway())
|
||
|
||
|
||
def _build_no_backend_setup_message() -> str:
|
||
"""Build an actionable error string when no FAL backend is reachable.
|
||
|
||
Used by the in-tree FAL path. Mentions:
|
||
- FAL_KEY signup link
|
||
- managed-gateway status (if Nous tools are enabled)
|
||
- plugin alternative pointer (so users on a stale ``image_gen.provider``
|
||
know the registry exists and how to inspect it)
|
||
"""
|
||
lines = ["Image generation is unavailable in this environment.", ""]
|
||
lines.append("Missing requirements:")
|
||
if managed_nous_tools_enabled():
|
||
lines.append(
|
||
" - FAL_KEY is not set and the managed FAL gateway is unreachable"
|
||
)
|
||
else:
|
||
lines.append(" - FAL_KEY environment variable is not set")
|
||
gateway_message = nous_tool_gateway_unavailable_message(
|
||
"managed FAL image generation",
|
||
)
|
||
if gateway_message:
|
||
lines.append(f" - {gateway_message}")
|
||
lines.append("")
|
||
lines.append("To enable image generation, do one of:")
|
||
lines.append(
|
||
" 1. Get a free API key at https://fal.ai and set "
|
||
"FAL_KEY=<your-key> (then restart the session)"
|
||
)
|
||
if managed_nous_tools_enabled():
|
||
lines.append(
|
||
" 2. Sign in to a Nous account that has the managed FAL "
|
||
"gateway enabled (`hermes setup`)"
|
||
)
|
||
lines.append(
|
||
" 3. Configure a different image_gen provider via `hermes tools` "
|
||
"→ Image Generation (run `hermes plugins list` to see installed "
|
||
"backends)"
|
||
)
|
||
return "\n".join(lines)
|
||
|
||
|
||
def check_image_generation_requirements() -> bool:
|
||
"""True if any image gen backend is available.
|
||
|
||
Providers are considered in this order:
|
||
|
||
1. The in-tree FAL backend (FAL_KEY or managed gateway).
|
||
2. Any plugin-registered provider whose ``is_available()`` returns True.
|
||
|
||
Plugins win only when the in-tree FAL path is NOT ready, which matches
|
||
the historical behavior: shipping hermes with a FAL key configured
|
||
should still expose the tool. The active selection among ready
|
||
providers is resolved per-call by ``image_gen.provider``.
|
||
"""
|
||
try:
|
||
if check_fal_api_key():
|
||
# Trigger the lazy fal_client import here as the SDK presence
|
||
# check. Raises ImportError if the optional ``fal-client``
|
||
# package isn't installed; the caller's except ImportError
|
||
# below catches that and continues to plugin probing.
|
||
_load_fal_client()
|
||
return True
|
||
except ImportError:
|
||
pass
|
||
|
||
# Probe plugin providers. Discovery is idempotent and cheap.
|
||
try:
|
||
from agent.image_gen_registry import list_providers
|
||
from hermes_cli.plugins import _ensure_plugins_discovered
|
||
|
||
_ensure_plugins_discovered()
|
||
for provider in list_providers():
|
||
try:
|
||
if provider.is_available():
|
||
return True
|
||
except Exception:
|
||
continue
|
||
except Exception:
|
||
pass
|
||
|
||
return False
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Demo / CLI entry point
|
||
# ---------------------------------------------------------------------------
|
||
if __name__ == "__main__":
|
||
print("🎨 Image Generation Tools — FAL.ai multi-model support")
|
||
print("=" * 60)
|
||
|
||
if not check_fal_api_key():
|
||
print("❌ FAL_KEY environment variable not set")
|
||
print(" Set it via: export FAL_KEY='your-key-here'")
|
||
print(" Get a key: https://fal.ai/")
|
||
raise SystemExit(1)
|
||
print("✅ FAL.ai API key found")
|
||
|
||
try:
|
||
import fal_client # noqa: F401
|
||
print("✅ fal_client library available")
|
||
except ImportError:
|
||
print("❌ fal_client library not found — pip install fal-client")
|
||
raise SystemExit(1)
|
||
|
||
model_id, meta = _resolve_fal_model()
|
||
print(f"🤖 Active model: {meta.get('display', model_id)} ({model_id})")
|
||
print(f" Speed: {meta.get('speed', '?')} · Price: {meta.get('price', '?')}")
|
||
print(f" Upscaler: {'on' if meta.get('upscale') else 'off'}")
|
||
|
||
print("\nAvailable models:")
|
||
for mid, m in FAL_MODELS.items():
|
||
marker = " ← active" if mid == model_id else ""
|
||
print(f" {mid:<32} {m.get('speed', '?'):<6} {m.get('price', '?')}{marker}")
|
||
|
||
if _debug.active:
|
||
print(f"\n🐛 Debug mode enabled — session {_debug.session_id}")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Registry
|
||
# ---------------------------------------------------------------------------
|
||
from tools.registry import registry, tool_error
|
||
|
||
IMAGE_GENERATE_SCHEMA = {
|
||
"name": "image_generate",
|
||
"description": (
|
||
"Generate high-quality images from text prompts, or edit/transform "
|
||
"existing images when `image_urls` is supplied (image-to-image). The "
|
||
"underlying backend (FAL, OpenAI, etc.) and model are user-configured "
|
||
"and not selectable by the agent. Returns either a URL or an absolute "
|
||
"file path in the `image` field; display it with markdown "
|
||
" and the gateway will deliver it. When "
|
||
"the active terminal backend has a different filesystem, successful "
|
||
"local-file results may also include `agent_visible_image` for "
|
||
"follow-up terminal/file operations."
|
||
),
|
||
"parameters": {
|
||
"type": "object",
|
||
"properties": {
|
||
"prompt": {
|
||
"type": "string",
|
||
"description": "The text prompt describing the desired image. Be detailed and descriptive. When editing (image_urls set), describe the change you want.",
|
||
},
|
||
"aspect_ratio": {
|
||
"type": "string",
|
||
"enum": list(VALID_ASPECT_RATIOS),
|
||
"description": "The aspect ratio of the generated image. 'landscape' is 16:9 wide, 'portrait' is 16:9 tall, 'square' is 1:1.",
|
||
"default": DEFAULT_ASPECT_RATIO,
|
||
},
|
||
"image_urls": {
|
||
"type": "array",
|
||
"items": {"type": "string"},
|
||
"description": (
|
||
"Optional list of input images to edit or use as "
|
||
"references (image-to-image). Each item is an http(s) URL, "
|
||
"a data: URI, or a local file path (e.g. an image a user "
|
||
"sent in chat). When supplied, the call routes to the "
|
||
"active model's edit endpoint if it has one; models "
|
||
"without edit support ignore the images and generate from "
|
||
"the prompt alone. Reference multiple images in the prompt "
|
||
"by index (e.g. 'put the person from image 1 in the scene "
|
||
"from image 2')."
|
||
),
|
||
},
|
||
},
|
||
"required": ["prompt"],
|
||
},
|
||
}
|
||
|
||
|
||
def _read_configured_image_model():
|
||
"""Return the value of ``image_gen.model`` from config.yaml, or None."""
|
||
try:
|
||
from hermes_cli.config import load_config
|
||
cfg = load_config()
|
||
section = cfg.get("image_gen") if isinstance(cfg, dict) else None
|
||
if isinstance(section, dict):
|
||
value = section.get("model")
|
||
if isinstance(value, str) and value.strip():
|
||
return value.strip()
|
||
except Exception as exc:
|
||
logger.debug("Could not read image_gen.model: %s", exc)
|
||
return None
|
||
|
||
|
||
def _read_configured_image_provider():
|
||
"""Return the value of ``image_gen.provider`` from config.yaml, or None.
|
||
|
||
We only consult the plugin registry when this is explicitly set — an
|
||
unset value keeps users on the in-tree FAL fallback even when other
|
||
providers happen to be registered (e.g. a user has OPENAI_API_KEY set
|
||
for other features but never asked for OpenAI image gen). ``"fal"``
|
||
explicitly routes through ``plugins/image_gen/fal/`` (which delegates
|
||
back into this module's pipeline via call-time indirection — see
|
||
issue #26241).
|
||
"""
|
||
try:
|
||
from hermes_cli.config import load_config
|
||
cfg = load_config()
|
||
section = cfg.get("image_gen") if isinstance(cfg, dict) else None
|
||
if isinstance(section, dict):
|
||
value = section.get("provider")
|
||
if isinstance(value, str) and value.strip():
|
||
return value.strip()
|
||
except Exception as exc:
|
||
logger.debug("Could not read image_gen.provider: %s", exc)
|
||
return None
|
||
|
||
|
||
def _dispatch_to_plugin_provider(prompt: str, aspect_ratio: str):
|
||
"""Route the call to a plugin-registered provider when one is selected.
|
||
|
||
Returns a JSON string on dispatch, or ``None`` to fall through to the
|
||
in-tree FAL fallback in ``image_generate_tool``.
|
||
|
||
Dispatch fires when ``image_gen.provider`` is explicitly set — including
|
||
``"fal"`` itself, which now resolves to the
|
||
``plugins/image_gen/fal/`` plugin (the plugin re-enters this module's
|
||
pipeline via ``_it`` indirection so behavior is identical to the
|
||
direct call, just routed through the registry).
|
||
"""
|
||
configured = _read_configured_image_provider()
|
||
if not configured:
|
||
return None
|
||
|
||
# Also read configured model so we can pass it to the plugin
|
||
configured_model = _read_configured_image_model()
|
||
|
||
try:
|
||
# Import locally so plugin discovery isn't triggered just by
|
||
# importing this module (tests rely on that).
|
||
from agent.image_gen_registry import get_provider
|
||
from hermes_cli.plugins import _ensure_plugins_discovered
|
||
|
||
_ensure_plugins_discovered()
|
||
provider = get_provider(configured)
|
||
except Exception as exc:
|
||
logger.debug("image_gen plugin dispatch skipped: %s", exc)
|
||
return None
|
||
|
||
if provider is None:
|
||
try:
|
||
# Long-lived sessions may have discovered plugins before a bundled
|
||
# backend was patched in or before config changed. Retry once with
|
||
# a forced refresh before surfacing a missing-provider error.
|
||
_ensure_plugins_discovered(force=True)
|
||
provider = get_provider(configured)
|
||
except Exception as exc:
|
||
logger.debug("image_gen plugin force-refresh skipped: %s", exc)
|
||
|
||
if provider is None:
|
||
return json.dumps({
|
||
"success": False,
|
||
"image": None,
|
||
"error": (
|
||
f"image_gen.provider='{configured}' is set but no plugin "
|
||
f"registered that name. Run `hermes plugins list` to see "
|
||
f"available image gen backends."
|
||
),
|
||
"error_type": "provider_not_registered",
|
||
})
|
||
|
||
try:
|
||
kwargs = {"prompt": prompt, "aspect_ratio": aspect_ratio}
|
||
if configured_model:
|
||
kwargs["model"] = configured_model
|
||
result = provider.generate(**kwargs)
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"Image gen provider '%s' raised: %s",
|
||
getattr(provider, "name", "?"), exc,
|
||
)
|
||
return json.dumps({
|
||
"success": False,
|
||
"image": None,
|
||
"error": f"Provider '{getattr(provider, 'name', '?')}' error: {exc}",
|
||
"error_type": "provider_exception",
|
||
})
|
||
if not isinstance(result, dict):
|
||
return json.dumps({
|
||
"success": False,
|
||
"image": None,
|
||
"error": "Provider returned a non-dict result",
|
||
"error_type": "provider_contract",
|
||
})
|
||
return json.dumps(result)
|
||
|
||
|
||
def _handle_image_generate(args, **kw):
|
||
prompt = args.get("prompt", "")
|
||
if not prompt:
|
||
return tool_error("prompt is required for image generation")
|
||
aspect_ratio = args.get("aspect_ratio", DEFAULT_ASPECT_RATIO)
|
||
image_urls = args.get("image_urls")
|
||
task_id = kw.get("task_id")
|
||
|
||
# Route to a plugin-registered provider if one is active (and it's
|
||
# not the in-tree FAL path). Plugin providers don't yet accept input
|
||
# images through this tool, so when the caller supplied image_urls we
|
||
# skip plugin dispatch and use the in-tree FAL edit path rather than
|
||
# silently dropping the images.
|
||
has_input_images = bool(
|
||
image_urls if isinstance(image_urls, (list, tuple)) else
|
||
(isinstance(image_urls, str) and image_urls.strip())
|
||
)
|
||
if not has_input_images:
|
||
dispatched = _dispatch_to_plugin_provider(prompt, aspect_ratio)
|
||
if dispatched is not None:
|
||
return _postprocess_image_generate_result(dispatched, task_id=task_id)
|
||
|
||
# If a non-FAL plugin provider is configured but the caller supplied input
|
||
# images, we fall back to the in-tree FAL edit path (plugin providers don't
|
||
# accept image input through this tool yet). Note the backend switch in the
|
||
# result so it isn't a silent surprise.
|
||
reroute_note = None
|
||
if has_input_images:
|
||
configured = _read_configured_image_provider()
|
||
if configured and configured.lower() != "fal":
|
||
reroute_note = (
|
||
f"Image input was supplied, so the call used the built-in FAL "
|
||
f"edit path instead of the configured image_gen.provider="
|
||
f"'{configured}' (that provider does not accept image input "
|
||
f"through this tool)."
|
||
)
|
||
|
||
raw = image_generate_tool(
|
||
prompt=prompt,
|
||
aspect_ratio=aspect_ratio,
|
||
image_urls=image_urls,
|
||
)
|
||
if reroute_note:
|
||
try:
|
||
payload = json.loads(raw)
|
||
if isinstance(payload, dict):
|
||
# Don't clobber an existing edit_note; append.
|
||
existing = payload.get("note")
|
||
payload["note"] = (existing + " " + reroute_note) if existing else reroute_note
|
||
raw = json.dumps(payload, ensure_ascii=False)
|
||
except Exception:
|
||
pass
|
||
return _postprocess_image_generate_result(raw, task_id=task_id)
|
||
|
||
|
||
registry.register(
|
||
name="image_generate",
|
||
toolset="image_gen",
|
||
schema=IMAGE_GENERATE_SCHEMA,
|
||
handler=_handle_image_generate,
|
||
check_fn=check_image_generation_requirements,
|
||
requires_env=[],
|
||
is_async=False, # sync fal_client API to avoid "Event loop is closed" in gateway
|
||
emoji="🎨",
|
||
)
|