From d18424a416452c38c407f073f6eca72ad080f5a4 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 21 May 2026 07:06:26 +0000 Subject: [PATCH] feat(M2): Manipulation spatiale des calques personnage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit L'utilisateur peut désormais déplacer, redimensionner, rotater et flipper le personnage généré sans devoir tout régénérer. DB: - Nouveau champ transform JSON sur frameVariants - Format: {x, y, scale, rotation, flipH, flipV} avec coords relatives Backend (sharp): - compositeLayers applique transform avant le blend: * scale: resize layer (peut être >100% ou <100%) * rotation: sharp.rotate avec fond transparent * flipH/flipV: flop/flip * x/y: offset en pourcentage de la base (centré + delta) - Gestion intelligente des layers qui dépassent: extract crop (sharp interdit top/left négatifs et inputs plus grands que la base) - compositing.composeFrame récupère le transform de la variant character active automatiquement - Nouveau endpoint frameVariants.updateTransform Frontend (LayerManipulator): - Composant overlay avec bounding box pointillée + 8 handles - Handles coins = scale, handle haut = rotation, area centrale = move - CSS transform live (translate/scale/rotate/scaleX(-1) pour flip) - Toolbar flottante: flip H/V, position/scale/rotation affichés en live - Reset button quand transformé - Bouton "Recomposer" déclenche composeFrame avec le nouveau transform - Save backend automatique au release de souris ViewportPanel: - Bouton "Manipuler" dans toolbar (visible uniquement mode composite) - Active LayerManipulator overlay, mutuellement exclusif avec Annoter/Loupe - Désactivé si pas de variant character actif (toast warn) Workflow: 1. Mode composite dans viewport 2. Click "Manipuler" → handles apparaissent 3. Drag pour déplacer / corners pour scale / handle haut pour rotation 4. Sauvegarde auto au release (en DB) 5. Click "Recomposer" → sharp regenère avec transform appliqué 6. Nouvelle variante composite créée (Module 1) 7. La galerie M1 montre l'avant/après Co-Authored-By: Claude Opus 4.7 (1M context) --- client/src/components/LayerManipulator.tsx | 291 +++++++++++++++++++++ client/src/components/ViewportPanel.tsx | 57 +++- drizzle/schema.ts | 2 + server/routers.ts | 25 +- server/segmentationService.ts | 97 ++++++- 5 files changed, 458 insertions(+), 14 deletions(-) create mode 100644 client/src/components/LayerManipulator.tsx diff --git a/client/src/components/LayerManipulator.tsx b/client/src/components/LayerManipulator.tsx new file mode 100644 index 0000000..094f6e2 --- /dev/null +++ b/client/src/components/LayerManipulator.tsx @@ -0,0 +1,291 @@ +import { useState, useRef, useCallback, useEffect } from "react"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { trpc } from "@/lib/trpc"; +import { toast } from "sonner"; +import { + Move, FlipHorizontal, FlipVertical, RotateCw, Maximize2, Loader2, + RefreshCw, Sparkles, +} from "lucide-react"; + +export interface LayerTransform { + x?: number; // -1..1 relative to canvas width + y?: number; // -1..1 relative to canvas height + scale?: number; // multiplier, 1=original + rotation?: number; // degrees + flipH?: boolean; + flipV?: boolean; +} + +interface LayerManipulatorProps { + /** The character variant ID to modify (for backend save) */ + variantId: number | undefined; + /** Character image URL */ + imageUrl: string; + /** Initial transform */ + initialTransform?: LayerTransform | null; + /** Canvas dimensions */ + width: number; + height: number; + /** Triggered when user requests recompose */ + onRecompose?: () => void; + /** Whether recompose is in progress */ + isRecomposing?: boolean; +} + +const DEFAULT_TRANSFORM: LayerTransform = { x: 0, y: 0, scale: 1, rotation: 0, flipH: false, flipV: false }; + +export default function LayerManipulator({ + variantId, + imageUrl, + initialTransform, + width, + height, + onRecompose, + isRecomposing, +}: LayerManipulatorProps) { + const [transform, setTransform] = useState(initialTransform || DEFAULT_TRANSFORM); + const [dragMode, setDragMode] = useState<"none" | "move" | "scale" | "rotate">("none"); + const dragStartRef = useRef<{ x: number; y: number; transform: LayerTransform } | null>(null); + const containerRef = useRef(null); + const [isSaving, setIsSaving] = useState(false); + + // Sync with new initial transform when variant changes + useEffect(() => { + setTransform(initialTransform || DEFAULT_TRANSFORM); + }, [variantId, initialTransform]); + + const updateTransform = trpc.frameVariants.updateTransform.useMutation({ + onError: (err) => toast.error(`Transform: ${err.message}`), + }); + + const saveTransform = useCallback(async (newTransform: LayerTransform) => { + if (!variantId) return; + setIsSaving(true); + try { + await updateTransform.mutateAsync({ variantId, transform: newTransform }); + } finally { + setIsSaving(false); + } + }, [variantId, updateTransform]); + + const resetTransform = useCallback(() => { + setTransform(DEFAULT_TRANSFORM); + saveTransform(DEFAULT_TRANSFORM); + }, [saveTransform]); + + const handleFlipH = () => { + const next = { ...transform, flipH: !transform.flipH }; + setTransform(next); + saveTransform(next); + }; + + const handleFlipV = () => { + const next = { ...transform, flipV: !transform.flipV }; + setTransform(next); + saveTransform(next); + }; + + // Mouse handling for drag + const handleMouseDown = (mode: "move" | "scale" | "rotate") => (e: React.MouseEvent) => { + e.stopPropagation(); + e.preventDefault(); + setDragMode(mode); + dragStartRef.current = { x: e.clientX, y: e.clientY, transform: { ...transform } }; + }; + + useEffect(() => { + if (dragMode === "none") return; + + const onMove = (e: MouseEvent) => { + if (!dragStartRef.current || !containerRef.current) return; + const rect = containerRef.current.getBoundingClientRect(); + const dx = e.clientX - dragStartRef.current.x; + const dy = e.clientY - dragStartRef.current.y; + const start = dragStartRef.current.transform; + + if (dragMode === "move") { + // Convert pixel delta to relative + setTransform({ + ...start, + x: (start.x ?? 0) + dx / rect.width, + y: (start.y ?? 0) + dy / rect.height, + }); + } else if (dragMode === "scale") { + // Distance from center increase = scale up + const distance = Math.sqrt(dx * dx + dy * dy); + const sign = dx + dy > 0 ? 1 : -1; + const factor = 1 + (sign * distance) / 200; + const newScale = Math.max(0.1, Math.min(5, (start.scale ?? 1) * factor)); + setTransform({ ...start, scale: newScale }); + } else if (dragMode === "rotate") { + // dx pixels = degree change + setTransform({ + ...start, + rotation: ((start.rotation ?? 0) + dx) % 360, + }); + } + }; + + const onUp = () => { + setDragMode("none"); + // Save on release + if (dragStartRef.current) { + const finalTransform = transform; + saveTransform(finalTransform); + dragStartRef.current = null; + } + }; + + document.addEventListener("mousemove", onMove); + document.addEventListener("mouseup", onUp); + return () => { + document.removeEventListener("mousemove", onMove); + document.removeEventListener("mouseup", onUp); + }; + }, [dragMode, transform, saveTransform]); + + // CSS transform: combine all + const cssTransform = [ + `translate(${(transform.x ?? 0) * 100}%, ${(transform.y ?? 0) * 100}%)`, + `scale(${transform.scale ?? 1})`, + `rotate(${transform.rotation ?? 0}deg)`, + transform.flipH ? "scaleX(-1)" : "", + transform.flipV ? "scaleY(-1)" : "", + ].filter(Boolean).join(" "); + + const isModified = + (transform.x ?? 0) !== 0 || + (transform.y ?? 0) !== 0 || + (transform.scale ?? 1) !== 1 || + (transform.rotation ?? 0) !== 0 || + transform.flipH || + transform.flipV; + + return ( +
+ {/* Layer image with live CSS transform */} +
+ Character +
+ + {/* Bounding box + handles */} +
+
+ {/* Move area in center */} +
+
+ +
+
+ + {/* Corner handles (scale) */} + {[ + { pos: "top-0 left-0 -translate-x-1/2 -translate-y-1/2", cursor: "nwse-resize" }, + { pos: "top-0 right-0 translate-x-1/2 -translate-y-1/2", cursor: "nesw-resize" }, + { pos: "bottom-0 left-0 -translate-x-1/2 translate-y-1/2", cursor: "nesw-resize" }, + { pos: "bottom-0 right-0 translate-x-1/2 translate-y-1/2", cursor: "nwse-resize" }, + ].map((h, i) => ( +
+ ))} + + {/* Rotation handle (top center, extended above) */} +
+
+
+
+
+
+ + {/* Toolbar */} +
+ + MANIPULATION + +
+ + + + +
+ + {/* Live values */} +
+ + + {((transform.x ?? 0) * 100).toFixed(0)},{((transform.y ?? 0) * 100).toFixed(0)}% + + + + {((transform.scale ?? 1) * 100).toFixed(0)}% + + + + {(transform.rotation ?? 0).toFixed(0)}° + +
+ +
+ + {isModified && ( + + )} + + {isSaving && } + + {onRecompose && ( + + )} +
+
+ ); +} diff --git a/client/src/components/ViewportPanel.tsx b/client/src/components/ViewportPanel.tsx index af82f8d..01a2056 100644 --- a/client/src/components/ViewportPanel.tsx +++ b/client/src/components/ViewportPanel.tsx @@ -4,6 +4,7 @@ import type { ViewMode } from "@/contexts/ShortcutsContext"; import ShortcutSettings from "@/components/ShortcutSettings"; import CompositePreview from "@/components/CompositePreview"; import AnnotationCanvas from "@/components/AnnotationCanvas"; +import LayerManipulator from "@/components/LayerManipulator"; import { Button } from "@/components/ui/button"; import { Badge } from "@/components/ui/badge"; import { Slider } from "@/components/ui/slider"; @@ -31,6 +32,7 @@ import { Circle, Square, Brush, + Move, } from "lucide-react"; type FrameUrlMap = Map; @@ -97,6 +99,7 @@ function ViewportPanelInner({ project, selectedFrame, layers, frameUrlMap, isPla const [onionFrameOffset, setOnionFrameOffset] = useState(1); const [imgErrors, setImgErrors] = useState>({}); const [annotating, setAnnotating] = useState(false); + const [manipulating, setManipulating] = useState(false); const containerRef = useRef(null); // Loupe state @@ -123,6 +126,22 @@ function ViewportPanelInner({ project, selectedFrame, layers, frameUrlMap, isPla const originalFrameUrl = cachedFrame?.originalUrl || frameData?.originalUrl || null; const regeneratedFrameUrl = cachedFrame?.compositedUrl || frameData?.compositedUrl || frameData?.regeneratedBgUrl || null; + // Character variants (for spatial transform manipulator) + const utils = trpc.useUtils(); + const { data: charVariants } = trpc.frameVariants.list.useQuery( + { frameId: (frameData as any)?.id || 0, type: "character" }, + { enabled: !!(frameData as any)?.id && manipulating, staleTime: 5_000 } + ); + const activeCharVariant = charVariants?.find(v => v.isActive); + + const composeFrame = trpc.compositing.composeFrame.useMutation({ + onSuccess: () => { + utils.frames.getByIndex.invalidate(); + utils.frames.list.invalidate(); + utils.frameVariants.list.invalidate(); + }, + }); + // Preload upcoming frames during playback useEffect(() => { if (!isPlaying || !frameUrlMap) return; @@ -499,7 +518,7 @@ function ViewportPanelInner({ project, selectedFrame, layers, frameUrlMap, isPla variant={annotating ? "secondary" : "ghost"} size="sm" className={`h-6 text-[10px] px-1.5 gap-0.5 ${annotating ? "ring-1 ring-primary/50" : ""}`} - onClick={() => { setAnnotating(prev => !prev); setLoupe(prev => ({ ...prev, active: false })); }} + onClick={() => { setAnnotating(prev => !prev); setLoupe(prev => ({ ...prev, active: false })); setManipulating(false); }} > Annoter @@ -510,6 +529,28 @@ function ViewportPanelInner({ project, selectedFrame, layers, frameUrlMap, isPla + {/* Layer manipulator (composite mode only) */} + {viewMode === "composite" && ( + + + + + + Déplacer / redimensionner / rotater le personnage + {!activeCharVariant && ⚠ Générez d'abord un personnage} + + + )} + {/* Loupe controls - shown when active */} {loupe.active && ( <> @@ -872,8 +913,20 @@ function ViewportPanelInner({ project, selectedFrame, layers, frameUrlMap, isPla
COMPOSITE
+ {/* Layer manipulator overlay (composite mode) */} + {manipulating && activeCharVariant && project?.id && ( + composeFrame.mutate({ projectId: project.id, frameIndex: selectedFrame, featherRadius: 2 })} + isRecomposing={composeFrame.isPending} + /> + )} {/* Loupe overlay */} - {loupe.active && renderLoupeOverlay()} + {loupe.active && !manipulating && renderLoupeOverlay()}
)} diff --git a/drizzle/schema.ts b/drizzle/schema.ts index 0a255c0..df34aa5 100644 --- a/drizzle/schema.ts +++ b/drizzle/schema.ts @@ -197,6 +197,8 @@ export const frameVariants = mysqlTable("frameVariants", { isPinned: boolean("isPinned").default(false), label: varchar("label", { length: 255 }), // user-defined name metadata: json("metadata"), // {model, prompt, style, etc} + /** Spatial transform applied when compositing: {x, y, scaleX, scaleY, rotation, flipH, flipV} - all relative (x,y in [-1,1], scale multiplier, rotation degrees) */ + transform: json("transform"), createdAt: timestamp("createdAt").defaultNow().notNull(), }); diff --git a/server/routers.ts b/server/routers.ts index aff104d..d3c8b38 100644 --- a/server/routers.ts +++ b/server/routers.ts @@ -793,6 +793,26 @@ Réponds de manière concise et technique, en français. Propose des actions con await db.renameVariant(input.variantId, input.label); return { success: true }; }), + updateTransform: protectedProcedure + .input(z.object({ + variantId: z.number(), + transform: z.object({ + x: z.number().optional(), + y: z.number().optional(), + scale: z.number().optional(), + rotation: z.number().optional(), + flipH: z.boolean().optional(), + flipV: z.boolean().optional(), + }).nullable(), + })) + .mutation(async ({ input }) => { + const dbInstance = await db.getDb(); + if (!dbInstance) return { success: false }; + const { frameVariants } = await import("../drizzle/schema"); + const { eq } = await import("drizzle-orm"); + await dbInstance.update(frameVariants).set({ transform: input.transform as any }).where(eq(frameVariants.id, input.variantId)); + return { success: true }; + }), }), // ============ COMPOSITING ============ @@ -810,7 +830,7 @@ Réponds de manière concise et technique, en français. Propose des actions con const projectLayers = await db.listLayers(input.projectId); const visibleLayers = projectLayers.filter(l => l.visible).sort((a, b) => (a.order ?? 0) - (b.order ?? 0)); - const composites: Array<{ imageUrl: string; opacity: number; blendMode?: "normal" | "multiply" | "screen" | "overlay"; maskUrl?: string }> = []; + const composites: Array<{ imageUrl: string; opacity: number; blendMode?: "normal" | "multiply" | "screen" | "overlay"; maskUrl?: string; transform?: any }> = []; const bgUrl = frame.regeneratedBgUrl || frame.backgroundUrl || frame.originalUrl; if (bgUrl) { @@ -819,11 +839,14 @@ Réponds de manière concise et technique, en français. Propose des actions con const fgUrl = frame.regeneratedFgUrl || frame.foregroundUrl; if (fgUrl) { + // Look up active character variant for its transform + const charVariant = await db.getActiveVariant(frame.id, "character"); composites.push({ imageUrl: fgUrl, opacity: 100, blendMode: "normal", maskUrl: frame.maskUrl || undefined, + transform: (charVariant?.transform as any) || undefined, }); } diff --git a/server/segmentationService.ts b/server/segmentationService.ts index f1bb6d0..35f2d68 100644 --- a/server/segmentationService.ts +++ b/server/segmentationService.ts @@ -333,11 +333,27 @@ export async function propagateMask( * - Black = background visible * - Gray = partial transparency (edges, anti-aliasing) */ +export interface LayerTransform { + /** Translation X relative to canvas width: -1..1 (0 = no shift) */ + x?: number; + /** Translation Y relative to canvas height: -1..1 */ + y?: number; + /** Scale multiplier (1.0 = original size) */ + scale?: number; + /** Rotation in degrees */ + rotation?: number; + /** Horizontal flip */ + flipH?: boolean; + /** Vertical flip */ + flipV?: boolean; +} + export interface CompositeLayer { imageUrl: string; opacity: number; blendMode?: "normal" | "multiply" | "screen" | "overlay"; maskUrl?: string; + transform?: LayerTransform; } type SharpBlendMode = "over" | "multiply" | "screen" | "overlay"; @@ -388,23 +404,31 @@ export async function compositeLayers( const layer = layers[i]; let layerBuffer = await fetchImageBuffer(layer.imageUrl); - // Resize to match base - let layerPipeline = sharp(layerBuffer).resize(width, height, { fit: "contain", background: { r: 0, g: 0, b: 0, alpha: 0 } }); + // Apply transform if present (translate, scale, rotate, flip) + const t = layer.transform; + const hasTransform = t && (t.x || t.y || (t.scale !== undefined && t.scale !== 1) || t.rotation || t.flipH || t.flipV); - // Apply mask if provided + // Compute target layer dimensions + const scale = t?.scale ?? 1; + const layerW = Math.max(1, Math.round(width * scale)); + const layerH = Math.max(1, Math.round(height * scale)); + + let layerPipeline = sharp(layerBuffer).resize(layerW, layerH, { fit: "contain", background: { r: 0, g: 0, b: 0, alpha: 0 } }); + + // Apply mask if provided (mask is matched to layer pre-transform) if (layer.maskUrl) { const maskBuffer = await fetchImageBuffer(layer.maskUrl); - let maskPipeline = sharp(maskBuffer).resize(width, height, { fit: "contain" }).greyscale(); + let maskPipeline = sharp(maskBuffer).resize(layerW, layerH, { fit: "contain" }).greyscale(); if (options.featherRadius && options.featherRadius > 0) { maskPipeline = maskPipeline.blur(options.featherRadius); } const maskRaw = await maskPipeline.raw().toBuffer(); - layerPipeline = layerPipeline.ensureAlpha().joinChannel(maskRaw, { raw: { width, height, channels: 1 } }); + layerPipeline = layerPipeline.ensureAlpha().joinChannel(maskRaw, { raw: { width: layerW, height: layerH, channels: 1 } }); } else { layerPipeline = layerPipeline.ensureAlpha(); } - // Apply layer opacity via composite + // Apply opacity if (layer.opacity < 100) { const alphaMultiplier = layer.opacity / 100; const rawData = await layerPipeline.raw().toBuffer({ resolveWithObject: true }); @@ -412,15 +436,66 @@ export async function compositeLayers( for (let p = 0; p < rawData.data.length; p += channels) { rawData.data[p + channels - 1] = Math.round(rawData.data[p + channels - 1] * alphaMultiplier); } - layerBuffer = await sharp(rawData.data, { raw: { width, height, channels } }).png().toBuffer(); + layerBuffer = await sharp(rawData.data, { raw: { width: layerW, height: layerH, channels } }).png().toBuffer(); } else { layerBuffer = await layerPipeline.png().toBuffer(); } - composites.push({ - input: layerBuffer, - blend: blendModeMap[layer.blendMode || "normal"] || "over", - }); + // Apply flip + rotation + if (hasTransform) { + let transformedPipeline = sharp(layerBuffer); + if (t.flipH) transformedPipeline = transformedPipeline.flop(); + if (t.flipV) transformedPipeline = transformedPipeline.flip(); + if (t.rotation && t.rotation !== 0) { + transformedPipeline = transformedPipeline.rotate(t.rotation, { background: { r: 0, g: 0, b: 0, alpha: 0 } }); + } + layerBuffer = await transformedPipeline.png().toBuffer(); + } + + // Compute placement on the base canvas + const finalDims = await sharp(layerBuffer).metadata(); + const fW = finalDims.width || layerW; + const fH = finalDims.height || layerH; + const baseLeft = Math.round((width - fW) / 2); + const baseTop = Math.round((height - fH) / 2); + const offsetX = Math.round((t?.x ?? 0) * width); + const offsetY = Math.round((t?.y ?? 0) * height); + const placeLeft = baseLeft + offsetX; + const placeTop = baseTop + offsetY; + + // Sharp's composite requires top/left >= 0 AND layer to fit within base. + // For transformed layers that overflow or have negative placement, we + // need to extract the visible portion from the layer before compositing. + if (fW > width || fH > height || placeLeft < 0 || placeTop < 0 || placeLeft + fW > width || placeTop + fH > height) { + // Compute visible crop from layer + const cropLeft = Math.max(0, -placeLeft); + const cropTop = Math.max(0, -placeTop); + const cropRight = Math.min(fW, width - placeLeft); + const cropBottom = Math.min(fH, height - placeTop); + const cropW = Math.max(0, cropRight - cropLeft); + const cropH = Math.max(0, cropBottom - cropTop); + + if (cropW > 0 && cropH > 0) { + const croppedLayer = await sharp(layerBuffer) + .extract({ left: cropLeft, top: cropTop, width: cropW, height: cropH }) + .png() + .toBuffer(); + composites.push({ + input: croppedLayer, + blend: blendModeMap[layer.blendMode || "normal"] || "over", + top: Math.max(0, placeTop), + left: Math.max(0, placeLeft), + }); + } + // else: layer is fully out of bounds, skip it + } else { + composites.push({ + input: layerBuffer, + blend: blendModeMap[layer.blendMode || "normal"] || "over", + top: placeTop, + left: placeLeft, + }); + } } const finalBuffer = await pipeline.composite(composites).png().toBuffer();