feat: panneau Génération IA avec regénération fond/personnage
- Nouvel onglet "Génération IA" dans le workspace avec 2 boutons: * Regénérer l'arrière-plan (prompt + style) * Redessiner le personnage (prompt + sélecteur de character sheet) - 3 endpoints tRPC: generation.regenerateBackground, generation.regenerateCharacter, generation.inpaintBackground - Fix URLs relatives -> signed URLs S3 absolues pour l'API Forge - Résultat affiché en preview dans le panneau - Testé: génération cyberpunk sur frame 200 -> PNG 1344x768 OK Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ubuntu
parent
a6f508fa8b
commit
51bb69eb88
3 changed files with 316 additions and 101 deletions
|
|
@ -11,7 +11,7 @@ import { trpc } from "@/lib/trpc";
|
|||
import {
|
||||
ArrowLeft, Play, Pause, SkipBack, SkipForward,
|
||||
Layers, Eye, EyeOff, Lock, Unlock,
|
||||
Wand2, MessageSquare, Settings, Film,
|
||||
Wand2, MessageSquare, Settings, Film, Sparkles,
|
||||
Scissors, Image, Users, Bot, Download,
|
||||
ChevronLeft, ChevronRight, Loader2
|
||||
} from "lucide-react";
|
||||
|
|
@ -310,6 +310,10 @@ export default function ProjectWorkspace() {
|
|||
<Users className="h-3.5 w-3.5" />
|
||||
Personnages
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value="generate" className="gap-1 text-xs">
|
||||
<Wand2 className="h-3.5 w-3.5" />
|
||||
Génération IA
|
||||
</TabsTrigger>
|
||||
</TabsList>
|
||||
<TabsContent value="timeline" className="flex-1 m-0 overflow-hidden">
|
||||
<TimelinePanel
|
||||
|
|
@ -334,6 +338,9 @@ export default function ProjectWorkspace() {
|
|||
<TabsContent value="characters" className="flex-1 m-0 overflow-hidden p-3">
|
||||
<CharactersPanel projectId={projectId} />
|
||||
</TabsContent>
|
||||
<TabsContent value="generate" className="flex-1 m-0 overflow-hidden p-3">
|
||||
<GenerationPanel projectId={projectId} selectedFrame={selectedFrame} />
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -593,6 +600,7 @@ function CharactersPanel({ projectId }: { projectId: number }) {
|
|||
)}
|
||||
|
||||
{/* Full-screen preview modal */}
|
||||
{/* eslint-disable-next-line @typescript-eslint/no-use-before-define */}
|
||||
{previewUrl && (
|
||||
<div
|
||||
className="fixed inset-0 z-50 bg-black/80 flex items-center justify-center p-8 cursor-pointer"
|
||||
|
|
@ -612,3 +620,123 @@ function CharactersPanel({ projectId }: { projectId: number }) {
|
|||
</ScrollArea>
|
||||
);
|
||||
}
|
||||
|
||||
// Generation panel for AI-powered background/character regeneration
|
||||
function GenerationPanel({ projectId, selectedFrame }: { projectId: number; selectedFrame: number }) {
|
||||
const [bgPrompt, setBgPrompt] = useState("Redessiner cet arrière-plan dans un style Studio Ghibli, couleurs douces et atmosphère onirique");
|
||||
const [charPrompt, setCharPrompt] = useState("Redessiner ce personnage dans un style moderne, lignes nettes et couleurs vives");
|
||||
const [bgStyle, setBgStyle] = useState("Studio Ghibli");
|
||||
const [resultUrl, setResultUrl] = useState<string | null>(null);
|
||||
|
||||
const { data: characters } = trpc.characters.list.useQuery({ projectId });
|
||||
const [selectedCharId, setSelectedCharId] = useState<number | undefined>();
|
||||
|
||||
const regenBg = trpc.generation.regenerateBackground.useMutation({
|
||||
onSuccess: (data) => {
|
||||
setResultUrl(data.resultUrl);
|
||||
toast.success("Arrière-plan regénéré !");
|
||||
},
|
||||
onError: (err) => toast.error(`Erreur: ${err.message}`),
|
||||
});
|
||||
|
||||
const regenChar = trpc.generation.regenerateCharacter.useMutation({
|
||||
onSuccess: (data) => {
|
||||
setResultUrl(data.resultUrl);
|
||||
toast.success("Personnage redessiné !");
|
||||
},
|
||||
onError: (err) => toast.error(`Erreur: ${err.message}`),
|
||||
});
|
||||
|
||||
const isGenerating = regenBg.isPending || regenChar.isPending;
|
||||
|
||||
return (
|
||||
<ScrollArea className="h-full">
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<div className="text-xs text-muted-foreground font-mono col-span-2 flex items-center gap-2">
|
||||
<Wand2 className="h-3.5 w-3.5 text-primary" />
|
||||
Frame sélectionnée : #{selectedFrame + 1}
|
||||
</div>
|
||||
|
||||
{/* Background regeneration */}
|
||||
<div className="p-3 rounded-lg border border-border bg-card/50 space-y-2">
|
||||
<div className="flex items-center gap-2">
|
||||
<Image className="h-4 w-4 text-blue-400" />
|
||||
<span className="text-sm font-medium">Regénérer l'arrière-plan</span>
|
||||
</div>
|
||||
<textarea
|
||||
value={bgPrompt}
|
||||
onChange={(e) => setBgPrompt(e.target.value)}
|
||||
placeholder="Décrivez le nouveau style d'arrière-plan..."
|
||||
className="w-full px-2 py-1.5 text-xs rounded border border-border bg-input resize-none h-12"
|
||||
/>
|
||||
<div className="flex items-center gap-2">
|
||||
<input
|
||||
type="text"
|
||||
value={bgStyle}
|
||||
onChange={(e) => setBgStyle(e.target.value)}
|
||||
placeholder="Style"
|
||||
className="flex-1 px-2 py-1 text-xs rounded border border-border bg-input"
|
||||
/>
|
||||
<Button
|
||||
size="sm"
|
||||
className="gap-1 text-xs h-7"
|
||||
disabled={isGenerating || !bgPrompt.trim()}
|
||||
onClick={() => regenBg.mutate({ projectId, frameIndex: selectedFrame, prompt: bgPrompt, style: bgStyle })}
|
||||
>
|
||||
{regenBg.isPending ? <Loader2 className="h-3 w-3 animate-spin" /> : <Wand2 className="h-3 w-3" />}
|
||||
Générer le fond
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Character regeneration */}
|
||||
<div className="p-3 rounded-lg border border-border bg-card/50 space-y-2">
|
||||
<div className="flex items-center gap-2">
|
||||
<Users className="h-4 w-4 text-green-400" />
|
||||
<span className="text-sm font-medium">Redessiner le personnage</span>
|
||||
</div>
|
||||
<textarea
|
||||
value={charPrompt}
|
||||
onChange={(e) => setCharPrompt(e.target.value)}
|
||||
placeholder="Décrivez le nouveau style du personnage..."
|
||||
className="w-full px-2 py-1.5 text-xs rounded border border-border bg-input resize-none h-12"
|
||||
/>
|
||||
<div className="flex items-center gap-2">
|
||||
{characters && characters.length > 0 && (
|
||||
<select
|
||||
value={selectedCharId || ""}
|
||||
onChange={(e) => setSelectedCharId(e.target.value ? Number(e.target.value) : undefined)}
|
||||
className="flex-1 px-2 py-1 text-xs rounded border border-border bg-input"
|
||||
>
|
||||
<option value="">Sans référence</option>
|
||||
{characters.map(c => (
|
||||
<option key={c.id} value={c.id}>{c.name}</option>
|
||||
))}
|
||||
</select>
|
||||
)}
|
||||
<Button
|
||||
size="sm"
|
||||
className="gap-1 text-xs h-7"
|
||||
disabled={isGenerating || !charPrompt.trim()}
|
||||
onClick={() => regenChar.mutate({ projectId, frameIndex: selectedFrame, prompt: charPrompt, characterId: selectedCharId })}
|
||||
>
|
||||
{regenChar.isPending ? <Loader2 className="h-3 w-3 animate-spin" /> : <Wand2 className="h-3 w-3" />}
|
||||
Générer le personnage
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Result preview */}
|
||||
{resultUrl && (
|
||||
<div className="p-3 rounded-lg border border-primary/30 bg-card/50 space-y-2 col-span-2">
|
||||
<div className="flex items-center gap-2">
|
||||
<Sparkles className="h-4 w-4 text-primary" />
|
||||
<span className="text-sm font-medium">Résultat généré</span>
|
||||
</div>
|
||||
<img src={resultUrl} alt="Résultat" className="max-h-32 rounded border border-border" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</ScrollArea>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -520,6 +520,104 @@ Réponds de manière concise et technique, en français. Propose des actions con
|
|||
}),
|
||||
}),
|
||||
|
||||
// ============ GENERATION ============
|
||||
generation: router({
|
||||
regenerateBackground: protectedProcedure
|
||||
.input(z.object({
|
||||
projectId: z.number(),
|
||||
frameIndex: z.number(),
|
||||
prompt: z.string().min(1),
|
||||
style: z.string().optional(),
|
||||
}))
|
||||
.mutation(async ({ input }) => {
|
||||
const frame = await db.getFrame(input.projectId, input.frameIndex);
|
||||
if (!frame?.originalUrl) throw new TRPCError({ code: "NOT_FOUND", message: "Frame not found" });
|
||||
|
||||
const { regenerateBackground } = await import("./segmentationService");
|
||||
const resultUrl = await regenerateBackground(
|
||||
frame.originalUrl,
|
||||
input.prompt,
|
||||
input.style || "same art style"
|
||||
);
|
||||
|
||||
await db.updateFrame(frame.id, { regeneratedBgUrl: resultUrl });
|
||||
await db.createGenerationJob({
|
||||
projectId: input.projectId,
|
||||
type: "background_gen",
|
||||
prompt: input.prompt,
|
||||
status: "completed",
|
||||
resultUrl,
|
||||
});
|
||||
|
||||
return { success: true, resultUrl };
|
||||
}),
|
||||
|
||||
regenerateCharacter: protectedProcedure
|
||||
.input(z.object({
|
||||
projectId: z.number(),
|
||||
frameIndex: z.number(),
|
||||
prompt: z.string().min(1),
|
||||
characterId: z.number().optional(),
|
||||
}))
|
||||
.mutation(async ({ input }) => {
|
||||
const frame = await db.getFrame(input.projectId, input.frameIndex);
|
||||
if (!frame?.originalUrl) throw new TRPCError({ code: "NOT_FOUND", message: "Frame not found" });
|
||||
|
||||
let characterSheet: string | undefined;
|
||||
let characterConfig: { name: string; modelType: "lora" | "ip_adapter" | "none" } | undefined;
|
||||
|
||||
if (input.characterId) {
|
||||
const chars = await db.listCharacters(input.projectId);
|
||||
const char = chars.find(c => c.id === input.characterId);
|
||||
if (char) {
|
||||
characterSheet = char.referenceSheetUrl || undefined;
|
||||
characterConfig = { name: char.name, modelType: (char.modelType as any) || "none" };
|
||||
}
|
||||
}
|
||||
|
||||
const { regenerateCharacter } = await import("./segmentationService");
|
||||
const resultUrl = await regenerateCharacter(
|
||||
frame.originalUrl,
|
||||
input.prompt,
|
||||
characterSheet,
|
||||
frame.maskUrl || undefined,
|
||||
characterConfig
|
||||
);
|
||||
|
||||
await db.updateFrame(frame.id, { regeneratedFgUrl: resultUrl });
|
||||
await db.createGenerationJob({
|
||||
projectId: input.projectId,
|
||||
type: "character_gen",
|
||||
prompt: input.prompt,
|
||||
status: "completed",
|
||||
resultUrl,
|
||||
});
|
||||
|
||||
return { success: true, resultUrl };
|
||||
}),
|
||||
|
||||
inpaintBackground: protectedProcedure
|
||||
.input(z.object({
|
||||
projectId: z.number(),
|
||||
frameIndex: z.number(),
|
||||
prompt: z.string().optional(),
|
||||
}))
|
||||
.mutation(async ({ input }) => {
|
||||
const frame = await db.getFrame(input.projectId, input.frameIndex);
|
||||
if (!frame?.originalUrl) throw new TRPCError({ code: "NOT_FOUND", message: "Frame not found" });
|
||||
|
||||
const { inpaintBackground } = await import("./segmentationService");
|
||||
const resultUrl = await inpaintBackground(
|
||||
frame.originalUrl,
|
||||
frame.maskUrl || frame.originalUrl,
|
||||
input.prompt
|
||||
);
|
||||
|
||||
await db.updateFrame(frame.id, { backgroundUrl: resultUrl });
|
||||
return { success: true, resultUrl };
|
||||
}),
|
||||
}),
|
||||
|
||||
// ============ EXPORT ============
|
||||
export: router({
|
||||
start: protectedProcedure
|
||||
|
|
|
|||
|
|
@ -11,8 +11,15 @@
|
|||
|
||||
import { generateImage } from "./_core/imageGeneration";
|
||||
import { invokeLLM } from "./_core/llm";
|
||||
import { storageGetSignedUrl } from "./storage";
|
||||
import { getServicesConfig, callExternalSAM2, getCharacterGenerationStrategy, buildPoseConstraints } from "./servicesConfig";
|
||||
|
||||
async function resolveToAbsoluteUrl(url: string): Promise<string> {
|
||||
if (url.startsWith("http")) return url;
|
||||
const key = url.replace(/^\/manus-storage\//, "");
|
||||
return storageGetSignedUrl(key);
|
||||
}
|
||||
|
||||
export interface SegmentationResult {
|
||||
maskUrl: string;
|
||||
foregroundUrl: string;
|
||||
|
|
@ -123,33 +130,24 @@ export async function inpaintBackground(
|
|||
maskUrl: string,
|
||||
prompt?: string
|
||||
): Promise<string> {
|
||||
try {
|
||||
// Build a detailed prompt that instructs the model to use the mask
|
||||
const inpaintPrompt = [
|
||||
prompt || "Clean background plate, seamlessly fill the masked areas",
|
||||
"Maintain original art style, color palette, and perspective.",
|
||||
"The second reference image is the mask: white areas should be inpainted,",
|
||||
"black areas should remain unchanged. Produce a complete background without characters.",
|
||||
].join(" ");
|
||||
const absoluteFrameUrl = await resolveToAbsoluteUrl(frameUrl);
|
||||
const absoluteMaskUrl = await resolveToAbsoluteUrl(maskUrl);
|
||||
|
||||
const result = await generateImage({
|
||||
prompt: inpaintPrompt,
|
||||
originalImages: [
|
||||
{
|
||||
url: frameUrl,
|
||||
mimeType: "image/png",
|
||||
},
|
||||
{
|
||||
url: maskUrl,
|
||||
mimeType: "image/png",
|
||||
},
|
||||
],
|
||||
});
|
||||
return result.url || frameUrl;
|
||||
} catch (error) {
|
||||
console.error("[Segmentation] Inpainting failed:", error);
|
||||
return frameUrl; // Fallback to original
|
||||
}
|
||||
const inpaintPrompt = [
|
||||
prompt || "Clean background plate, seamlessly fill the masked areas",
|
||||
"Maintain original art style, color palette, and perspective.",
|
||||
"The second reference image is the mask: white areas should be inpainted,",
|
||||
"black areas should remain unchanged. Produce a complete background without characters.",
|
||||
].join(" ");
|
||||
|
||||
const result = await generateImage({
|
||||
prompt: inpaintPrompt,
|
||||
originalImages: [
|
||||
{ url: absoluteFrameUrl, mimeType: "image/jpeg" },
|
||||
{ url: absoluteMaskUrl, mimeType: "image/jpeg" },
|
||||
],
|
||||
});
|
||||
return result.url || frameUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -161,29 +159,25 @@ export async function regenerateBackground(
|
|||
prompt: string,
|
||||
style: string = "same art style"
|
||||
): Promise<string> {
|
||||
try {
|
||||
const fullPrompt = [
|
||||
prompt,
|
||||
`Maintain exact same perspective, composition, and spatial layout.`,
|
||||
`Style: ${style}.`,
|
||||
`This is a background for animation - no characters should be present.`,
|
||||
`Keep the same camera angle and depth of field as the reference.`,
|
||||
].join(" ");
|
||||
|
||||
const result = await generateImage({
|
||||
prompt: fullPrompt,
|
||||
originalImages: [
|
||||
{
|
||||
url: referenceFrameUrl,
|
||||
mimeType: "image/png",
|
||||
},
|
||||
],
|
||||
});
|
||||
return result.url || referenceFrameUrl;
|
||||
} catch (error) {
|
||||
console.error("[Segmentation] Background regeneration failed:", error);
|
||||
return referenceFrameUrl;
|
||||
}
|
||||
const absoluteUrl = await resolveToAbsoluteUrl(referenceFrameUrl);
|
||||
const fullPrompt = [
|
||||
prompt,
|
||||
`Maintain exact same perspective, composition, and spatial layout.`,
|
||||
`Style: ${style}.`,
|
||||
`This is a background for animation - no characters should be present.`,
|
||||
`Keep the same camera angle and depth of field as the reference.`,
|
||||
].join(" ");
|
||||
|
||||
const result = await generateImage({
|
||||
prompt: fullPrompt,
|
||||
originalImages: [
|
||||
{
|
||||
url: absoluteUrl,
|
||||
mimeType: "image/jpeg",
|
||||
},
|
||||
],
|
||||
});
|
||||
return result.url || referenceFrameUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -198,59 +192,54 @@ export async function regenerateCharacter(
|
|||
maskUrl?: string,
|
||||
characterConfig?: { name: string; modelType: "lora" | "ip_adapter" | "none" }
|
||||
): Promise<string> {
|
||||
try {
|
||||
const images: Array<{ url: string; mimeType: "image/png" | "image/jpeg" }> = [
|
||||
{ url: characterFrameUrl, mimeType: "image/png" },
|
||||
];
|
||||
|
||||
// Determine generation strategy based on character config
|
||||
let strategyPrefix = "";
|
||||
if (characterConfig) {
|
||||
const strategy = getCharacterGenerationStrategy(
|
||||
characterConfig.modelType,
|
||||
characterConfig.name,
|
||||
characterSheet
|
||||
);
|
||||
strategyPrefix = strategy.promptPrefix;
|
||||
// Add strategy-specific reference images
|
||||
for (const ref of strategy.referenceImages) {
|
||||
if (!images.some(img => img.url === ref.url)) {
|
||||
images.push(ref);
|
||||
}
|
||||
}
|
||||
} else if (characterSheet) {
|
||||
images.push({ url: characterSheet, mimeType: "image/png" });
|
||||
}
|
||||
|
||||
// Add mask to indicate which area contains the character
|
||||
if (maskUrl) {
|
||||
images.push({ url: maskUrl, mimeType: "image/png" });
|
||||
}
|
||||
const absoluteFrameUrl = await resolveToAbsoluteUrl(characterFrameUrl);
|
||||
const images: Array<{ url: string; mimeType: "image/png" | "image/jpeg" }> = [
|
||||
{ url: absoluteFrameUrl, mimeType: "image/jpeg" },
|
||||
];
|
||||
|
||||
// Build pose-constrained prompt
|
||||
const basePrompt = [
|
||||
strategyPrefix,
|
||||
prompt,
|
||||
characterSheet && !characterConfig ? "Match the character reference sheet style exactly." : "",
|
||||
maskUrl ? "The mask image indicates the character silhouette to preserve." : "",
|
||||
"Output only the character on a transparent background.",
|
||||
].filter(Boolean).join(" ");
|
||||
|
||||
const fullPrompt = buildPoseConstraints(basePrompt, {
|
||||
preservePose: true,
|
||||
preserveProportions: true,
|
||||
preservePosition: true,
|
||||
});
|
||||
|
||||
const result = await generateImage({
|
||||
prompt: fullPrompt,
|
||||
originalImages: images,
|
||||
});
|
||||
return result.url || characterFrameUrl;
|
||||
} catch (error) {
|
||||
console.error("[Segmentation] Character regeneration failed:", error);
|
||||
return characterFrameUrl;
|
||||
let strategyPrefix = "";
|
||||
if (characterConfig) {
|
||||
const strategy = getCharacterGenerationStrategy(
|
||||
characterConfig.modelType,
|
||||
characterConfig.name,
|
||||
characterSheet
|
||||
);
|
||||
strategyPrefix = strategy.promptPrefix;
|
||||
for (const ref of strategy.referenceImages) {
|
||||
const absRef = await resolveToAbsoluteUrl(ref.url);
|
||||
if (!images.some(img => img.url === absRef)) {
|
||||
images.push({ url: absRef, mimeType: ref.mimeType });
|
||||
}
|
||||
}
|
||||
} else if (characterSheet) {
|
||||
const absSheet = await resolveToAbsoluteUrl(characterSheet);
|
||||
images.push({ url: absSheet, mimeType: "image/jpeg" });
|
||||
}
|
||||
|
||||
if (maskUrl) {
|
||||
const absMask = await resolveToAbsoluteUrl(maskUrl);
|
||||
images.push({ url: absMask, mimeType: "image/jpeg" });
|
||||
}
|
||||
|
||||
const basePrompt = [
|
||||
strategyPrefix,
|
||||
prompt,
|
||||
characterSheet && !characterConfig ? "Match the character reference sheet style exactly." : "",
|
||||
maskUrl ? "The mask image indicates the character silhouette to preserve." : "",
|
||||
"Output only the character on a transparent background.",
|
||||
].filter(Boolean).join(" ");
|
||||
|
||||
const fullPrompt = buildPoseConstraints(basePrompt, {
|
||||
preservePose: true,
|
||||
preserveProportions: true,
|
||||
preservePosition: true,
|
||||
});
|
||||
|
||||
const result = await generateImage({
|
||||
prompt: fullPrompt,
|
||||
originalImages: images,
|
||||
});
|
||||
return result.url || characterFrameUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue