import { useState, useCallback } from "react"; import { useAgent } from "agents/react"; import type { SoundscapeAgent, SoundscapeState, Scene } from "../agents/soundscape"; import { Button, InputArea, Surface, Text, Empty } from "@cloudflare/kumo"; import { PlayIcon, PauseIcon, MagicWandIcon, FloppyDiskIcon, WaveformIcon, SpinnerIcon, SpeakerHighIcon, CircleIcon } from "@phosphor-icons/react"; import { AudioButton } from "../components/audio-player"; type GenerationStep = | { stage: "idle" } | { stage: "expanding" } | { stage: "generating"; narrationDone: boolean; effectsDone: number; totalEffects: number; } | { stage: "done"; scene: Scene }; export function SoundscapeTab() { const [connected, setConnected] = useState(false); const [description, setDescription] = useState(""); const [step, setStep] = useState({ stage: "idle" }); const [currentScene, setCurrentScene] = useState(null); const [playingAll, setPlayingAll] = useState(false); const [audioElements, setAudioElements] = useState([]); const agent = useAgent({ agent: "SoundscapeAgent", onOpen: useCallback(() => setConnected(true), []), onClose: useCallback(() => setConnected(false), []) }); const savedScenes = (agent.state?.scenes ?? []) as Scene[]; const generate = useCallback(async () => { const desc = description.trim(); if (!desc) return; setStep({ stage: "expanding" }); setCurrentScene(null); try { const plan = (await agent.call("expandScene", [desc])) as { narration: string; effects: string[]; }; const scene: Scene = { id: crypto.randomUUID(), name: desc, narrationText: plan.narration, effects: plan.effects.map((prompt) => ({ id: crypto.randomUUID(), prompt, audio: "" })) }; setCurrentScene(scene); setStep({ stage: "generating", narrationDone: false, effectsDone: 0, totalEffects: plan.effects.length }); const narrationPromise = agent .call("generateNarration", [plan.narration]) .then((audio) => { scene.narrationAudio = audio as string; setCurrentScene({ ...scene }); setStep((prev) => prev.stage === "generating" ? { ...prev, narrationDone: true } : prev ); }); const effectPromises = plan.effects.map((prompt, i) => agent.call("generateEffect", [prompt]).then((audio) => { scene.effects[i] = { ...scene.effects[i], audio: audio as string }; setCurrentScene({ ...scene }); setStep((prev) => prev.stage === "generating" ? { ...prev, effectsDone: prev.effectsDone + 1 } : prev ); }) ); await Promise.all([narrationPromise, ...effectPromises]); setStep({ stage: "done", scene }); } catch (e) { console.error("Generation failed:", e); setStep({ stage: "idle" }); } }, [description, agent]); const playAll = useCallback(() => { if (!currentScene) return; if (playingAll) { audioElements.forEach((a) => { a.pause(); a.currentTime = 0; }); setAudioElements([]); setPlayingAll(false); return; } const elements: HTMLAudioElement[] = []; if (currentScene.narrationAudio) { const a = new Audio(currentScene.narrationAudio); elements.push(a); } for (const effect of currentScene.effects) { if (effect.audio) { const a = new Audio(effect.audio); a.volume = 0.4; a.loop = true; elements.push(a); } } elements.forEach((a) => a.play()); setAudioElements(elements); setPlayingAll(true); if (elements[0]) { elements[0].onended = () => { elements.forEach((a) => { a.pause(); a.currentTime = 0; }); setAudioElements([]); setPlayingAll(false); }; } }, [currentScene, playingAll, audioElements]); const saveScene = useCallback(async () => { if (!currentScene) return; await agent.call("saveScene", [currentScene]); }, [agent, currentScene]); const loadScene = useCallback((scene: Scene) => { setCurrentScene(scene); setStep({ stage: "done", scene }); setDescription(scene.name); }, []); const isGenerating = step.stage === "expanding" || step.stage === "generating"; return (

{/* Main panel */}

{/* Input */}

Describe a scene

{connected ? "Connected" : "Disconnected"}

{[ "A rainy café in Tokyo with jazz playing softly", "A campfire in the woods at night with crickets and an owl", "A busy spaceport with engines, announcements, and crowds", "An underwater coral reef with whale songs and bubbles" ].map((preset) => ( ))}

{/* Progress */} {step.stage === "expanding" && (

AI is designing your soundscape...

)} {step.stage === "generating" && (

{step.narrationDone ? ( ) : ( )} Narration {step.narrationDone ? "ready" : "generating..."}

{step.effectsDone === step.totalEffects ? ( ) : ( )} Sound effects {step.effectsDone}/{step.totalEffects}

)} {/* Scene preview */} {currentScene && (

{currentScene.name}

{step.stage === "done" && ( <> )}

{/* Narration */}

Narration

{currentScene.narrationText}

{currentScene.narrationAudio && ( )}

{/* Sound effects */}

Ambient Layers

{currentScene.effects.map((effect) => (

{effect.prompt} {effect.audio ? ( ) : ( )}

))}

)} {!currentScene && step.stage === "idle" && ( } title="No soundscape yet" contents="Describe a scene above and the AI will generate narration and ambient sound effects using ElevenLabs." /> )}

{/* Saved scenes sidebar */} {savedScenes.length > 0 && (

Saved Scenes {savedScenes.map((scene) => ( ))}

)}

); }