diff --git a/app-config.ts b/app-config.ts index cb1a9fd5d..a6ab196ae 100644 --- a/app-config.ts +++ b/app-config.ts @@ -13,6 +13,7 @@ export interface AppConfig { accent?: string; logoDark?: string; accentDark?: string; + audioVisualizer?: 'bar' | 'radial' | 'grid' | 'aura' | 'wave'; // for LiveKit Cloud Sandbox sandboxId?: string; @@ -34,6 +35,7 @@ export const APP_CONFIG_DEFAULTS: AppConfig = { logoDark: '/lk-logo-dark.svg', accentDark: '#1fd5f9', startButtonText: 'Start call', + audioVisualizer: 'aura', // for LiveKit Cloud Sandbox sandboxId: undefined, diff --git a/app/layout.tsx b/app/layout.tsx index 171d4453e..252e2bcbd 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -1,7 +1,8 @@ import { Public_Sans } from 'next/font/google'; import localFont from 'next/font/local'; import { headers } from 'next/headers'; -import { ApplyThemeScript, ThemeToggle } from '@/components/app/theme-toggle'; +import { ThemeProvider } from '@/components/app/theme-provider'; +import { ThemeToggle } from '@/components/app/theme-toggle'; import { cn, getAppConfig, getStyles } from '@/lib/utils'; import '@/styles/globals.css'; @@ -61,13 +62,19 @@ export default async function RootLayout({ children }: RootLayoutProps) { {styles && } {pageTitle} - - {children} -
- -
+ + {children} +
+ +
+ ); diff --git a/app/ui/(landing-page)/page.tsx b/app/ui/(landing-page)/page.tsx new file mode 100644 index 000000000..46ad2c8df --- /dev/null +++ b/app/ui/(landing-page)/page.tsx @@ -0,0 +1,102 @@ +'use client'; + +import Link from 'next/link'; +import { useVoiceAssistant } from '@livekit/components-react'; +import { AgentControlBar } from '@/components/livekit/agent-control-bar/agent-control-bar'; +import { AudioBarVisualizer } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer'; +import { Button } from '@/components/livekit/button'; +import { ChatEntry } from '@/components/livekit/chat-entry'; +import { useMicrophone } from '@/hooks/useMicrophone'; + +export default function Page() { + const { state, audioTrack } = useVoiceAssistant(); + + useMicrophone(); + + return ( + <> +
+

+ + + + UI +

+

+ A set of Open Source UI components for +
+ building beautiful voice experiences. +

+
+ + +
+
+ +
+
+
+
+
+ +
+ +
+
+
+
+
+ + +
+ +
+
+
+
+ + ); +} diff --git a/app/ui/README.md b/app/ui/README.md new file mode 100644 index 000000000..472efb953 --- /dev/null +++ b/app/ui/README.md @@ -0,0 +1,13 @@ +THIS IS NOT PART OF THE MAIN APPLICATION CODE. + +This folder contains code for testing and previewing LiveKit's UI component library in isolation. + +## Getting started + +To run the development server, run the following command: + +```bash +npm run dev +``` + +Then, navigate to `http://localhost:3000/ui` to see the components. diff --git a/app/ui/components/[...slug]/page.tsx b/app/ui/components/[...slug]/page.tsx new file mode 100644 index 000000000..08f917075 --- /dev/null +++ b/app/ui/components/[...slug]/page.tsx @@ -0,0 +1,21 @@ +'use client'; + +import dynamic from 'next/dynamic'; +import { redirect, useParams } from 'next/navigation'; + +export default function Page() { + const { slug = [] } = useParams(); + const [componentName] = slug; + const ComponentDemo = dynamic(() => import(`@/components/demos/${componentName}`)); + + if (!ComponentDemo) { + return redirect('/ui'); + } + + return ( + <> +

{componentName}

+ + + ); +} diff --git a/app/ui/components/layout.tsx b/app/ui/components/layout.tsx new file mode 100644 index 000000000..495d12fff --- /dev/null +++ b/app/ui/components/layout.tsx @@ -0,0 +1,26 @@ +import { SideNav } from '@/components/docs/side-nav'; +import { getComponentNames } from '@/lib/components'; + +interface LayoutProps { + children: React.ReactNode; +} + +export default function Layout({ children }: LayoutProps) { + const componentNames = getComponentNames(); + + return ( +
+ + +
+
{children}
+
+ + +
+ ); +} diff --git a/app/ui/components/page.tsx b/app/ui/components/page.tsx new file mode 100644 index 000000000..decfd333a --- /dev/null +++ b/app/ui/components/page.tsx @@ -0,0 +1,31 @@ +import Link from 'next/link'; +import { getComponentNames } from '@/lib/components'; + +export default function Page() { + const componentNames = getComponentNames(); + + return ( + <> +

+ Components +

+

+ Build beautiful voice experiences with our components. +

+ +
+ {componentNames + .sort((a, b) => a.localeCompare(b)) + .map((componentName) => ( + + {componentName} + + ))} +
+ + ); +} diff --git a/app/ui/layout.tsx b/app/ui/layout.tsx index 5cfb89816..db1a09f4b 100644 --- a/app/ui/layout.tsx +++ b/app/ui/layout.tsx @@ -1,44 +1,69 @@ import { headers } from 'next/headers'; +import Link from 'next/link'; import { ConnectionProvider } from '@/hooks/useConnection'; import { getAppConfig } from '@/lib/utils'; -interface LayoutProps { - children: React.ReactNode; -} - -export default async function Layout({ children }: LayoutProps) { +export default async function Layout({ children }: { children: React.ReactNode }) { const hdrs = await headers(); const appConfig = await getAppConfig(hdrs); return ( -
-
-
-

LiveKit UI

-

- A set of UI Layouts for building LiveKit-powered voice experiences. -

-

- Built with{' '} - - Shadcn - - ,{' '} - - Motion - - , and{' '} - - LiveKit - - . -

-

Open Source.

+
+
+
+ + + + + UI + + + Docs + + + Components +
-
{children}
+ {children}
+ +
); diff --git a/app/ui/page.tsx b/app/ui/page.tsx deleted file mode 100644 index 83e1a7ba4..000000000 --- a/app/ui/page.tsx +++ /dev/null @@ -1,268 +0,0 @@ -import { type VariantProps } from 'class-variance-authority'; -import { Track } from 'livekit-client'; -import { MicrophoneIcon } from '@phosphor-icons/react/dist/ssr'; -import { AgentControlBar } from '@/components/livekit/agent-control-bar/agent-control-bar'; -import { TrackDeviceSelect } from '@/components/livekit/agent-control-bar/track-device-select'; -import { TrackSelector } from '@/components/livekit/agent-control-bar/track-selector'; -import { TrackToggle } from '@/components/livekit/agent-control-bar/track-toggle'; -import { Alert, AlertDescription, AlertTitle, alertVariants } from '@/components/livekit/alert'; -import { AlertToast } from '@/components/livekit/alert-toast'; -import { Button, buttonVariants } from '@/components/livekit/button'; -import { ChatEntry } from '@/components/livekit/chat-entry'; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from '@/components/livekit/select'; -import { ShimmerText } from '@/components/livekit/shimmer-text'; -import { Toggle, toggleVariants } from '@/components/livekit/toggle'; -import { cn } from '@/lib/utils'; - -type toggleVariantsType = VariantProps['variant']; -type toggleVariantsSizeType = VariantProps['size']; -type buttonVariantsType = VariantProps['variant']; -type buttonVariantsSizeType = VariantProps['size']; -type alertVariantsType = VariantProps['variant']; - -interface ContainerProps { - componentName?: string; - children: React.ReactNode; - className?: string; -} - -function Container({ componentName, children, className }: ContainerProps) { - return ( -
-

- {componentName} -

-
- {children} -
-
- ); -} - -function StoryTitle({ children }: { children: React.ReactNode }) { - return

{children}

; -} - -export default function Base() { - return ( - <> -

Primitives

- - {/* Button */} - - - - - - - - - - - - - {['default', 'primary', 'secondary', 'outline', 'ghost', 'link', 'destructive'].map( - (variant) => ( - - - {['sm', 'default', 'lg', 'icon'].map((size) => ( - - ))} - - ) - )} - -
SmallDefaultLargeIcon
{variant} - -
- - - {/* Toggle */} - - - - - - - - - - - - - {['default', 'primary', 'secondary', 'outline'].map((variant) => ( - - - {['sm', 'default', 'lg', 'icon'].map((size) => ( - - ))} - - ))} - -
SmallDefaultLargeIcon
{variant} - - {size === 'icon' ? : 'Toggle'} - -
- - - {/* Alert */} - - {['default', 'destructive'].map((variant) => ( -
- {variant} - - Alert {variant} title - This is a {variant} alert description. - -
- ))} - - - {/* Select */} - -
-
- Size default - -
-
- Size sm - -
-
- - -

Components

- - {/* Agent control bar */} - -
- -
- - - {/* Track device select */} - -
-
- Size default - -
-
- Size sm - -
-
- - - {/* Track toggle */} - -
-
- Track.Source.Microphone - -
-
- Track.Source.Camera - -
-
- - - {/* Track selector */} - -
-
- Track.Source.Camera - -
-
- Track.Source.Microphone - -
-
- - - {/* Chat entry */} - -
- - -
- - - {/* Shimmer text */} - -
- This is shimmer text -
- - - {/* Alert toast */} - - Alert toast -
- -
- - - ); -} diff --git a/components/app/chat-transcript.tsx b/components/app/chat-transcript.tsx index 520f955c5..e2b197186 100644 --- a/components/app/chat-transcript.tsx +++ b/components/app/chat-transcript.tsx @@ -1,8 +1,9 @@ 'use client'; import { AnimatePresence, type HTMLMotionProps, motion } from 'motion/react'; -import { type ReceivedMessage } from '@livekit/components-react'; +import { type ReceivedMessage, useVoiceAssistant } from '@livekit/components-react'; import { ChatEntry } from '@/components/livekit/chat-entry'; +import { ChatIndicator } from '@/components/livekit/chat-indicator'; const MotionContainer = motion.create('div'); const MotionChatEntry = motion.create(ChatEntry); @@ -24,7 +25,6 @@ const CONTAINER_MOTION_PROPS = { delay: 0.2, ease: 'easeOut', duration: 0.3, - stagerDelay: 0.2, staggerChildren: 0.1, staggerDirection: 1, }, @@ -58,31 +58,35 @@ export function ChatTranscript({ messages = [], ...props }: ChatTranscriptProps & Omit, 'ref'>) { + const { state: agentState } = useVoiceAssistant(); return ( - - {!hidden && ( - - {messages.map((receivedMessage) => { - const { id, timestamp, from, message } = receivedMessage; - const locale = navigator?.language ?? 'en-US'; - const messageOrigin = from?.isLocal ? 'local' : 'remote'; - const hasBeenEdited = - receivedMessage.type === 'chatMessage' && !!receivedMessage.editTimestamp; + <> + + {!hidden && ( + + {messages.map((receivedMessage) => { + const { id, timestamp, from, message } = receivedMessage; + const locale = navigator?.language ?? 'en-US'; + const messageOrigin = from?.isLocal ? 'local' : 'remote'; + const hasBeenEdited = + receivedMessage.type === 'chatMessage' && !!receivedMessage.editTimestamp; - return ( - - ); - })} - - )} - + return ( + + ); + })} + + + )} + + ); } diff --git a/components/app/session-view.tsx b/components/app/session-view.tsx index 380295648..5bfa85e38 100644 --- a/components/app/session-view.tsx +++ b/components/app/session-view.tsx @@ -108,7 +108,7 @@ export const SessionView = ({
{/* Tile Layout */} - + {/* Bottom */} ) { + return {children}; +} diff --git a/components/app/theme-toggle.tsx b/components/app/theme-toggle.tsx index ffefc0da1..43d1cd59a 100644 --- a/components/app/theme-toggle.tsx +++ b/components/app/theme-toggle.tsx @@ -1,67 +1,15 @@ 'use client'; -import { useEffect, useState } from 'react'; +import { useTheme } from 'next-themes'; import { MonitorIcon, MoonIcon, SunIcon } from '@phosphor-icons/react'; -import { THEME_MEDIA_QUERY, THEME_STORAGE_KEY, cn } from '@/lib/utils'; - -const THEME_SCRIPT = ` - const doc = document.documentElement; - const theme = localStorage.getItem("${THEME_STORAGE_KEY}") ?? "system"; - - if (theme === "system") { - if (window.matchMedia("${THEME_MEDIA_QUERY}").matches) { - doc.classList.add("dark"); - } else { - doc.classList.add("light"); - } - } else { - doc.classList.add(theme); - } -` - .trim() - .replace(/\n/g, '') - .replace(/\s+/g, ' '); - -export type ThemeMode = 'dark' | 'light' | 'system'; - -function applyTheme(theme: ThemeMode) { - const doc = document.documentElement; - - doc.classList.remove('dark', 'light'); - localStorage.setItem(THEME_STORAGE_KEY, theme); - - if (theme === 'system') { - if (window.matchMedia(THEME_MEDIA_QUERY).matches) { - doc.classList.add('dark'); - } else { - doc.classList.add('light'); - } - } else { - doc.classList.add(theme); - } -} +import { cn } from '@/lib/utils'; interface ThemeToggleProps { className?: string; } -export function ApplyThemeScript() { - return ; -} - export function ThemeToggle({ className }: ThemeToggleProps) { - const [theme, setTheme] = useState(undefined); - - useEffect(() => { - const storedTheme = (localStorage.getItem(THEME_STORAGE_KEY) as ThemeMode) ?? 'system'; - - setTheme(storedTheme); - }, []); - - function handleThemeChange(theme: ThemeMode) { - applyTheme(theme); - setTheme(theme); - } + const { theme, setTheme } = useTheme(); return (
Color scheme toggle -
); diff --git a/components/app/tile-layout.tsx b/components/app/tile-layout.tsx index 333722762..aa52f8828 100644 --- a/components/app/tile-layout.tsx +++ b/components/app/tile-layout.tsx @@ -1,15 +1,21 @@ import React, { useMemo } from 'react'; +import { useTheme } from 'next-themes'; import { Track } from 'livekit-client'; import { AnimatePresence, motion } from 'motion/react'; import { - BarVisualizer, type TrackReference, VideoTrack, useLocalParticipant, useTracks, useVoiceAssistant, } from '@livekit/components-react'; +import { AppConfig } from '@/app-config'; +import { AudioBarVisualizer } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer'; +import { AudioShaderVisualizer } from '@/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer'; import { cn } from '@/lib/utils'; +import { AudioGridVisualizer } from '../livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer'; +import { AudioOscilloscopeVisualizer } from '../livekit/audio-visualizer/audio-oscilloscope-visualizer/audio-oscilloscope-visualizer'; +import { AudioRadialVisualizer } from '../livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer'; const MotionContainer = motion.create('div'); @@ -71,9 +77,11 @@ export function useLocalTrackRef(source: Track.Source) { interface TileLayoutProps { chatOpen: boolean; + appConfig: AppConfig; } -export function TileLayout({ chatOpen }: TileLayoutProps) { +export function TileLayout({ chatOpen, appConfig }: TileLayoutProps) { + const { theme } = useTheme(); const { state: agentState, audioTrack: agentAudioTrack, @@ -112,36 +120,77 @@ export function TileLayout({ chatOpen }: TileLayoutProps) { layoutId="agent" initial={{ opacity: 0, - scale: 0, + scale: chatOpen ? 1 : 6, }} animate={{ opacity: 1, - scale: chatOpen ? 1 : 5, + scale: chatOpen ? 1 : 6, }} transition={{ ...ANIMATION_TRANSITION, delay: animationDelay, }} className={cn( - 'bg-background aspect-square h-[90px] rounded-md border border-transparent transition-[border,drop-shadow]', + 'bg-background flex aspect-square h-[90px] items-center justify-center rounded-md border border-transparent transition-[border,drop-shadow]', chatOpen && 'border-input/50 drop-shadow-lg/10 delay-200' )} > - - + ))} + {appConfig.audioVisualizer === 'radial' && ( + + )} + {appConfig.audioVisualizer === 'grid' && ( + + )} + {appConfig.audioVisualizer === 'aura' && ( + + )} + {appConfig.audioVisualizer === 'wave' && ( + - + )} )} diff --git a/components/demos/AgentControlBar.tsx b/components/demos/AgentControlBar.tsx new file mode 100644 index 000000000..179df920b --- /dev/null +++ b/components/demos/AgentControlBar.tsx @@ -0,0 +1,24 @@ +import { useMicrophone } from '@/hooks/useMicrophone'; +import { Container } from '../docs/container'; +import { AgentControlBar } from '../livekit/agent-control-bar/agent-control-bar'; + +export default function AgentControlBarDemo() { + useMicrophone(); + + return ( + +
+ +
+ + ); +} diff --git a/components/demos/Alert.tsx b/components/demos/Alert.tsx new file mode 100644 index 000000000..1b5435a0b --- /dev/null +++ b/components/demos/Alert.tsx @@ -0,0 +1,22 @@ +import { type VariantProps } from 'class-variance-authority'; +import { Container } from '@/components/docs/container'; +import { StoryTitle } from '@/components/docs/story-title'; +import { Alert, AlertDescription, AlertTitle, alertVariants } from '@/components/livekit/alert'; + +type alertVariantsType = VariantProps['variant']; + +export default function AlertDemo() { + return ( + + {['default', 'destructive'].map((variant) => ( +
+ {variant} + + Alert {variant} title + This is a {variant} alert description. + +
+ ))} + + ); +} diff --git a/components/demos/AlertToast.tsx b/components/demos/AlertToast.tsx new file mode 100644 index 000000000..a0523af8f --- /dev/null +++ b/components/demos/AlertToast.tsx @@ -0,0 +1,18 @@ +import { Container } from '../docs/container'; +import { StoryTitle } from '../docs/story-title'; +import { AlertToast } from '../livekit/alert-toast'; + +export default function AlertToastDemo() { + return ( + + Alert toast +
+ +
+ + ); +} diff --git a/components/demos/AudioBarVisualizer.tsx b/components/demos/AudioBarVisualizer.tsx new file mode 100644 index 000000000..7e77237ca --- /dev/null +++ b/components/demos/AudioBarVisualizer.tsx @@ -0,0 +1,144 @@ +import { useMemo, useState } from 'react'; +import { type VariantProps } from 'class-variance-authority'; +import { Track } from 'livekit-client'; +import { + type AgentState, + type TrackReference, + type TrackReferenceOrPlaceholder, + useLocalParticipant, +} from '@livekit/components-react'; +import { Container } from '@/components/docs/container'; +import { + AudioBarVisualizer, + audioBarVisualizerVariants, +} from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer'; +import { Button } from '@/components/livekit/button'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/livekit/select'; +import { useMicrophone } from '@/hooks/useMicrophone'; + +type audioBarVisualizerVariantsSizeType = VariantProps['size']; + +export default function AudioBarVisualizerDemo() { + const barCounts = ['0', '3', '5', '7', '9']; + const sizes = ['icon', 'sm', 'md', 'lg', 'xl']; + const states = [ + 'disconnected', + 'connecting', + 'initializing', + 'listening', + 'thinking', + 'speaking', + ] as AgentState[]; + + const { microphoneTrack, localParticipant } = useLocalParticipant(); + const [barCount, setBarCount] = useState(barCounts[0]); + const [size, setSize] = useState( + 'md' as audioBarVisualizerVariantsSizeType + ); + const [state, setState] = useState(states[0]); + + const micTrackRef = useMemo(() => { + return state === 'speaking' + ? ({ + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference) + : undefined; + }, [state, localParticipant, microphoneTrack]); + + useMicrophone(); + + return ( + +
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+ +
+ {/*
+ + Original BarVisualizer + +
+ +
+
*/} +
+ +
+ {states.map((stateType) => ( + + ))} +
+ + ); +} diff --git a/components/demos/AudioGridVisualizer.tsx b/components/demos/AudioGridVisualizer.tsx new file mode 100644 index 000000000..940f1d13d --- /dev/null +++ b/components/demos/AudioGridVisualizer.tsx @@ -0,0 +1,154 @@ +import { useMemo, useState } from 'react'; +import { Track } from 'livekit-client'; +import { useLocalParticipant } from '@livekit/components-react'; +import { + type AgentState, + type TrackReference, + type TrackReferenceOrPlaceholder, +} from '@livekit/components-react'; +import { Container } from '@/components/docs/container'; +import { StoryTitle } from '@/components/docs/story-title'; +import { AudioGridVisualizer } from '@/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer'; +import { type GridOptions } from '@/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer'; +import { gridVariants } from '@/components/livekit/audio-visualizer/audio-grid-visualizer/demos'; +import { Button } from '@/components/livekit/button'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/livekit/select'; +import { useMicrophone } from '@/hooks/useMicrophone'; + +export default function AudioGridVisualizerDemo() { + const rowCounts = ['3', '5', '7', '9', '11', '13', '15']; + const columnCounts = ['3', '5', '7', '9', '11', '13', '15']; + const states = [ + 'disconnected', + 'connecting', + 'initializing', + 'listening', + 'thinking', + 'speaking', + ] as AgentState[]; + + const [rowCount, setRowCount] = useState(rowCounts[0]); + const [columnCount, setColumnCount] = useState(columnCounts[0]); + const [state, setState] = useState(states[0]); + const [demoIndex, setDemoIndex] = useState(0); + const { microphoneTrack, localParticipant } = useLocalParticipant(); + + const micTrackRef = useMemo(() => { + return state === 'speaking' + ? ({ + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference) + : undefined; + }, [state, localParticipant, microphoneTrack]); + + useMicrophone(); + + const demoOptions = { + rowCount: parseInt(rowCount), + columnCount: parseInt(columnCount), + ...gridVariants[demoIndex], + }; + + return ( + +
+
+ + +
+ +
+ + +
+ +
+ + +
+
+ +
+ +
+ +
+ {states.map((stateType) => ( + + ))} +
+ +
+ Demo options +
+ 
+            {JSON.stringify(demoOptions, null, 2)}
+
+
+
+ + ); +} diff --git a/components/demos/AudioOscilloscopeVisualizer.tsx b/components/demos/AudioOscilloscopeVisualizer.tsx new file mode 100644 index 000000000..851c5d29a --- /dev/null +++ b/components/demos/AudioOscilloscopeVisualizer.tsx @@ -0,0 +1,206 @@ +import { useMemo, useState } from 'react'; +import { type VariantProps } from 'class-variance-authority'; +import { Track } from 'livekit-client'; +import { + type AgentState, + type TrackReference, + type TrackReferenceOrPlaceholder, +} from '@livekit/components-react'; +import { useLocalParticipant } from '@livekit/components-react'; +import { Container } from '@/components/docs/container'; +import { StoryTitle } from '@/components/docs/story-title'; +import { + AudioOscilloscopeVisualizer, + audioOscilloscopeVisualizerVariants, +} from '@/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/audio-oscilloscope-visualizer'; +import { Button } from '@/components/livekit/button'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/livekit/select'; +import { useMicrophone } from '@/hooks/useMicrophone'; +import { cn } from '@/lib/utils'; + +type audioOscilloscopeVisualizerVariantsSizeType = VariantProps< + typeof audioOscilloscopeVisualizerVariants +>['size']; + +export default function AudioOscilloscopeVisualizerDemo() { + // line width + const [lineWidth, setLineWidth] = useState(2.5); + // smoothing + const [smoothing, setSmoothing] = useState(0.0); + + const sizes = ['icon', 'sm', 'md', 'lg', 'xl']; + const states = [ + 'disconnected', + 'connecting', + 'initializing', + 'listening', + 'thinking', + 'speaking', + ] as AgentState[]; + const colors: [number, number, number][] = [ + [31.0 / 255, 213.0 / 255, 249.0 / 255], // LiveKit Blue + [0.0, 0.0, 1.0], // Blue + [0.0, 1.0, 0.0], // Green + [1.0, 0.0, 0.0], // Red + [1.0, 0.0, 1.0], // Purple + ]; + + const [rgbColor, setRgbColor] = useState<[number, number, number]>(colors[0]); + + const [size, setSize] = useState('xl'); + const [state, setState] = useState(states[1]); + + const { microphoneTrack, localParticipant } = useLocalParticipant(); + const micTrackRef = useMemo(() => { + return state === 'speaking' + ? ({ + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference) + : undefined; + }, [state, localParticipant, microphoneTrack]); + + useMicrophone(); + + const handleColorChange = (e: React.ChangeEvent) => { + const hexColor = e.target.value; + + try { + const rgbColor = hexColor.match(/^#([0-9a-fA-F]{2})([0-9a-fA-F]{2})([0-9a-fA-F]{2})$/); + + if (rgbColor) { + const [, r, g, b] = rgbColor; + const color = [r, g, b].map((c) => parseInt(c, 16) / 255); + + setRgbColor(color as [number, number, number]); + } + } catch (error) { + console.error(error); + } + }; + + const fields = [ + ['line width', lineWidth, setLineWidth, 1, 10, 1], + ['smoothing', smoothing, setSmoothing, 0, 10, 0.1], + ] as const; + + return ( + +
+
+ + +
+
+ +
+ +
+ +
+ {states.map((stateType) => ( + + ))} +
+ +
+
+ Color +
+ {colors.map((color) => ( +
setRgbColor(color)} + style={{ backgroundColor: `rgb(${color.map((c) => c * 255).join(',')})` }} + className={cn( + 'h-4 w-4 cursor-pointer rounded-full', + rgbColor.toString() === color.toString() && + 'ring-muted-foreground ring-offset-background ring-1 ring-offset-2' + )} + /> + ))} + + +
+
+ +
+ {fields.map(([name, value, setValue, min = 0.1, max = 10, step = 0.1]) => { + return ( +
+
+ {name} +
{String(value)}
+
+ setValue(parseFloat(e.target.value))} + className="w-full" + /> +
+ ); + })} +
+
+ + ); +} diff --git a/components/demos/AudioRadialVisualizer.tsx b/components/demos/AudioRadialVisualizer.tsx new file mode 100644 index 000000000..b3b03b474 --- /dev/null +++ b/components/demos/AudioRadialVisualizer.tsx @@ -0,0 +1,130 @@ +import { useMemo, useState } from 'react'; +import { type VariantProps } from 'class-variance-authority'; +import { Track } from 'livekit-client'; +import { useLocalParticipant } from '@livekit/components-react'; +import { + type AgentState, + type TrackReference, + type TrackReferenceOrPlaceholder, +} from '@livekit/components-react'; +import { Container } from '@/components/docs/container'; +import { + AudioRadialVisualizer, + audioRadialVisualizerVariants, +} from '@/components/livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer'; +import { Button } from '@/components/livekit/button'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/livekit/select'; +import { useMicrophone } from '@/hooks/useMicrophone'; + +type audioRadialVisualizerVariantsSizeType = VariantProps< + typeof audioRadialVisualizerVariants +>['size']; + +export default function AudioRadialVisualizerDemo() { + const barCounts = ['0', '4', '8', '12', '16', '24']; + const sizes = ['icon', 'sm', 'md', 'lg', 'xl']; + const states = [ + 'disconnected', + 'connecting', + 'initializing', + 'listening', + 'thinking', + 'speaking', + ] as AgentState[]; + + const { microphoneTrack, localParticipant } = useLocalParticipant(); + const [barCount, setBarCount] = useState(barCounts[0]); + const [size, setSize] = useState( + 'md' as audioRadialVisualizerVariantsSizeType + ); + const [state, setState] = useState(states[0]); + + const micTrackRef = useMemo(() => { + return state === 'speaking' + ? ({ + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference) + : undefined; + }, [state, localParticipant, microphoneTrack]); + + useMicrophone(); + + return ( + +
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+
+ +
+ {states.map((stateType) => ( + + ))} +
+ + ); +} diff --git a/components/demos/AudioShaderVisualizer.tsx b/components/demos/AudioShaderVisualizer.tsx new file mode 100644 index 000000000..7ca08ad20 --- /dev/null +++ b/components/demos/AudioShaderVisualizer.tsx @@ -0,0 +1,215 @@ +import { useMemo, useState } from 'react'; +import { type VariantProps } from 'class-variance-authority'; +import { Track } from 'livekit-client'; +import { + type AgentState, + type TrackReference, + type TrackReferenceOrPlaceholder, +} from '@livekit/components-react'; +import { useLocalParticipant } from '@livekit/components-react'; +import { Container } from '@/components/docs/container'; +import { StoryTitle } from '@/components/docs/story-title'; +import { + AudioShaderVisualizer, + audioShaderVisualizerVariants, +} from '@/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer'; +import { Button } from '@/components/livekit/button'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/livekit/select'; +import { useMicrophone } from '@/hooks/useMicrophone'; +import { cn } from '@/lib/utils'; + +type audioShaderVisualizerVariantsSizeType = VariantProps< + typeof audioShaderVisualizerVariants +>['size']; + +export default function AudioShaderVisualizerDemo() { + // shape + const [shape, setShape] = useState(1.0); + // color scale + const [colorShift, setColorShift] = useState(0.3); + + const sizes = ['icon', 'sm', 'md', 'lg', 'xl']; + const states = [ + 'disconnected', + 'connecting', + 'initializing', + 'listening', + 'thinking', + 'speaking', + ] as AgentState[]; + const colors: [number, number, number][] = [ + [31.0 / 255, 213.0 / 255, 249.0 / 255], // LiveKit Blue + [0.0, 0.0, 1.0], // Blue + [0.0, 1.0, 0.0], // Green + [1.0, 0.0, 0.0], // Red + [1.0, 0.0, 1.0], // Purple + ]; + + const [rgbColor, setRgbColor] = useState<[number, number, number]>(colors[0]); + const [size, setSize] = useState('lg'); + const [state, setState] = useState(states[1]); + + const { microphoneTrack, localParticipant } = useLocalParticipant(); + const micTrackRef = useMemo(() => { + return state === 'speaking' + ? ({ + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference) + : undefined; + }, [state, localParticipant, microphoneTrack]); + + useMicrophone(); + + const handleColorChange = (e: React.ChangeEvent) => { + const hexColor = e.target.value; + + try { + const rgbColor = hexColor.match(/^#([0-9a-fA-F]{2})([0-9a-fA-F]{2})([0-9a-fA-F]{2})$/); + + if (rgbColor) { + const [, r, g, b] = rgbColor; + const color = [r, g, b].map((c) => parseInt(c, 16) / 255); + + setRgbColor(color as [number, number, number]); + } + } catch (error) { + console.error(error); + } + }; + + const fields = [['color shift', colorShift, setColorShift, 0, 1, 0.01]] as const; + + return ( + +
+
+ + +
+ +
+ + +
+
+ +
+ +
+ +
+ {states.map((stateType) => ( + + ))} +
+ +
+
+ Color +
+ {colors.map((color) => ( +
setRgbColor(color)} + style={{ backgroundColor: `rgb(${color.map((c) => c * 255).join(',')})` }} + className={cn( + 'h-4 w-4 cursor-pointer rounded-full', + rgbColor.toString() === color.toString() && + 'ring-muted-foreground ring-offset-background ring-1 ring-offset-2' + )} + /> + ))} + + +
+
+ + {fields.map(([name, value, setValue, min = 0.1, max = 10, step = 0.1]) => { + return ( +
+
+ {name} +
{String(value)}
+
+ setValue(parseFloat(e.target.value))} + className="w-full" + /> +
+ ); + })} +
+ + ); +} diff --git a/components/demos/Button.tsx b/components/demos/Button.tsx new file mode 100644 index 000000000..475030bca --- /dev/null +++ b/components/demos/Button.tsx @@ -0,0 +1,44 @@ +import { type VariantProps } from 'class-variance-authority'; +import { MicrophoneIcon } from '@phosphor-icons/react/dist/ssr'; +import { Container } from '@/components/docs/container'; +import { Button, buttonVariants } from '@/components/livekit/button'; + +type buttonVariantsSizeType = VariantProps['size']; +type buttonVariantsType = VariantProps['variant']; + +export default function ButtonDemo() { + return ( + + + + + + + + + + + + + {['default', 'primary', 'secondary', 'outline', 'ghost', 'link', 'destructive'].map( + (variant) => ( + + + {['sm', 'default', 'lg', 'icon'].map((size) => ( + + ))} + + ) + )} + +
SmallDefaultLargeIcon
{variant} + +
+ + ); +} diff --git a/components/demos/ChatEntry.tsx b/components/demos/ChatEntry.tsx new file mode 100644 index 000000000..5cbdae862 --- /dev/null +++ b/components/demos/ChatEntry.tsx @@ -0,0 +1,25 @@ +import { Container } from '../docs/container'; +import { ChatEntry } from '../livekit/chat-entry'; + +export default function ChatEntryDemo() { + return ( + +
+ + +
+ + ); +} diff --git a/components/demos/ChatIndicator.tsx b/components/demos/ChatIndicator.tsx new file mode 100644 index 000000000..6b0331235 --- /dev/null +++ b/components/demos/ChatIndicator.tsx @@ -0,0 +1,10 @@ +import { Container } from '../docs/container'; +import { ChatIndicator } from '../livekit/chat-indicator'; + +export default function ChatIndicatorDemo() { + return ( + + + + ); +} diff --git a/components/demos/Select.tsx b/components/demos/Select.tsx new file mode 100644 index 000000000..5ee77223b --- /dev/null +++ b/components/demos/Select.tsx @@ -0,0 +1,44 @@ +import { Container } from '@/components/docs/container'; +import { StoryTitle } from '@/components/docs/story-title'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/livekit/select'; + +export default function SelectDemo() { + return ( + +
+
+ Size default + +
+
+ Size sm + +
+
+ + ); +} diff --git a/components/demos/ShimmerText.tsx b/components/demos/ShimmerText.tsx new file mode 100644 index 000000000..6cee31223 --- /dev/null +++ b/components/demos/ShimmerText.tsx @@ -0,0 +1,12 @@ +import { Container } from '../docs/container'; +import { ShimmerText } from '../livekit/shimmer-text'; + +export default function ShimmerTextDemo() { + return ( + +
+ This is shimmer text +
+ + ); +} diff --git a/components/demos/Toggle.tsx b/components/demos/Toggle.tsx new file mode 100644 index 000000000..c292cdb2c --- /dev/null +++ b/components/demos/Toggle.tsx @@ -0,0 +1,42 @@ +import { type VariantProps } from 'class-variance-authority'; +import { MicrophoneIcon } from '@phosphor-icons/react/dist/ssr'; +import { Container } from '@/components/docs/container'; +import { Toggle, toggleVariants } from '@/components/livekit/toggle'; + +type toggleVariantsSizeType = VariantProps['size']; +type toggleVariantsType = VariantProps['variant']; + +export default function ToggleDemo() { + return ( + + + + + + + + + + + + + {['default', 'primary', 'secondary', 'outline'].map((variant) => ( + + + {['sm', 'default', 'lg', 'icon'].map((size) => ( + + ))} + + ))} + +
SmallDefaultLargeIcon
{variant} + + {size === 'icon' ? : 'Toggle'} + +
+ + ); +} diff --git a/components/demos/TrackControl.tsx b/components/demos/TrackControl.tsx new file mode 100644 index 000000000..fd5b5bb58 --- /dev/null +++ b/components/demos/TrackControl.tsx @@ -0,0 +1,50 @@ +import { useMemo } from 'react'; +import { Track } from 'livekit-client'; +import { + type TrackReference, + type TrackReferenceOrPlaceholder, + useLocalParticipant, +} from '@livekit/components-react'; +import { Container } from '@/components/docs/container'; +import { StoryTitle } from '@/components/docs/story-title'; +import { TrackControl } from '@/components/livekit/agent-control-bar/track-control'; +import { useMicrophone } from '@/hooks/useMicrophone'; + +export default function TrackControlDemo() { + const { microphoneTrack, localParticipant } = useLocalParticipant(); + const micTrackRef = useMemo(() => { + return { + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference; + }, [localParticipant, microphoneTrack]); + + useMicrophone(); + + return ( + +
+
+
+ Track.Source.Microphone + +
+
+ Track.Source.Microphone + +
+
+ +
+ Track.Source.Camera + +
+
+ + ); +} diff --git a/components/docs/container.tsx b/components/docs/container.tsx new file mode 100644 index 000000000..da1ee7752 --- /dev/null +++ b/components/docs/container.tsx @@ -0,0 +1,15 @@ +interface ContainerProps { + componentName: string; + children: React.ReactNode; + className?: string; +} + +export function Container({ children, className }: ContainerProps) { + return ( +
+
+ {children} +
+
+ ); +} diff --git a/components/docs/side-nav.tsx b/components/docs/side-nav.tsx new file mode 100644 index 000000000..591e8de33 --- /dev/null +++ b/components/docs/side-nav.tsx @@ -0,0 +1,45 @@ +'use client'; + +import { usePathname } from 'next/navigation'; +import Link from 'next/link'; +import { cn } from '@/lib/utils'; + +interface SideNavProps { + + componentNames: string[]; +} + +export function SideNav({ componentNames }: SideNavProps) { + const pathname = usePathname(); + const isActive = (componentName: string) => pathname.endsWith(componentName); + + return ( + <> +

+ + Components + +

+ {[...componentNames] + .sort((a, b) => a.localeCompare(b)) + .map((componentName) => ( + + {componentName} + + ))} + + ); +} \ No newline at end of file diff --git a/components/docs/story-title.tsx b/components/docs/story-title.tsx new file mode 100644 index 000000000..8b4144a24 --- /dev/null +++ b/components/docs/story-title.tsx @@ -0,0 +1,7 @@ +interface StoryTitleProps { + children: React.ReactNode; +} + +export function StoryTitle({ children }: StoryTitleProps) { + return

{children}

; +} diff --git a/components/livekit/agent-control-bar/agent-control-bar.tsx b/components/livekit/agent-control-bar/agent-control-bar.tsx index 52b99b59a..489c7c10e 100644 --- a/components/livekit/agent-control-bar/agent-control-bar.tsx +++ b/components/livekit/agent-control-bar/agent-control-bar.tsx @@ -11,7 +11,7 @@ import { cn } from '@/lib/utils'; import { ChatInput } from './chat-input'; import { UseInputControlsProps, useInputControls } from './hooks/use-input-controls'; import { usePublishPermissions } from './hooks/use-publish-permissions'; -import { TrackSelector } from './track-selector'; +import { TrackControl } from './track-control'; export interface ControlBarControls { leave?: boolean; @@ -100,7 +100,7 @@ export function AgentControlBar({
{/* Toggle Microphone */} {visibleControls.microphone && ( - [0]['source']; pressed?: boolean; @@ -22,7 +19,7 @@ interface TrackSelectorProps { onActiveDeviceChange?: (deviceId: string) => void; } -export function TrackSelector({ +export function TrackControl({ kind, source, pressed, @@ -33,7 +30,7 @@ export function TrackSelector({ onPressedChange, onMediaDeviceError, onActiveDeviceChange, -}: TrackSelectorProps) { +}: TrackControlProps) { return (
{audioTrackRef && ( - - - + )}
diff --git a/components/livekit/audio-visualizer/audio-bar-visualizer/_bar-visualizer.tsx b/components/livekit/audio-visualizer/audio-bar-visualizer/_bar-visualizer.tsx new file mode 100644 index 000000000..aa3ab23ba --- /dev/null +++ b/components/livekit/audio-visualizer/audio-bar-visualizer/_bar-visualizer.tsx @@ -0,0 +1,99 @@ +import { useEffect, useMemo, useRef } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { + type AgentState, + BarVisualizer as LiveKitBarVisualizer, + type TrackReferenceOrPlaceholder, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; + +const MIN_HEIGHT = 15; // 15% + +export const barVisualizerVariants = cva( + ['relative flex aspect-square h-36 items-center justify-center'], + { + variants: { + size: { + default: 'h-32', + icon: 'h-6', + xs: 'h-8', + sm: 'h-16', + md: 'h-32', + lg: 'h-64', + xl: 'h-96', + '2xl': 'h-128', + }, + }, + defaultVariants: { + size: 'default', + }, + } +); + +interface BarVisualizerProps { + state?: AgentState; + barCount?: number; + audioTrack?: TrackReferenceOrPlaceholder; + className?: string; +} + +export function BarVisualizer({ + size, + state, + barCount, + audioTrack, + className, +}: BarVisualizerProps & VariantProps) { + const ref = useRef(null); + const _barCount = useMemo(() => { + if (barCount) { + return barCount; + } + switch (size) { + case 'icon': + case 'xs': + return 3; + default: + return 5; + } + }, [barCount, size]); + + const x = (1 / (_barCount + (_barCount + 1) / 2)) * 100; + + // reset bars height when audio track is disconnected + useEffect(() => { + if (ref.current && !audioTrack) { + const bars = [...(ref.current.querySelectorAll('& > span') ?? [])] as HTMLElement[]; + + bars.forEach((bar) => { + bar.style.height = `${MIN_HEIGHT}%`; + }); + } + }, [audioTrack]); + + return ( + + + + ); +} diff --git a/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx b/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx new file mode 100644 index 000000000..bd56bd934 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx @@ -0,0 +1,108 @@ +import { type ReactNode, useMemo } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { type LocalAudioTrack, type RemoteAudioTrack } from 'livekit-client'; +import { + type AgentState, + type TrackReferenceOrPlaceholder, + useMultibandTrackVolume, +} from '@livekit/components-react'; +import { cloneSingleChild, cn } from '@/lib/utils'; +import { useBarAnimator } from './hooks/useBarAnimator'; + +export const audioBarVisualizerVariants = cva( + [ + 'relative flex items-center justify-center', + '[&_>_*]:rounded-full [&_>_*]:transition-colors [&_>_*]:duration-250 [&_>_*]:ease-linear', + '[&_>_*]:bg-(--audio-visualizer-idle) [&_>_*]:data-[lk-highlighted=true]:bg-(--audio-visualizer-active)', + ], + { + variants: { + size: { + icon: ['h-[24px] gap-[2px]', '[&_>_*]:w-[4px] [&_>_*]:min-h-[4px]'], + sm: ['h-[56px] gap-[4px]', '[&_>_*]:w-[8px] [&_>_*]:min-h-[8px]'], + md: ['h-[112px] gap-[8px]', '[&_>_*]:w-[16px] [&_>_*]:min-h-[16px]'], + lg: ['h-[224px] gap-[16px]', '[&_>_*]:w-[32px] [&_>_*]:min-h-[32px]'], + xl: ['h-[448px] gap-[32px]', '[&_>_*]:w-[64px] [&_>_*]:min-h-[64px]'], + }, + }, + defaultVariants: { + size: 'md', + }, + } +); + +interface AudioBarVisualizerProps { + state?: AgentState; + barCount?: number; + audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder; + className?: string; + children?: ReactNode | ReactNode[]; +} + +export function AudioBarVisualizer({ + size, + state, + barCount, + audioTrack, + className, + children, +}: AudioBarVisualizerProps & VariantProps) { + const _barCount = useMemo(() => { + if (barCount) { + return barCount; + } + switch (size) { + case 'icon': + case 'sm': + return 3; + default: + return 5; + } + }, [barCount, size]); + + const volumeBands = useMultibandTrackVolume(audioTrack, { + bands: _barCount, + loPass: 100, + hiPass: 200, + }); + + const sequencerInterval = useMemo(() => { + switch (state) { + case 'connecting': + return 2000 / _barCount; + case 'initializing': + return 2000; + case 'listening': + return 500; + case 'thinking': + return 150; + default: + return 1000; + } + }, [state, _barCount]); + + const highlightedIndices = useBarAnimator(state, _barCount, sequencerInterval); + const bands = audioTrack ? volumeBands : new Array(_barCount).fill(0); + + return ( +
+ {bands.map((band, idx) => + children ? ( + cloneSingleChild(children, { + key: idx, + 'data-lk-index': idx, + 'data-lk-highlighted': highlightedIndices.includes(idx), + style: { height: `${band * 100}%` }, + }) + ) : ( +
+ ) + )} +
+ ); +} diff --git a/components/livekit/audio-visualizer/audio-bar-visualizer/hooks/useBarAnimator.ts b/components/livekit/audio-visualizer/audio-bar-visualizer/hooks/useBarAnimator.ts new file mode 100644 index 000000000..070bfa0bf --- /dev/null +++ b/components/livekit/audio-visualizer/audio-bar-visualizer/hooks/useBarAnimator.ts @@ -0,0 +1,84 @@ +import { useEffect, useRef, useState } from 'react'; +import { type AgentState } from '@livekit/components-react'; + +function generateConnectingSequenceBar(columns: number): number[][] { + const seq = []; + + for (let x = 0; x < columns; x++) { + seq.push([x, columns - 1 - x]); + } + + return seq; +} + +function generateListeningSequenceBar(columns: number): number[][] { + const center = Math.floor(columns / 2); + const noIndex = -1; + + return [[center], [noIndex]]; +} + +// function generateThinkingSequenceBar(columns: number): number[][] { +// const seq = []; +// for (let x = 0; x < columns; x++) { +// seq.push([x]); +// } + +// for (let x = columns - 1; x >= 0; x--) { +// seq.push([x]); +// } + +// return seq; +// } + +export const useBarAnimator = ( + state: AgentState | undefined, + columns: number, + interval: number +): number[] => { + const [index, setIndex] = useState(0); + const [sequence, setSequence] = useState([[]]); + + useEffect(() => { + if (state === 'thinking') { + setSequence(generateListeningSequenceBar(columns)); + // setSequence(generateThinkingSequenceBar(columns)); + } else if (state === 'connecting' || state === 'initializing') { + const sequence = [...generateConnectingSequenceBar(columns)]; + setSequence(sequence); + } else if (state === 'listening') { + setSequence(generateListeningSequenceBar(columns)); + } else if (state === undefined || state === 'speaking') { + setSequence([new Array(columns).fill(0).map((_, idx) => idx)]); + } else { + setSequence([[]]); + } + setIndex(0); + }, [state, columns]); + + const animationFrameId = useRef(null); + useEffect(() => { + let startTime = performance.now(); + + const animate = (time: DOMHighResTimeStamp) => { + const timeElapsed = time - startTime; + + if (timeElapsed >= interval) { + setIndex((prev) => prev + 1); + startTime = time; + } + + animationFrameId.current = requestAnimationFrame(animate); + }; + + animationFrameId.current = requestAnimationFrame(animate); + + return () => { + if (animationFrameId.current !== null) { + cancelAnimationFrame(animationFrameId.current); + } + }; + }, [interval, columns, state, sequence.length]); + + return sequence[index % sequence.length]; +}; diff --git a/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer.tsx b/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer.tsx new file mode 100644 index 000000000..e3e11c114 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer.tsx @@ -0,0 +1,175 @@ +import { CSSProperties, type ReactNode, memo, useMemo } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { LocalAudioTrack, RemoteAudioTrack } from 'livekit-client'; +import { + type AgentState, + type TrackReferenceOrPlaceholder, + useMultibandTrackVolume, +} from '@livekit/components-react'; +import { cloneSingleChild, cn } from '@/lib/utils'; +import { type Coordinate, useGridAnimator } from './hooks/useGridAnimator'; + +export const audioGridVisualizerVariants = cva( + [ + 'grid', + '[&_>_*]:size-1 [&_>_*]:rounded-full', + '[&_>_*]:bg-foreground/10 [&_>_[data-lk-highlighted=true]]:bg-foreground [&_>_[data-lk-highlighted=true]]:scale-125 [&_>_[data-lk-highlighted=true]]:shadow-[0px_0px_10px_2px_rgba(255,255,255,0.4)]', + ], + { + variants: { + size: { + icon: ['gap-[2px] [&_>_*]:size-[4px]'], + sm: ['gap-[4px] [&_>_*]:size-[4px]'], + md: ['gap-[8px] [&_>_*]:size-[8px]'], + lg: ['gap-[8px] [&_>_*]:size-[8px]'], + xl: ['gap-[8px] [&_>_*]:size-[8px]'], + }, + }, + defaultVariants: { + size: 'md', + }, + } +); + +export interface GridOptions { + radius?: number; + interval?: number; + rowCount?: number; + columnCount?: number; + transformer?: (index: number, rowCount: number, columnCount: number) => CSSProperties; + className?: string; + children?: ReactNode; +} + +const sizeDefaults = { + icon: 3, + sm: 5, + md: 5, + lg: 5, + xl: 5, +}; + +function useGrid( + size: VariantProps['size'] = 'md', + columnCount = sizeDefaults[size as keyof typeof sizeDefaults], + rowCount = sizeDefaults[size as keyof typeof sizeDefaults] +) { + return useMemo(() => { + const _columnCount = columnCount; + const _rowCount = rowCount ?? columnCount; + const items = new Array(_columnCount * _rowCount).fill(0).map((_, idx) => idx); + + return { columnCount: _columnCount, rowCount: _rowCount, items }; + }, [columnCount, rowCount]); +} + +interface GridCellProps { + index: number; + state: AgentState; + interval: number; + transformer?: (index: number, rowCount: number, columnCount: number) => CSSProperties; + rowCount: number; + columnCount: number; + volumeBands: number[]; + highlightedCoordinate: Coordinate; + children: ReactNode; +} + +const GridCell = memo(function GridCell({ + index, + state, + interval, + transformer, + rowCount, + columnCount, + volumeBands, + highlightedCoordinate, + children, +}: GridCellProps) { + if (state === 'speaking') { + const y = Math.floor(index / columnCount); + const rowMidPoint = Math.floor(rowCount / 2); + const volumeChunks = 1 / (rowMidPoint + 1); + const distanceToMid = Math.abs(rowMidPoint - y); + const threshold = distanceToMid * volumeChunks; + const isHighlighted = volumeBands[index % columnCount] >= threshold; + + return cloneSingleChild(children, { + 'data-lk-index': index, + 'data-lk-highlighted': isHighlighted, + }); + } + + let transformerStyle: CSSProperties | undefined; + if (transformer) { + transformerStyle = transformer(index, rowCount, columnCount); + } + + const isHighlighted = + highlightedCoordinate.x === index % columnCount && + highlightedCoordinate.y === Math.floor(index / columnCount); + + const transitionDurationInSeconds = interval / (isHighlighted ? 1000 : 100); + + return cloneSingleChild(children, { + 'data-lk-index': index, + 'data-lk-highlighted': isHighlighted, + style: { + transitionProperty: 'all', + transitionDuration: `${transitionDurationInSeconds}s`, + transitionTimingFunction: 'ease-out', + ...transformerStyle, + }, + }); +}); + +export type AudioGridVisualizerProps = GridOptions & { + state: AgentState; + audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder; + className?: string; + children?: ReactNode; +} & VariantProps; + +export function AudioGridVisualizer({ + size = 'md', + state, + radius, + rowCount: _rowCount = 5, + columnCount: _columnCount = 5, + transformer, + interval = 100, + className, + children, + audioTrack, +}: AudioGridVisualizerProps) { + const { columnCount, rowCount, items } = useGrid(size, _columnCount, _rowCount); + const highlightedCoordinate = useGridAnimator(state, rowCount, columnCount, interval, radius); + const volumeBands = useMultibandTrackVolume(audioTrack, { + bands: columnCount, + loPass: 100, + hiPass: 200, + }); + + return ( +
+ {items.map((idx) => ( + + {children ??
} + + ))} +
+ ); +} diff --git a/components/livekit/audio-visualizer/audio-grid-visualizer/demos.tsx b/components/livekit/audio-visualizer/audio-grid-visualizer/demos.tsx new file mode 100644 index 000000000..16d4ae9ac --- /dev/null +++ b/components/livekit/audio-visualizer/audio-grid-visualizer/demos.tsx @@ -0,0 +1,175 @@ +import { cn } from '@/lib/utils'; +import { GridOptions } from './audio-grid-visualizer'; + +type SVGIconProps = React.SVGProps; + +function SVGIcon({ className, children, ...props }: SVGIconProps) { + return ( + <> + + {children} + + + ); +} + +function EyeSVG(props: SVGIconProps) { + return ( + + + + ); +} + +function PlusSVG(props: SVGIconProps) { + return ( + + + + ); +} + +export const gridVariants: GridOptions[] = [ + // 1 + { + radius: 6, + interval: 75, + className: + 'gap-4 [&_>_*]:size-1 [&_>_*]:rounded-full [&_>_*]:bg-foreground/10 [&_>_[data-lk-highlighted=true]]:bg-foreground [&_>_[data-lk-highlighted=true]]:scale-125 [&_>_[data-lk-highlighted=true]]:shadow-[0px_0px_10px_2px_rgba(255,255,255,0.4)]', + }, + // 2 + { + interval: 50, + className: + 'gap-2 [&_>_*]:w-4 [&_>_*]:h-1 [&_>_*]:bg-foreground/10 [&_>_[data-lk-highlighted=true]]:bg-[#F9B11F] [&_>_[data-lk-highlighted=true]]:shadow-[0px_0px_14.8px_2px_#F9B11F]', + }, + // 3 + { + interval: 50, + className: + 'gap-2 [&_>_*]:size-2 [&_>_*]:rounded-full [&_>_*]:bg-foreground/10 [&_>_[data-lk-highlighted=true]]:bg-[#1FD5F9] [&_>_[data-lk-highlighted=true]]:shadow-[0px_0px_8px_3px_rgba(31,213,249,0.44)]', + }, + // 4 + { + interval: 50, + className: + 'gap-4 [&_>_*]:size-4 [&_>_*]:rounded-full [&_>_*]:bg-foreground/5 [&_>_[data-lk-highlighted=true]]:bg-[#FF6352] [&_>_[data-lk-highlighted=true]]:shadow-[0px_0px_32px_8px_rgba(255,99,82,0.8)]', + }, + // 5 + { + interval: 50, + className: + 'gap-4 [&_>_*]:size-2 [&_>_*]:rounded-full [&_>_*]:bg-foreground/10 [&_>_[data-lk-highlighted=true]]:bg-[#1F8CF9] [&_>_[data-lk-highlighted=true]]:shadow-[0px_0px_14.8px_2px_#1F8CF9]', + transformer: (index: number, rowCount: number, columnCount: number) => { + const rowMidPoint = Math.floor(rowCount / 2); + const distanceFromCenter = Math.sqrt( + Math.pow(rowMidPoint - (index % columnCount), 2) + + Math.pow(rowMidPoint - Math.floor(index / columnCount), 2) + ); + + return { + opacity: 1 - distanceFromCenter / columnCount, + transform: `scale(${1 - (distanceFromCenter / (columnCount * 2)) * 1.75})`, + }; + }, + }, + // 6 + { + radius: 2, + interval: 150, + className: + 'gap-2 [&_>_*]:size-2 [&_>_*]:bg-foreground/8 [&_>_[data-lk-highlighted=true]]:bg-[#F91F8C] [&_>_[data-lk-highlighted=true]]:shadow-[0px_0px_14.8px_4px_#F91F8C] [&_>_[data-lk-highlighted=true]]:scale-150', + transformer: (index: number, rowCount: number, columnCount: number) => { + const rowMidPoint = Math.floor(rowCount / 2); + const distanceFromCenter = Math.sqrt( + Math.pow(rowMidPoint - (index % columnCount), 2) + + Math.pow(rowMidPoint - Math.floor(index / columnCount), 2) + ); + + return { + opacity: 0.5 - distanceFromCenter / columnCount, + }; + }, + }, + // 7 + { + interval: 50, + className: + 'gap-4 [&_>_*]:size-2.5 [&_>_*]:rounded-1.5 [&_>_*]:bg-foreground/15 [&_>_[data-lk-highlighted=true]]:bg-[#FFB6C1] [&_>_[data-lk-highlighted=true]]:shadow-[0px_0px_24px_3px_rgba(255,182,193,0.8)]', + }, + // 8 + { + interval: 50, + className: + 'gap-8 [&_>_*]:size-2.5 [&_>_*]:bg-foreground/5 [&_>_[data-lk-highlighted=true]]:bg-[#FFB6C1] [&_>_[data-lk-highlighted=true]]:shadow-[0px_0px_8px_1px_rgba(255,182,193,0.8)]', + transformer: (index: number, rowCount: number, columnCount: number) => { + const rowMidPoint = Math.floor(rowCount / 2); + const distanceFromCenter = Math.sqrt( + Math.pow(rowMidPoint - (index % columnCount), 2) + + Math.pow(rowMidPoint - Math.floor(index / columnCount), 2) + ); + const maxDistance = Math.sqrt(4); // maximum distance in a normalized grid + const scaleFactor = 1 + (maxDistance - distanceFromCenter / maxDistance); + + return { + transform: `scale(${scaleFactor})`, + }; + }, + }, + // 9 + { + radius: 4, + interval: 75, + className: + 'gap-2.5 [&_>_*]:w-1.5 [&_>_*]:h-px [&_>_*]:rotate-45 [&_>_*]:bg-foreground/10 [&_>_[data-lk-highlighted=true]]:bg-foreground [&_>_[data-lk-highlighted=true]]:drop-shadow-[0px_0px_8px_2px_rgba(255,182,193,0.4)] [&_>_[data-lk-highlighted=true]]:rotate-[405deg] [&_>_[data-lk-highlighted=true]]:scale-200', + }, + // 10 + { + radius: 4, + interval: 75, + children: , + className: + 'gap-2 [&_>_*]:size-2 [&_>_*]:rotate-45 [&_>_*]:text-foreground/10 [&_>_[data-lk-highlighted=true]]:text-[#F97A1F] [&_>_[data-lk-highlighted=true]]:drop-shadow-[0px_0px_3.2px_#F97A1F] [&_>_[data-lk-highlighted=true]]:rotate-[405deg] [&_>_[data-lk-highlighted=true]]:scale-250', + }, + // 11 + { + radius: 3, + interval: 75, + children: , + className: + 'gap-2 [&_>_*]:size-2 [&_>_*]:text-foreground/10 [&_>_[data-lk-highlighted=true]]:text-[#B11FF9] [&_>_[data-lk-highlighted=true]]:drop-shadow-[0px_0px_16px_2px_rgba(177,31,249,0.6)] [&_>_[data-lk-highlighted=true]]:scale-150', + transformer: (index: number, rowCount: number, columnCount: number) => { + const rowMidPoint = Math.floor(rowCount / 2); + const distanceFromCenter = Math.sqrt( + Math.pow(rowMidPoint - (index % columnCount), 2) + + Math.pow(rowMidPoint - Math.floor(index / columnCount), 2) + ); + + return { + opacity: 0.5 - distanceFromCenter / columnCount, + }; + }, + }, + // 12 + { + radius: 6, + interval: 75, + className: + 'gap-2.5 [&_>_*]:w-3 [&_>_*]:h-px [&_>_*]:rotate-45 [&_>_*]:bg-foreground/10 [&_>_*]:rotate-45 [&_>_*]:scale-100 [&_>_[data-lk-highlighted=true]]:bg-[#FFB6C1] [&_>_[data-lk-highlighted=true]]:shadow-[0px_0px_8px_2px_rgba(255,182,193,0.4)] [&_>_[data-lk-highlighted=true]]:rotate-[405deg] [&_>_[data-lk-highlighted=true]]:scale-200', + }, +]; diff --git a/components/livekit/audio-visualizer/audio-grid-visualizer/hooks/useGridAnimator.ts b/components/livekit/audio-visualizer/audio-grid-visualizer/hooks/useGridAnimator.ts new file mode 100644 index 000000000..ce9272eba --- /dev/null +++ b/components/livekit/audio-visualizer/audio-grid-visualizer/hooks/useGridAnimator.ts @@ -0,0 +1,115 @@ +import { useEffect, useState } from 'react'; +import { type AgentState } from '@livekit/components-react'; + +export interface Coordinate { + x: number; + y: number; +} + +export function generateConnectingSequence(rows: number, columns: number, radius: number) { + const seq = []; + // const centerX = Math.floor(columns / 2); + const centerY = Math.floor(rows / 2); + + // Calculate the boundaries of the ring based on the ring distance + const topLeft = { + x: Math.max(0, centerY - radius), + y: Math.max(0, centerY - radius), + }; + const bottomRight = { + x: columns - 1 - topLeft.x, + y: Math.min(rows - 1, centerY + radius), + }; + + // Top edge + for (let x = topLeft.x; x <= bottomRight.x; x++) { + seq.push({ x, y: topLeft.y }); + } + + // Right edge + for (let y = topLeft.y + 1; y <= bottomRight.y; y++) { + seq.push({ x: bottomRight.x, y }); + } + + // Bottom edge + for (let x = bottomRight.x - 1; x >= topLeft.x; x--) { + seq.push({ x, y: bottomRight.y }); + } + + // Left edge + for (let y = bottomRight.y - 1; y > topLeft.y; y--) { + seq.push({ x: topLeft.x, y }); + } + + return seq; +} + +export function generateListeningSequence(rows: number, columns: number) { + const center = { x: Math.floor(columns / 2), y: Math.floor(rows / 2) }; + const noIndex = { x: -1, y: -1 }; + + return [center, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex]; +} + +export function generateThinkingSequence(rows: number, columns: number) { + const seq = []; + const y = Math.floor(rows / 2); + for (let x = 0; x < columns; x++) { + seq.push({ x, y }); + } + for (let x = columns - 1; x >= 0; x--) { + seq.push({ x, y }); + } + + return seq; +} + +export function useGridAnimator( + state: AgentState, + rows: number, + columns: number, + interval: number, + radius?: number +): Coordinate { + const [index, setIndex] = useState(0); + const [sequence, setSequence] = useState(() => [ + { + x: Math.floor(columns / 2), + y: Math.floor(rows / 2), + }, + ]); + + useEffect(() => { + const clampedRadius = radius + ? Math.min(radius, Math.floor(Math.max(rows, columns) / 2)) + : Math.floor(Math.max(rows, columns) / 2); + + if (state === 'thinking') { + setSequence(generateThinkingSequence(rows, columns)); + } else if (state === 'connecting' || state === 'initializing') { + const sequence = [...generateConnectingSequence(rows, columns, clampedRadius)]; + setSequence(sequence); + } else if (state === 'listening') { + setSequence(generateListeningSequence(rows, columns)); + } else { + setSequence([{ x: Math.floor(columns / 2), y: Math.floor(rows / 2) }]); + } + setIndex(0); + }, [state, rows, columns, radius]); + + useEffect(() => { + if (state === 'speaking') { + return; + } + + const indexInterval = setInterval(() => { + setIndex((prev) => { + return prev + 1; + }); + }, interval); + + return () => clearInterval(indexInterval); + }, [interval, columns, rows, state, sequence.length]); + + return sequence[index % sequence.length]; +} diff --git a/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/audio-oscilloscope-visualizer.tsx b/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/audio-oscilloscope-visualizer.tsx new file mode 100644 index 000000000..d36b2c133 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/audio-oscilloscope-visualizer.tsx @@ -0,0 +1,163 @@ +'use client'; + +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { + type AnimationPlaybackControlsWithThen, + type ValueAnimationTransition, + animate, + useMotionValue, + useMotionValueEvent, +} from 'motion/react'; +import { + type AgentState, + type TrackReference, + type TrackReferenceOrPlaceholder, + // useMultibandTrackVolume, + useTrackVolume, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; +import { OscilliscopeShader, type OscilliscopeShaderProps } from './shader'; + +const DEFAULT_SPEED = 5; +const DEFAULT_AMPLITUDE = 0.025; +const DEFAULT_FREQUENCY = 10; +const DEFAULT_TRANSITION: ValueAnimationTransition = { duration: 0.2, ease: 'easeOut' }; + +function useAnimatedValue(initialValue: T) { + const [value, setValue] = useState(initialValue); + const motionValue = useMotionValue(initialValue); + const controlsRef = useRef(null); + useMotionValueEvent(motionValue, 'change', (value) => setValue(value as T)); + + const animateFn = useCallback( + (targetValue: T | T[], transition: ValueAnimationTransition) => { + controlsRef.current = animate(motionValue, targetValue, transition); + }, + [motionValue] + ); + + return { value, controls: controlsRef, animate: animateFn }; +} + +export const audioOscilloscopeVisualizerVariants = cva(['aspect-square'], { + variants: { + size: { + icon: 'h-[24px] gap-[2px]', + sm: 'h-[56px] gap-[4px]', + md: 'h-[112px] gap-[8px]', + lg: 'h-[224px] gap-[16px]', + xl: 'h-[448px] gap-[32px]', + }, + }, + defaultVariants: { + size: 'md', + }, +}); + +interface AudioOscilloscopeVisualizerProps { + speed?: number; + state?: AgentState; + audioTrack: TrackReferenceOrPlaceholder; + className?: string; +} + +export function AudioOscilloscopeVisualizer({ + size = 'lg', + state = 'speaking', + rgbColor, + lineWidth, + smoothing, + audioTrack, + className, +}: AudioOscilloscopeVisualizerProps & + OscilliscopeShaderProps & + VariantProps) { + const [speed, setSpeed] = useState(DEFAULT_SPEED); + const { value: amplitude, animate: animateAmplitude } = useAnimatedValue(DEFAULT_AMPLITUDE); + const { value: frequency, animate: animateFrequency } = useAnimatedValue(DEFAULT_FREQUENCY); + const { value: opacity, animate: animateOpacity } = useAnimatedValue(1.0); + + const _lineWidth = useMemo(() => { + if (lineWidth) { + return lineWidth; + } + switch (size) { + case 'icon': + case 'sm': + return 2; + default: + return 1; + } + }, [lineWidth, size]); + + const volume = useTrackVolume(audioTrack as TrackReference, { + fftSize: 512, + smoothingTimeConstant: 0.55, + }); + + useEffect(() => { + switch (state) { + case 'disconnected': + setSpeed(DEFAULT_SPEED); + animateAmplitude(0, DEFAULT_TRANSITION); + animateFrequency(0, DEFAULT_TRANSITION); + animateOpacity(1.0, DEFAULT_TRANSITION); + return; + case 'listening': + setSpeed(DEFAULT_SPEED); + animateAmplitude(DEFAULT_AMPLITUDE, DEFAULT_TRANSITION); + animateFrequency(DEFAULT_FREQUENCY, DEFAULT_TRANSITION); + animateOpacity([1.0, 0.3], { + duration: 0.75, + repeat: Infinity, + repeatType: 'mirror', + }); + return; + case 'thinking': + case 'connecting': + case 'initializing': + setSpeed(DEFAULT_SPEED * 4); + animateAmplitude(DEFAULT_AMPLITUDE / 4, DEFAULT_TRANSITION); + animateFrequency(DEFAULT_FREQUENCY * 4, DEFAULT_TRANSITION); + animateOpacity([1.0, 0.3], { + duration: 0.4, + repeat: Infinity, + repeatType: 'mirror', + }); + return; + case 'speaking': + default: + setSpeed(DEFAULT_SPEED * 2); + animateAmplitude(DEFAULT_AMPLITUDE, DEFAULT_TRANSITION); + animateFrequency(DEFAULT_FREQUENCY, DEFAULT_TRANSITION); + animateOpacity(1.0, DEFAULT_TRANSITION); + return; + } + }, [state, setSpeed, animateAmplitude, animateFrequency, animateOpacity]); + + useEffect(() => { + if (state === 'speaking' && volume > 0) { + animateAmplitude(0.015 + 0.4 * volume, { duration: 0 }); + animateFrequency(20 + 60 * volume, { duration: 0 }); + } + }, [state, volume, animateAmplitude, animateFrequency]); + + return ( + + ); +} diff --git a/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/shader.tsx b/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/shader.tsx new file mode 100644 index 000000000..e2bfe83e6 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/shader.tsx @@ -0,0 +1,186 @@ +'use client'; + +import React, { forwardRef } from 'react'; +import { Shader } from '@/components/livekit/react-shader/react-shader'; + +const shaderSource = ` +const float TAU = 6.28318530718; + +// Noise for dithering +vec2 randFibo(vec2 p) { + p = fract(p * vec2(443.897, 441.423)); + p += dot(p, p.yx + 19.19); + return fract((p.xx + p.yx) * p.xy); +} + +// Luma for alpha +float luma(vec3 color) { + return dot(color, vec3(0.299, 0.587, 0.114)); +} + +// Bell curve function for attenuation from center with rounded top +float bellCurve(float distanceFromCenter, float maxDistance) { + float normalizedDistance = distanceFromCenter / maxDistance; + // Use cosine with high power for smooth rounded top + return pow(cos(normalizedDistance * (3.14159265359 / 4.0)), 16.0); +} + +// Calculate the sine wave +float oscilloscopeWave(float x, float centerX, float time) { + float relativeX = x - centerX; + float maxDistance = centerX; + float distanceFromCenter = abs(relativeX); + + // Apply bell curve for amplitude attenuation + float bell = bellCurve(distanceFromCenter, maxDistance); + + // Calculate wave with uniforms and bell curve attenuation + float wave = sin(relativeX * uFrequency + time * uSpeed) * uAmplitude * bell; + + return wave; +} + +void mainImage(out vec4 fragColor, in vec2 fragCoord) { + vec2 uv = fragCoord / iResolution.xy; + vec2 pos = uv - 0.5; + + // Calculate center and positions + float centerX = 0.5; + float centerY = 0.5; + float x = uv.x; + float y = uv.y; + + // Convert line width from pixels to UV space + // Use the average of width and height to handle aspect ratio + float pixelSize = 2.0 / (iResolution.x + iResolution.y); + float lineWidthUV = uLineWidth * pixelSize; + float smoothingUV = uSmoothing * pixelSize; + + // Find minimum distance to the wave by sampling nearby points + // This gives us consistent line width without high-frequency artifacts + const int NUM_SAMPLES = 50; // Must be const for GLSL loop + float minDist = 1000.0; + float sampleRange = 0.02; // Range to search for closest point + + for(int i = 0; i < NUM_SAMPLES; i++) { + float offset = (float(i) / float(NUM_SAMPLES - 1) - 0.5) * sampleRange; + float sampleX = x + offset; + float waveY = centerY + oscilloscopeWave(sampleX, centerX, iTime); + + // Calculate distance from current pixel to this point on the wave + vec2 wavePoint = vec2(sampleX, waveY); + vec2 currentPoint = vec2(x, y); + float dist = distance(currentPoint, wavePoint); + + minDist = min(minDist, dist); + } + + // Solid line with smooth edges using minimum distance + float line = smoothstep(lineWidthUV + smoothingUV, lineWidthUV - smoothingUV, minDist); + + // Calculate color position based on x position for gradient effect + float colorPos = x; + vec3 color = uColor; + + // Apply line intensity + color *= line; + + // Add dithering for smoother gradients + // color += (randFibo(fragCoord).x - 0.5) / 255.0; + + // Calculate alpha based on line intensity + float alpha = line * uMix; + + fragColor = vec4(color * uMix, alpha); +}`; + +export interface OscilliscopeShaderProps extends React.HTMLAttributes { + /** + * Class name + * @default '' + */ + className?: string; + /** + * Speed of the oscilloscope + * @default 10 + */ + speed?: number; + /** + * Amplitude of the oscilloscope + * @default 0.02 + */ + amplitude?: number; + /** + * Frequency of the oscilloscope + * @default 20.0 + */ + frequency?: number; + /** + * RGB color of the oscilloscope + * @default [31.0 / 255, 213.0 / 255, 249.0 / 255] + */ + rgbColor?: [number, number, number]; + /** + * Mix of the oscilloscope + * @default 1.0 + */ + mix?: number; + /** + * Line width of the oscilloscope in pixels + * @default 2.0 + */ + lineWidth?: number; + /** + * Smoothing of the oscilloscope in pixels + * @default 0.5 + */ + smoothing?: number; +} + +export const OscilliscopeShader = forwardRef( + ( + { + className, + speed = 10, + amplitude = 0.02, + frequency = 20.0, + rgbColor = [31.0 / 255, 213.0 / 255, 249.0 / 255], // LiveKit Blue + mix = 1.0, + lineWidth = 2.0, + smoothing = 0.5, + ...props + }, + ref + ) => { + const globalThis = typeof window !== 'undefined' ? window : global; + + return ( +
+ { + console.error('Shader error:', error); + }} + onWarning={(warning) => { + console.warn('Shader warning:', warning); + }} + style={{ width: '100%', height: '100%' } as CSSStyleDeclaration} + /> +
+ ); + } +); + +OscilliscopeShader.displayName = 'OscilliscopeShader'; + +export default OscilliscopeShader; diff --git a/components/livekit/audio-visualizer/audio-radial-visualizer/_bar-visualizer.tsx b/components/livekit/audio-visualizer/audio-radial-visualizer/_bar-visualizer.tsx new file mode 100644 index 000000000..aa3ab23ba --- /dev/null +++ b/components/livekit/audio-visualizer/audio-radial-visualizer/_bar-visualizer.tsx @@ -0,0 +1,99 @@ +import { useEffect, useMemo, useRef } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { + type AgentState, + BarVisualizer as LiveKitBarVisualizer, + type TrackReferenceOrPlaceholder, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; + +const MIN_HEIGHT = 15; // 15% + +export const barVisualizerVariants = cva( + ['relative flex aspect-square h-36 items-center justify-center'], + { + variants: { + size: { + default: 'h-32', + icon: 'h-6', + xs: 'h-8', + sm: 'h-16', + md: 'h-32', + lg: 'h-64', + xl: 'h-96', + '2xl': 'h-128', + }, + }, + defaultVariants: { + size: 'default', + }, + } +); + +interface BarVisualizerProps { + state?: AgentState; + barCount?: number; + audioTrack?: TrackReferenceOrPlaceholder; + className?: string; +} + +export function BarVisualizer({ + size, + state, + barCount, + audioTrack, + className, +}: BarVisualizerProps & VariantProps) { + const ref = useRef(null); + const _barCount = useMemo(() => { + if (barCount) { + return barCount; + } + switch (size) { + case 'icon': + case 'xs': + return 3; + default: + return 5; + } + }, [barCount, size]); + + const x = (1 / (_barCount + (_barCount + 1) / 2)) * 100; + + // reset bars height when audio track is disconnected + useEffect(() => { + if (ref.current && !audioTrack) { + const bars = [...(ref.current.querySelectorAll('& > span') ?? [])] as HTMLElement[]; + + bars.forEach((bar) => { + bar.style.height = `${MIN_HEIGHT}%`; + }); + } + }, [audioTrack]); + + return ( + + + + ); +} diff --git a/components/livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer.tsx b/components/livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer.tsx new file mode 100644 index 000000000..86ac399e1 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer.tsx @@ -0,0 +1,151 @@ +import { ReactNode, useMemo } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { type LocalAudioTrack, type RemoteAudioTrack } from 'livekit-client'; +import { + type AgentState, + type TrackReferenceOrPlaceholder, + useMultibandTrackVolume, +} from '@livekit/components-react'; +import { cloneSingleChild, cn } from '@/lib/utils'; +import { useBarAnimator } from './hooks/useBarAnimator'; + +export const audioRadialVisualizerVariants = cva( + [ + 'relative flex items-center justify-center', + '[&_[data-lk-index]]:absolute [&_[data-lk-index]]:top-1/2 [&_[data-lk-index]]:left-1/2 [&_[data-lk-index]]:origin-bottom [&_[data-lk-index]]:-translate-x-1/2', + '[&_[data-lk-index]]:rounded-full [&_[data-lk-index]]:transition-colors [&_[data-lk-index]]:duration-250 [&_[data-lk-index]]:ease-linear [&_[data-lk-index]]:bg-(--audio-visualizer-idle) [&_[data-lk-index]]:data-[lk-highlighted=true]:bg-(--audio-visualizer-active)', + ], + { + variants: { + size: { + icon: ['h-[24px] gap-[2px]', '[&_[data-lk-index]]:w-[4px] [&_[data-lk-index]]:min-h-[4px]'], + sm: ['h-[56px] gap-[4px]', '[&_[data-lk-index]]:w-[8px] [&_[data-lk-index]]:min-h-[8px]'], + md: [ + 'h-[112px] gap-[8px]', + '[&_[data-lk-index]]:w-[16px] [&_[data-lk-index]]:min-h-[16px]', + ], + lg: [ + 'h-[224px] gap-[16px]', + '[&_[data-lk-index]]:w-[32px] [&_[data-lk-index]]:min-h-[32px]', + ], + xl: [ + 'h-[448px] gap-[32px]', + '[&_[data-lk-index]]:w-[64px] [&_[data-lk-index]]:min-h-[64px]', + ], + }, + }, + defaultVariants: { + size: 'md', + }, + } +); + +interface AudioRadialVisualizerProps { + state?: AgentState; + radius?: number; + barCount?: number; + audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder; + className?: string; + children?: ReactNode; +} + +export function AudioRadialVisualizer({ + size, + state, + radius, + barCount, + audioTrack, + className, + children, +}: AudioRadialVisualizerProps & VariantProps) { + const _barCount = useMemo(() => { + if (barCount) { + return barCount; + } + switch (size) { + case 'icon': + case 'sm': + return 8; + default: + return 12; + } + }, [barCount, size]); + + const volumeBands = useMultibandTrackVolume(audioTrack, { + bands: Math.ceil(_barCount / 2), + loPass: 100, + hiPass: 200, + }); + + const sequencerInterval = useMemo(() => { + switch (state) { + case 'connecting': + return 2000 / _barCount; + case 'initializing': + return 500; + case 'listening': + return 500; + case 'thinking': + return 150; + default: + return 1000; + } + }, [state, _barCount]); + + const distanceFromCenter = useMemo(() => { + if (radius) { + return radius; + } + switch (size) { + case 'icon': + return 6; + case 'xl': + return 128; + case 'lg': + return 64; + case 'sm': + return 16; + case 'md': + default: + return 32; + } + }, [size, radius]); + + const highlightedIndices = useBarAnimator(state, _barCount, sequencerInterval); + const bands = audioTrack + ? [...volumeBands, ...volumeBands].slice(0, _barCount) + : new Array(_barCount).fill(0); + + return ( +
+ {bands.map((band, idx) => { + const angle = (idx / _barCount) * Math.PI * 2; + + return ( +
+ {children ? ( + cloneSingleChild(children, { + key: idx, + 'data-lk-index': idx, + 'data-lk-highlighted': highlightedIndices.includes(idx), + }) + ) : ( +
+ )} +
+ ); + })} +
+ ); +} diff --git a/components/livekit/audio-visualizer/audio-radial-visualizer/hooks/useBarAnimator.ts b/components/livekit/audio-visualizer/audio-radial-visualizer/hooks/useBarAnimator.ts new file mode 100644 index 000000000..e0d1c808e --- /dev/null +++ b/components/livekit/audio-visualizer/audio-radial-visualizer/hooks/useBarAnimator.ts @@ -0,0 +1,83 @@ +import { useEffect, useRef, useState } from 'react'; +import { type AgentState } from '@livekit/components-react'; + +function generateConnectingSequenceBar(columns: number): number[][] { + const seq = []; + + for (let x = 0; x < columns; x++) { + seq.push([x]); + } + + return seq; +} + +function generateListeningSequenceBar(columns: number): number[][] { + const center = Math.floor(columns / 2); + const noIndex = -1; + + return [[center], [noIndex]]; +} + +// function generateThinkingSequenceBar(columns: number): number[][] { +// const seq = []; +// for (let x = 0; x < columns; x++) { +// seq.push([x]); +// } + +// for (let x = columns - 1; x >= 0; x--) { +// seq.push([x]); +// } + +// return seq; +// } + +export const useBarAnimator = ( + state: AgentState | undefined, + columns: number, + interval: number +): number[] => { + const [index, setIndex] = useState(0); + const [sequence, setSequence] = useState([[]]); + + useEffect(() => { + if (state === 'thinking') { + setSequence(generateListeningSequenceBar(columns)); + // setSequence(generateThinkingSequenceBar(columns)); + } else if (state === 'connecting' || state === 'initializing') { + setSequence(generateConnectingSequenceBar(columns)); + } else if (state === 'listening') { + setSequence(generateListeningSequenceBar(columns)); + } else if (state === undefined || state === 'speaking') { + setSequence([new Array(columns).fill(0).map((_, idx) => idx)]); + } else { + setSequence([[]]); + } + setIndex(0); + }, [state, columns]); + + const animationFrameId = useRef(null); + useEffect(() => { + let startTime = performance.now(); + + const animate = (time: DOMHighResTimeStamp) => { + const timeElapsed = time - startTime; + + if (timeElapsed >= interval) { + setIndex((prev) => prev + 1); + startTime = time; + } + + animationFrameId.current = requestAnimationFrame(animate); + }; + + animationFrameId.current = requestAnimationFrame(animate); + + return () => { + if (animationFrameId.current !== null) { + cancelAnimationFrame(animationFrameId.current); + } + }; + }, [interval, columns, state, sequence.length]); + + return sequence[index % sequence.length]; +}; diff --git a/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx b/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx new file mode 100644 index 000000000..4d24857e4 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx @@ -0,0 +1,167 @@ +'use client'; + +import { useCallback, useEffect, useRef, useState } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { + type AnimationPlaybackControlsWithThen, + type KeyframesTarget, + type ValueAnimationTransition, + animate, + useMotionValue, + useMotionValueEvent, +} from 'motion/react'; +import { + type AgentState, + type TrackReference, + type TrackReferenceOrPlaceholder, + // useMultibandTrackVolume, + useTrackVolume, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; +import { AuroraShaders, type AuroraShadersProps } from './shader'; + +const DEFAULT_SPEED = 10; +const DEFAULT_AMPLITUDE = 2; +const DEFAULT_FREQUENCY = 0.5; +const DEFAULT_SCALE = 0.2; +const DEFAULT_BRIGHTNESS = 1.5; +const DEFAULT_TRANSITION: ValueAnimationTransition = { duration: 0.5, ease: 'easeOut' }; +const DEFAULT_PULSE_TRANSITION: ValueAnimationTransition = { + duration: 0.35, + ease: 'easeOut', + repeat: Infinity, + repeatType: 'mirror', +}; + +function useAnimatedValue(initialValue: T) { + const [value, setValue] = useState(initialValue); + const motionValue = useMotionValue(initialValue); + const controlsRef = useRef(null); + useMotionValueEvent(motionValue, 'change', (value) => setValue(value as T)); + + const animateFn = useCallback( + (targetValue: T | KeyframesTarget, transition: ValueAnimationTransition) => { + controlsRef.current = animate(motionValue, targetValue, transition); + }, + [motionValue] + ); + + return { value, motionValue, controls: controlsRef, animate: animateFn }; +} + +export const audioShaderVisualizerVariants = cva(['aspect-square'], { + variants: { + size: { + icon: 'h-[24px] gap-[2px]', + sm: 'h-[56px] gap-[4px]', + md: 'h-[112px] gap-[8px]', + lg: 'h-[224px] gap-[16px]', + xl: 'h-[448px] gap-[32px]', + }, + }, + defaultVariants: { + size: 'md', + }, +}); + +interface AudioShaderVisualizerProps { + state?: AgentState; + audioTrack: TrackReferenceOrPlaceholder; +} + +export function AudioShaderVisualizer({ + size = 'md', + state = 'speaking', + mode, + rgbColor, + shape = 1, + colorShift = 0.05, + audioTrack, + className, +}: AudioShaderVisualizerProps & + AuroraShadersProps & + VariantProps) { + const [speed, setSpeed] = useState(DEFAULT_SPEED); + const { + value: scale, + animate: animateScale, + motionValue: scaleMotionValue, + } = useAnimatedValue(DEFAULT_SCALE); + const { value: amplitude, animate: animateAmplitude } = useAnimatedValue(DEFAULT_AMPLITUDE); + const { value: frequency, animate: animateFrequency } = useAnimatedValue(DEFAULT_FREQUENCY); + const { value: brightness, animate: animateBrightness } = useAnimatedValue(DEFAULT_BRIGHTNESS); + + const volume = useTrackVolume(audioTrack as TrackReference, { + fftSize: 512, + smoothingTimeConstant: 0.55, + }); + + useEffect(() => { + switch (state) { + case 'disconnected': + setSpeed(5); + animateScale(0.2, DEFAULT_TRANSITION); + animateAmplitude(1.2, DEFAULT_TRANSITION); + animateFrequency(0.4, DEFAULT_TRANSITION); + animateBrightness(1.0, DEFAULT_TRANSITION); + return; + case 'listening': + setSpeed(20); + animateScale(0.3, { type: 'spring', duration: 1.0, bounce: 0.35 }); + animateAmplitude(1.0, DEFAULT_TRANSITION); + animateFrequency(0.7, DEFAULT_TRANSITION); + animateBrightness([1.5, 2.0], DEFAULT_PULSE_TRANSITION); + return; + case 'thinking': + case 'connecting': + case 'initializing': + setSpeed(30); + animateScale(0.3, DEFAULT_TRANSITION); + animateAmplitude(0.5, DEFAULT_TRANSITION); + animateFrequency(1, DEFAULT_TRANSITION); + animateBrightness([0.5, 2.5], DEFAULT_PULSE_TRANSITION); + return; + case 'speaking': + setSpeed(70); + animateScale(0.3, DEFAULT_TRANSITION); + animateAmplitude(0.75, DEFAULT_TRANSITION); + animateFrequency(1.25, DEFAULT_TRANSITION); + animateBrightness([0.2, 2.5], DEFAULT_PULSE_TRANSITION); + return; + } + }, [state, shape, animateScale, animateAmplitude, animateFrequency, animateBrightness]); + + useEffect(() => { + if (state === 'speaking' && volume > 0 && !scaleMotionValue.isAnimating()) { + animateScale(0.2 + 0.2 * volume, { duration: 0 }); + } + }, [ + state, + volume, + scaleMotionValue, + animateScale, + animateAmplitude, + animateFrequency, + animateBrightness, + ]); + + return ( + + ); +} diff --git a/components/livekit/audio-visualizer/audio-shader-visualizer/shader.tsx b/components/livekit/audio-visualizer/audio-shader-visualizer/shader.tsx new file mode 100644 index 000000000..94f251a77 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-shader-visualizer/shader.tsx @@ -0,0 +1,304 @@ +'use client'; + +import React, { forwardRef } from 'react'; +import { Shader } from '@/components/livekit/react-shader/react-shader'; + +const shaderSource = ` +const float TAU = 6.28318530718; + +// Noise for dithering +vec2 randFibo(vec2 p) { + p = fract(p * vec2(443.897, 441.423)); + p += dot(p, p.yx + 19.19); + return fract((p.xx + p.yx) * p.xy); +} + +// Tonemap +vec3 Tonemap_Reinhard(vec3 x) { + x *= 4.0; + return x / (1.0 + x); +} + +// Luma for alpha +float luma(vec3 color) { + return dot(color, vec3(0.299, 0.587, 0.114)); +} + +// RGB to HSV +vec3 rgb2hsv(vec3 c) { + vec4 K = vec4(0.0, -1.0 / 3.0, 2.0 / 3.0, -1.0); + vec4 p = mix(vec4(c.bg, K.wz), vec4(c.gb, K.xy), step(c.b, c.g)); + vec4 q = mix(vec4(p.xyw, c.r), vec4(c.r, p.yzx), step(p.x, c.r)); + float d = q.x - min(q.w, q.y); + float e = 1.0e-10; + return vec3(abs(q.z + (q.w - q.y) / (6.0 * d + e)), d / (q.x + e), q.x); +} + +// HSV to RGB +vec3 hsv2rgb(vec3 c) { + vec4 K = vec4(1.0, 2.0 / 3.0, 1.0 / 3.0, 3.0); + vec3 p = abs(fract(c.xxx + K.xyz) * 6.0 - K.www); + return c.z * mix(K.xxx, clamp(p - K.xxx, 0.0, 1.0), c.y); +} + +// SDF shapes +float sdCircle(vec2 st, float r) { + return length(st) - r; +} + +float sdLine(vec2 p, float r) { + float halfLen = r * 2.0; + vec2 a = vec2(-halfLen, 0.0); + vec2 b = vec2(halfLen, 0.0); + vec2 pa = p - a; + vec2 ba = b - a; + float h = clamp(dot(pa, ba) / dot(ba, ba), 0.0, 1.0); + return length(pa - ba * h); +} + +float getSdf(vec2 st) { + if(uShape == 1.0) return sdCircle(st, uScale); + else if(uShape == 2.0) return sdLine(st, uScale); + return sdCircle(st, uScale); // Default +} + +vec2 turb(vec2 pos, float t, float it) { + // mat2 rot = mat2(0.6, -0.8, 0.8, 0.6); + mat2 rot = mat2(0.6, -0.25, 0.25, 0.9); + float freq = mix(2.0, 15.0, uFrequency); + float amp = uAmplitude; + float xp = 1.4; + float time = t * 0.1 * uSpeed; + + for(float i = 0.0; i < 4.0; i++) { + vec2 s = sin(freq * (pos * rot) + i * time + it); + pos += amp * rot[0] * s / freq; + rot *= mat2(0.6, -0.8, 0.8, 0.6); + amp *= mix(1.0, max(s.y, s.x), uVariance); + freq *= xp; + } + + return pos; +} + +const float ITERATIONS = 36.0; + +void mainImage(out vec4 fragColor, in vec2 fragCoord) { + vec2 uv = fragCoord / iResolution.xy; + + vec3 pp = vec3(0.0); + vec3 bloom = vec3(0.0); + float t = iTime * 0.5; + vec2 pos = uv - 0.5; + + vec2 prevPos = turb(pos, t, 0.0 - 1.0 / ITERATIONS); + float spacing = mix(1.0, TAU, uSpacing); + + for(float i = 1.0; i < ITERATIONS + 1.0; i++) { + float iter = i / ITERATIONS; + vec2 st = turb(pos, t, iter * spacing); + float d = abs(getSdf(st)); + float pd = distance(st, prevPos); + prevPos = st; + float dynamicBlur = exp2(pd * 2.0 * 1.4426950408889634) - 1.0; + float ds = smoothstep(0.0, uBlur * 0.05 + max(dynamicBlur * uSmoothing, 0.001), d); + + // Shift color based on iteration using uColorScale + vec3 color = uColor; + if(uColorShift > 0.01) { + vec3 hsv = rgb2hsv(color); + // Shift hue by iteration + hsv.x = fract(hsv.x + (1.0 - iter) * uColorShift * 0.3); + color = hsv2rgb(hsv); + } + + float invd = 1.0 / max(d + dynamicBlur, 0.001); + pp += (ds - 1.0) * color; + bloom += clamp(invd, 0.0, 250.0) * color; + } + + pp *= 1.0 / ITERATIONS; + + vec3 color; + + // Dark mode (default) + // use bloom effect + if(uMode < 0.5) { + bloom = bloom / (bloom + 2e4); + color = (-pp + bloom * 3.0 * uBloom) * 1.2; + color += (randFibo(fragCoord).x - 0.5) / 255.0; + color = Tonemap_Reinhard(color); + float alpha = luma(color) * uMix; + fragColor = vec4(color * uMix, alpha); + } + + // Light mode + // no bloom effect + else { + color = -pp; + color += (randFibo(fragCoord).x - 0.5) / 255.0; + + // Preserve hue by tone mapping brightness only + float brightness = length(color); + vec3 direction = brightness > 0.0 ? color / brightness : color; + + // Reinhard on brightness + float factor = 2.0; + float mappedBrightness = (brightness * factor) / (1.0 + brightness * factor); + color = direction * mappedBrightness; + + // Boost saturation to compensate for white background bleed-through + // When alpha < 1.0, white bleeds through making colors look desaturated + // So we increase saturation to maintain vibrant appearance + float gray = dot(color, vec3(0.2, 0.5, 0.1)); + float saturationBoost = 3.0; + color = mix(vec3(gray), color, saturationBoost); + + // Clamp between 0-1 + color = clamp(color, 0.0, 1.0); + + float alpha = mappedBrightness * clamp(uMix, 1.0, 2.0); + fragColor = vec4(color, alpha); + } +}`; + +export interface AuroraShadersProps extends React.HTMLAttributes { + /** + * Aurora wave speed + * @default 1.0 + */ + speed?: number; + + /** + * Turbulence amplitude + * @default 0.5 + */ + amplitude?: number; + + /** + * Wave frequency and complexity + * @default 0.5 + */ + frequency?: number; + + /** + * Shape scale + * @default 0.3 + */ + scale?: number; + + /** + * Shape type: 1=circle, 2=line + * @default 1 + */ + shape?: number; + + /** + * Edge blur/softness + * @default 1.0 + */ + blur?: number; + + /** + * RGB color + * @default [0.0, 0.0, 1.0] + */ + rgbColor?: [number, number, number]; + + /** + * Color variation across layers (0-1) + * Controls how much colors change between iterations + * @default 0.5 + * @example 0.0 - minimal color variation (more uniform) + * @example 0.5 - moderate variation (default) + * @example 1.0 - maximum variation (rainbow effect) + */ + colorShift?: number; + + /** + * Brightness of the aurora (0-1) + * @default 1.0 + */ + brightness?: number; + + /** + * Display mode for different backgrounds + * - 'dark': Optimized for dark backgrounds (default) + * - 'light': Optimized for light/white backgrounds (inverts colors) + * @default 'dark' + */ + mode?: 'dark' | 'light'; +} + +export const AuroraShaders = forwardRef( + ( + { + className, + shape = 1.0, + speed = 1.0, + amplitude = 0.5, + frequency = 0.5, + scale = 0.2, + blur = 1.0, + rgbColor = [0.12156862745098039, 0.8352941176470589, 0.9764705882352941], // LiveKit Blue, + colorShift = 1.0, + brightness = 1.0, + mode = typeof window !== 'undefined' && document.documentElement.classList.contains('dark') + ? 'dark' + : 'light', + ...props + }, + ref + ) => { + const globalThis = typeof window !== 'undefined' ? window : global; + return ( +
+ { + console.log('error', error); + }} + onWarning={(warning) => { + console.log('warning', warning); + }} + style={{ width: '100%', height: '100%' } as CSSStyleDeclaration} + /> +
+ ); + } +); + +AuroraShaders.displayName = 'AuroraShaders'; + +export default AuroraShaders; diff --git a/components/livekit/chat-indicator.tsx b/components/livekit/chat-indicator.tsx new file mode 100644 index 000000000..fcfb7d66b --- /dev/null +++ b/components/livekit/chat-indicator.tsx @@ -0,0 +1,49 @@ +import { AnimatePresence, motion } from 'motion/react'; +import { type AgentState } from '@livekit/components-react'; +import { cn } from '@/lib/utils'; + +const motionAnimationProps = { + variants: { + hidden: { + opacity: 0, + scale: 0.1, + transition: { + duration: 0.1, + ease: 'linear', + }, + }, + visible: { + opacity: [0.5, 1], + scale: [1, 1.2], + transition: { + type: 'spring', + bounce: 0, + duration: 0.5, + repeat: Infinity, + repeatType: 'mirror' as const, + }, + }, + }, + initial: 'hidden', + animate: 'visible', + exit: 'hidden', +}; + +export interface ChatIndicatorProps { + agentState?: AgentState; + className?: string; +} + +export function ChatIndicator({ agentState, className }: ChatIndicatorProps) { + return ( + + {agentState === 'thinking' && ( + + )} + + ); +} diff --git a/components/livekit/react-shader/logging.ts b/components/livekit/react-shader/logging.ts new file mode 100644 index 000000000..aa742d8c2 --- /dev/null +++ b/components/livekit/react-shader/logging.ts @@ -0,0 +1 @@ +export const log = (text: string) => `react-shaders: ${text}`; diff --git a/components/livekit/react-shader/react-shader.original.tsx b/components/livekit/react-shader/react-shader.original.tsx new file mode 100644 index 000000000..aa7093029 --- /dev/null +++ b/components/livekit/react-shader/react-shader.original.tsx @@ -0,0 +1,614 @@ +import { type CSSProperties, Component } from 'react'; +import { log } from './logging'; +import { + ClampToEdgeWrapping, + LinearFilter, + LinearMipMapLinearFilter, + LinearMipMapNearestFilter, + MirroredRepeatWrapping, + NearestFilter, + NearestMipMapLinearFilter, + NearestMipMapNearestFilter, + RepeatWrapping, + Texture, +} from './texture'; +import { + type UniformType, + type Vector2, + type Vector3, + type Vector4, + isMatrixType, + isVectorListType, + processUniform, + uniformTypeToGLSLType, +} from './uniforms'; + +export { + ClampToEdgeWrapping, + LinearFilter, + LinearMipMapLinearFilter, + LinearMipMapNearestFilter, + MirroredRepeatWrapping, + NearestFilter, + NearestMipMapLinearFilter, + NearestMipMapNearestFilter, + RepeatWrapping, +}; + +export type { Vector2, Vector3, Vector4 }; + +const PRECISIONS = ['lowp', 'mediump', 'highp']; +const FS_MAIN_SHADER = `\nvoid main(void){ + vec4 color = vec4(0.0,0.0,0.0,1.0); + mainImage( color, gl_FragCoord.xy ); + gl_FragColor = color; +}`; +const BASIC_FS = `void mainImage( out vec4 fragColor, in vec2 fragCoord ) { + vec2 uv = fragCoord/iResolution.xy; + vec3 col = 0.5 + 0.5*cos(iTime+uv.xyx+vec3(0,2,4)); + fragColor = vec4(col,1.0); +}`; +const BASIC_VS = `attribute vec3 aVertexPosition; +void main(void) { + gl_Position = vec4(aVertexPosition, 1.0); +}`; +const UNIFORM_TIME = 'iTime'; +const UNIFORM_TIMEDELTA = 'iTimeDelta'; +const UNIFORM_DATE = 'iDate'; +const UNIFORM_FRAME = 'iFrame'; +const UNIFORM_MOUSE = 'iMouse'; +const UNIFORM_RESOLUTION = 'iResolution'; +const UNIFORM_CHANNEL = 'iChannel'; +const UNIFORM_CHANNELRESOLUTION = 'iChannelResolution'; +const UNIFORM_DEVICEORIENTATION = 'iDeviceOrientation'; + +const latestPointerClientCoords = (e: MouseEvent | TouchEvent) => { + // @ts-expect-error TODO: Deal with this. + return [e.clientX || e.changedTouches[0].clientX, e.clientY || e.changedTouches[0].clientY]; +}; +const lerpVal = (v0: number, v1: number, t: number) => v0 * (1 - t) + v1 * t; +const insertStringAtIndex = (currentString: string, string: string, index: number) => + index > 0 + ? currentString.substring(0, index) + + string + + currentString.substring(index, currentString.length) + : string + currentString; + +type Uniform = { type: string; value: number[] | number }; +export type Uniforms = Record; +type TextureParams = { + url: string; + wrapS?: number; + wrapT?: number; + minFilter?: number; + magFilter?: number; + flipY?: number; +}; + +type Props = { + /** Fragment shader GLSL code. */ + fs: string; + + /** Vertex shader GLSL code. */ + vs?: string; + + /** + * Textures to be passed to the shader. Textures need to be squared or will be + * automatically resized. + * + * Options default to: + * + * ```js + * { + * minFilter: LinearMipMapLinearFilter, + * magFilter: LinearFilter, + * wrapS: RepeatWrapping, + * wrapT: RepeatWrapping, + * } + * ``` + * + * See [textures in the docs](https://rysana.com/docs/react-shaders#textures) + * for details. + */ + textures?: TextureParams[]; + + /** + * Custom uniforms to be passed to the shader. + * + * See [custom uniforms in the + * docs](https://rysana.com/docs/react-shaders#custom-uniforms) for details. + */ + uniforms?: Uniforms; + + /** + * Color used when clearing the canvas. + * + * See [the WebGL + * docs](https://developer.mozilla.org/en-US/docs/Web/API/WebGLRenderingContext/clearColor) + * for details. + */ + clearColor?: Vector4; + + /** + * GLSL precision qualifier. Defaults to `'highp'`. Balance between + * performance and quality. + */ + precision?: 'highp' | 'lowp' | 'mediump'; + + /** Custom inline style for canvas. */ + style?: CSSStyleDeclaration; + + /** Customize WebGL context attributes. See [the WebGL docs](https://developer.mozilla.org/en-US/docs/Web/API/WebGLRenderingContext/getContextAttributes) for details. */ + contextAttributes?: Record; + + /** Lerp value for `iMouse` built-in uniform. Must be between 0 and 1. */ + lerp?: number; + + /** Device pixel ratio. */ + devicePixelRatio?: number; + + /** + * Callback for when the textures are done loading. Useful if you want to do + * something like e.g. hide the canvas until textures are done loading. + */ + onDoneLoadingTextures?: () => void; + + /** Custom callback to handle errors. Defaults to `console.error`. */ + onError?: (error: string) => void; + + /** Custom callback to handle warnings. Defaults to `console.warn`. */ + onWarning?: (warning: string) => void; +}; +export class Shader extends Component { + uniforms: Record< + string, + { type: string; isNeeded: boolean; value?: number[] | number; arraySize?: string } + >; + constructor(props: Props) { + super(props); + this.uniforms = { + [UNIFORM_TIME]: { type: 'float', isNeeded: false, value: 0 }, + [UNIFORM_TIMEDELTA]: { type: 'float', isNeeded: false, value: 0 }, + [UNIFORM_DATE]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + [UNIFORM_MOUSE]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + [UNIFORM_RESOLUTION]: { type: 'vec2', isNeeded: false, value: [0, 0] }, + [UNIFORM_FRAME]: { type: 'int', isNeeded: false, value: 0 }, + [UNIFORM_DEVICEORIENTATION]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + }; + } + static defaultProps = { + textures: [], + contextAttributes: {}, + devicePixelRatio: 1, + vs: BASIC_VS, + precision: 'highp', + onError: console.error, + onWarn: console.warn, + }; + componentDidMount = () => { + this.initWebGL(); + const { fs, vs, clearColor = [0, 0, 0, 1] } = this.props; + const { gl } = this; + if (gl && this.canvas) { + gl.clearColor(...clearColor); + gl.clearDepth(1.0); + gl.enable(gl.DEPTH_TEST); + gl.depthFunc(gl.LEQUAL); + gl.viewport(0, 0, this.canvas.width, this.canvas.height); + this.canvas.height = this.canvas.clientHeight; + this.canvas.width = this.canvas.clientWidth; + this.processCustomUniforms(); + this.processTextures(); + this.initShaders(this.preProcessFragment(fs || BASIC_FS), vs || BASIC_VS); + this.initBuffers(); + requestAnimationFrame(this.drawScene); + this.addEventListeners(); + this.onResize(); + } + }; + shouldComponentUpdate = () => false; + componentWillUnmount() { + const { gl } = this; + if (gl) { + gl.getExtension('WEBGL_lose_context')?.loseContext(); + gl.useProgram(null); + gl.deleteProgram(this.shaderProgram ?? null); + if (this.texturesArr.length > 0) { + for (const texture of this.texturesArr as Texture[]) { + gl.deleteTexture(texture._webglTexture); + } + } + this.shaderProgram = null; + } + this.removeEventListeners(); + cancelAnimationFrame(this.animFrameId ?? 0); + } + setupChannelRes = ({ width, height }: Texture, id: number) => { + const { devicePixelRatio = 1 } = this.props; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iChannelResolution.value[id * 3] = width * devicePixelRatio; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iChannelResolution.value[id * 3 + 1] = height * devicePixelRatio; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iChannelResolution.value[id * 3 + 2] = 0; + // console.log(this.uniforms); + }; + initWebGL = () => { + const { contextAttributes } = this.props; + if (!this.canvas) return; + this.gl = (this.canvas.getContext('webgl', contextAttributes) || + this.canvas.getContext( + 'experimental-webgl', + contextAttributes + )) as WebGLRenderingContext | null; + this.gl?.getExtension('OES_standard_derivatives'); + this.gl?.getExtension('EXT_shader_texture_lod'); + }; + initBuffers = () => { + const { gl } = this; + this.squareVerticesBuffer = gl?.createBuffer(); + gl?.bindBuffer(gl.ARRAY_BUFFER, this.squareVerticesBuffer ?? null); + const vertices = [1.0, 1.0, 0.0, -1.0, 1.0, 0.0, 1.0, -1.0, 0.0, -1.0, -1.0, 0.0]; + gl?.bufferData(gl.ARRAY_BUFFER, new Float32Array(vertices), gl.STATIC_DRAW); + }; + addEventListeners = () => { + const options = { passive: true }; + if (this.uniforms.iMouse?.isNeeded && this.canvas) { + this.canvas.addEventListener('mousemove', this.mouseMove, options); + this.canvas.addEventListener('mouseout', this.mouseUp, options); + this.canvas.addEventListener('mouseup', this.mouseUp, options); + this.canvas.addEventListener('mousedown', this.mouseDown, options); + this.canvas.addEventListener('touchmove', this.mouseMove, options); + this.canvas.addEventListener('touchend', this.mouseUp, options); + this.canvas.addEventListener('touchstart', this.mouseDown, options); + } + if (this.uniforms.iDeviceOrientation?.isNeeded) { + window.addEventListener('deviceorientation', this.onDeviceOrientationChange, options); + } + if (this.canvas) { + this.resizeObserver = new ResizeObserver(this.onResize); + this.resizeObserver.observe(this.canvas); + } + }; + removeEventListeners = () => { + const options = { passive: true } as EventListenerOptions; + if (this.uniforms.iMouse?.isNeeded && this.canvas) { + this.canvas.removeEventListener('mousemove', this.mouseMove, options); + this.canvas.removeEventListener('mouseout', this.mouseUp, options); + this.canvas.removeEventListener('mouseup', this.mouseUp, options); + this.canvas.removeEventListener('mousedown', this.mouseDown, options); + this.canvas.removeEventListener('touchmove', this.mouseMove, options); + this.canvas.removeEventListener('touchend', this.mouseUp, options); + this.canvas.removeEventListener('touchstart', this.mouseDown, options); + } + if (this.uniforms.iDeviceOrientation?.isNeeded) { + window.removeEventListener('deviceorientation', this.onDeviceOrientationChange, options); + } + if (this.resizeObserver) { + this.resizeObserver.disconnect(); + } + }; + onDeviceOrientationChange = ({ alpha, beta, gamma }: DeviceOrientationEvent) => { + this.uniforms.iDeviceOrientation.value = [ + alpha ?? 0, + beta ?? 0, + gamma ?? 0, + window.orientation || 0, + ]; + }; + mouseDown = (e: MouseEvent | TouchEvent) => { + const [clientX, clientY] = latestPointerClientCoords(e); + const mouseX = clientX - (this.canvasPosition?.left ?? 0) - window.pageXOffset; + const mouseY = + (this.canvasPosition?.height ?? 0) - + clientY - + (this.canvasPosition?.top ?? 0) - + window.pageYOffset; + this.mousedown = true; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[2] = mouseX; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[3] = mouseY; + this.lastMouseArr[0] = mouseX; + this.lastMouseArr[1] = mouseY; + }; + mouseMove = (e: MouseEvent | TouchEvent) => { + this.canvasPosition = this.canvas?.getBoundingClientRect(); + const { lerp = 1 } = this.props; + const [clientX, clientY] = latestPointerClientCoords(e); + const mouseX = clientX - (this.canvasPosition?.left ?? 0); + const mouseY = (this.canvasPosition?.height ?? 0) - clientY - (this.canvasPosition?.top ?? 0); + if (lerp !== 1) { + this.lastMouseArr[0] = mouseX; + this.lastMouseArr[1] = mouseY; + } else { + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[0] = mouseX; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[1] = mouseY; + } + }; + mouseUp = () => { + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[2] = 0; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[3] = 0; + }; + onResize = () => { + const { gl } = this; + const { devicePixelRatio = 1 } = this.props; + if (!gl) return; + this.canvasPosition = this.canvas?.getBoundingClientRect(); + // Force pixel ratio to be one to avoid expensive calculus on retina display. + const realToCSSPixels = devicePixelRatio; + const displayWidth = Math.floor((this.canvasPosition?.width ?? 1) * realToCSSPixels); + const displayHeight = Math.floor((this.canvasPosition?.height ?? 1) * realToCSSPixels); + gl.canvas.width = displayWidth; + gl.canvas.height = displayHeight; + if (this.uniforms.iResolution?.isNeeded && this.shaderProgram) { + const rUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_RESOLUTION); + gl.uniform2fv(rUniform, [gl.canvas.width, gl.canvas.height]); + } + }; + drawScene = (timestamp: number) => { + const { gl } = this; + const { lerp = 1 } = this.props; + if (!gl) return; + gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight); + gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); + gl.bindBuffer(gl.ARRAY_BUFFER, this.squareVerticesBuffer ?? null); + gl.vertexAttribPointer(this.vertexPositionAttribute ?? 0, 3, gl.FLOAT, false, 0, 0); + this.setUniforms(timestamp); + gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); + if (this.uniforms.iMouse?.isNeeded && lerp !== 1) { + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[0] = lerpVal( + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[0], + this.lastMouseArr[0], + lerp + ); + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[1] = lerpVal( + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[1], + this.lastMouseArr[1], + lerp + ); + } + this.animFrameId = requestAnimationFrame(this.drawScene); + }; + createShader = (type: number, shaderCodeAsText: string) => { + const { gl } = this; + if (!gl) return null; + const shader = gl.createShader(type); + if (!shader) return null; + gl.shaderSource(shader, shaderCodeAsText); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + this.props.onWarning?.(log(`Error compiling the shader:\n${shaderCodeAsText}`)); + const compilationLog = gl.getShaderInfoLog(shader); + gl.deleteShader(shader); + this.props.onError?.(log(`Shader compiler log: ${compilationLog}`)); + } + return shader; + }; + initShaders = (fs: string, vs: string) => { + const { gl } = this; + if (!gl) return; + // console.log(fs, vs); + const fragmentShader = this.createShader(gl.FRAGMENT_SHADER, fs); + const vertexShader = this.createShader(gl.VERTEX_SHADER, vs); + this.shaderProgram = gl.createProgram(); + if (!this.shaderProgram || !vertexShader || !fragmentShader) return; + gl.attachShader(this.shaderProgram, vertexShader); + gl.attachShader(this.shaderProgram, fragmentShader); + gl.linkProgram(this.shaderProgram); + if (!gl.getProgramParameter(this.shaderProgram, gl.LINK_STATUS)) { + this.props.onError?.( + log(`Unable to initialize the shader program: ${gl.getProgramInfoLog(this.shaderProgram)}`) + ); + return; + } + gl.useProgram(this.shaderProgram); + this.vertexPositionAttribute = gl.getAttribLocation(this.shaderProgram, 'aVertexPosition'); + gl.enableVertexAttribArray(this.vertexPositionAttribute); + }; + processCustomUniforms = () => { + const { uniforms } = this.props; + if (uniforms) { + for (const name of Object.keys(uniforms)) { + const uniform = this.props.uniforms?.[name]; + if (!uniform) return; + const { value, type } = uniform; + const glslType = uniformTypeToGLSLType(type); + if (!glslType) return; + const tempObject: { arraySize?: string } = {}; + if (isMatrixType(type, value)) { + const arrayLength = type.length; + const val = Number.parseInt(type.charAt(arrayLength - 3)); + const numberOfMatrices = Math.floor(value.length / (val * val)); + if (value.length > val * val) tempObject.arraySize = `[${numberOfMatrices}]`; + } else if (isVectorListType(type, value)) { + tempObject.arraySize = `[${Math.floor(value.length / Number.parseInt(type.charAt(0)))}]`; + } + this.uniforms[name] = { type: glslType, isNeeded: false, value, ...tempObject }; + } + } + }; + processTextures = () => { + const { gl } = this; + const { textures, onDoneLoadingTextures } = this.props; + if (!gl) return; + if (textures && textures.length > 0) { + this.uniforms[`${UNIFORM_CHANNELRESOLUTION}`] = { + type: 'vec3', + isNeeded: false, + arraySize: `[${textures.length}]`, + value: [], + }; + const texturePromisesArr = textures.map((texture: TextureParams, id: number) => { + // Dynamically add textures uniforms. + this.uniforms[`${UNIFORM_CHANNEL}${id}`] = { + type: 'sampler2D', + isNeeded: false, + }; + // Initialize array with 0s: + // @ts-expect-error TODO: Deal with this. + this.setupChannelRes(texture, id); + this.texturesArr[id] = new Texture(gl); + return ( + this.texturesArr[id] + // @ts-expect-error TODO: Deal with this. + ?.load(texture, id) + .then((t: Texture) => { + this.setupChannelRes(t, id); + }) + ); + }); + Promise.all(texturePromisesArr) + .then(() => { + if (onDoneLoadingTextures) onDoneLoadingTextures(); + }) + .catch((e) => { + this.props.onError?.(e); + if (onDoneLoadingTextures) onDoneLoadingTextures(); + }); + } else if (onDoneLoadingTextures) onDoneLoadingTextures(); + }; + preProcessFragment = (fragment: string) => { + const { precision, devicePixelRatio = 1 } = this.props; + const isValidPrecision = PRECISIONS.includes(precision ?? 'highp'); + const precisionString = `precision ${isValidPrecision ? precision : PRECISIONS[1]} float;\n`; + if (!isValidPrecision) { + this.props.onWarning?.( + log( + `wrong precision type ${precision}, please make sure to pass one of a valid precision lowp, mediump, highp, by default you shader precision will be set to highp.` + ) + ); + } + let fs = precisionString + .concat(`#define DPR ${devicePixelRatio.toFixed(1)}\n`) + .concat(fragment.replace(/texture\(/g, 'texture2D(')); + for (const uniform of Object.keys(this.uniforms)) { + if (fragment.includes(uniform)) { + const u = this.uniforms[uniform]; + if (!u) continue; + fs = insertStringAtIndex( + fs, + `uniform ${u.type} ${uniform}${u.arraySize || ''}; \n`, + fs.lastIndexOf(precisionString) + precisionString.length + ); + u.isNeeded = true; + } + } + const isShadertoy = fragment.includes('mainImage'); + if (isShadertoy) fs = fs.concat(FS_MAIN_SHADER); + return fs; + }; + setUniforms = (timestamp: number) => { + const { gl } = this; + if (!gl || !this.shaderProgram) return; + const delta = this.lastTime ? (timestamp - this.lastTime) / 1000 : 0; + this.lastTime = timestamp; + if (this.props.uniforms) { + for (const name of Object.keys(this.props.uniforms)) { + const currentUniform = this.props.uniforms?.[name]; + if (!currentUniform) return; + if (this.uniforms[name]?.isNeeded) { + if (!this.shaderProgram) return; + const customUniformLocation = gl.getUniformLocation(this.shaderProgram, name); + if (!customUniformLocation) return; + processUniform( + gl, + customUniformLocation, + currentUniform.type as UniformType, + currentUniform.value + ); + } + } + } + if (this.uniforms.iMouse?.isNeeded) { + const mouseUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_MOUSE); + gl.uniform4fv(mouseUniform, this.uniforms.iMouse.value as number[]); + } + if (this.uniforms.iChannelResolution?.isNeeded) { + const channelResUniform = gl.getUniformLocation( + this.shaderProgram, + UNIFORM_CHANNELRESOLUTION + ); + gl.uniform3fv(channelResUniform, this.uniforms.iChannelResolution.value as number[]); + } + if (this.uniforms.iDeviceOrientation?.isNeeded) { + const deviceOrientationUniform = gl.getUniformLocation( + this.shaderProgram, + UNIFORM_DEVICEORIENTATION + ); + gl.uniform4fv(deviceOrientationUniform, this.uniforms.iDeviceOrientation.value as number[]); + } + if (this.uniforms.iTime?.isNeeded) { + const timeUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_TIME); + gl.uniform1f(timeUniform, (this.timer += delta)); + } + if (this.uniforms.iTimeDelta?.isNeeded) { + const timeDeltaUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_TIMEDELTA); + gl.uniform1f(timeDeltaUniform, delta); + } + if (this.uniforms.iDate?.isNeeded) { + const d = new Date(); + const month = d.getMonth() + 1; + const day = d.getDate(); + const year = d.getFullYear(); + const time = + d.getHours() * 60 * 60 + d.getMinutes() * 60 + d.getSeconds() + d.getMilliseconds() * 0.001; + const dateUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_DATE); + gl.uniform4fv(dateUniform, [year, month, day, time]); + } + if (this.uniforms.iFrame?.isNeeded) { + const timeDeltaUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_FRAME); + gl.uniform1i(timeDeltaUniform, (this.uniforms.iFrame.value as number)++); + } + if (this.texturesArr.length > 0) { + for (let index = 0; index < this.texturesArr.length; index++) { + // TODO: Don't use this casting if possible: + const texture = this.texturesArr[index] as Texture | undefined; + if (!texture) return; + const { isVideo, _webglTexture, source, flipY, isLoaded } = texture; + if (!isLoaded || !_webglTexture || !source) return; + if (this.uniforms[`iChannel${index}`]?.isNeeded) { + if (!this.shaderProgram) return; + const iChannel = gl.getUniformLocation(this.shaderProgram, `iChannel${index}`); + // @ts-expect-error TODO: Fix. Can't index WebGL context with this dynamic value. + gl.activeTexture(gl[`TEXTURE${index}`]); + gl.bindTexture(gl.TEXTURE_2D, _webglTexture); + gl.uniform1i(iChannel, index); + if (isVideo) { + texture.updateTexture(_webglTexture, source as HTMLVideoElement, flipY); + } + } + } + } + }; + registerCanvas = (r: HTMLCanvasElement) => { + this.canvas = r; + }; + gl?: WebGLRenderingContext | null; + squareVerticesBuffer?: WebGLBuffer | null; + shaderProgram?: WebGLProgram | null; + vertexPositionAttribute?: number; + animFrameId?: number; + canvas?: HTMLCanvasElement; + mousedown = false; + canvasPosition?: DOMRect; + timer = 0; + lastMouseArr: number[] = [0, 0]; + texturesArr: WebGLTexture[] = []; + lastTime = 0; + resizeObserver?: ResizeObserver; + render = () => ( + + ); +} diff --git a/components/livekit/react-shader/react-shader.tsx b/components/livekit/react-shader/react-shader.tsx new file mode 100644 index 000000000..b5ea8b76a --- /dev/null +++ b/components/livekit/react-shader/react-shader.tsx @@ -0,0 +1,649 @@ +import { type CSSProperties, useEffect, useRef } from 'react'; +import { log } from './logging'; +import { + ClampToEdgeWrapping, + LinearFilter, + LinearMipMapLinearFilter, + LinearMipMapNearestFilter, + MirroredRepeatWrapping, + NearestFilter, + NearestMipMapLinearFilter, + NearestMipMapNearestFilter, + RepeatWrapping, + Texture, +} from './texture'; +import { + type UniformType, + type Vector2, + type Vector3, + type Vector4, + isMatrixType, + isVectorListType, + processUniform, + uniformTypeToGLSLType, +} from './uniforms'; + +export { + ClampToEdgeWrapping, + LinearFilter, + LinearMipMapLinearFilter, + LinearMipMapNearestFilter, + MirroredRepeatWrapping, + NearestFilter, + NearestMipMapLinearFilter, + NearestMipMapNearestFilter, + RepeatWrapping, +}; + +export type { Vector2, Vector3, Vector4 }; + +const PRECISIONS = ['lowp', 'mediump', 'highp']; +const FS_MAIN_SHADER = `\nvoid main(void){ + vec4 color = vec4(0.0,0.0,0.0,1.0); + mainImage( color, gl_FragCoord.xy ); + gl_FragColor = color; +}`; +const BASIC_FS = `void mainImage( out vec4 fragColor, in vec2 fragCoord ) { + vec2 uv = fragCoord/iResolution.xy; + vec3 col = 0.5 + 0.5*cos(iTime+uv.xyx+vec3(0,2,4)); + fragColor = vec4(col,1.0); +}`; +const BASIC_VS = `attribute vec3 aVertexPosition; +void main(void) { + gl_Position = vec4(aVertexPosition, 1.0); +}`; +const UNIFORM_TIME = 'iTime'; +const UNIFORM_TIMEDELTA = 'iTimeDelta'; +const UNIFORM_DATE = 'iDate'; +const UNIFORM_FRAME = 'iFrame'; +const UNIFORM_MOUSE = 'iMouse'; +const UNIFORM_RESOLUTION = 'iResolution'; +const UNIFORM_CHANNEL = 'iChannel'; +const UNIFORM_CHANNELRESOLUTION = 'iChannelResolution'; +const UNIFORM_DEVICEORIENTATION = 'iDeviceOrientation'; + +const latestPointerClientCoords = (e: MouseEvent | TouchEvent) => { + // @ts-expect-error TODO: Deal with this. + return [e.clientX || e.changedTouches[0].clientX, e.clientY || e.changedTouches[0].clientY]; +}; +const lerpVal = (v0: number, v1: number, t: number) => v0 * (1 - t) + v1 * t; +const insertStringAtIndex = (currentString: string, string: string, index: number) => + index > 0 + ? currentString.substring(0, index) + + string + + currentString.substring(index, currentString.length) + : string + currentString; + +type Uniform = { type: string; value: number[] | number }; +export type Uniforms = Record; +type TextureParams = { + url: string; + wrapS?: number; + wrapT?: number; + minFilter?: number; + magFilter?: number; + flipY?: number; +}; + +type Props = { + /** Fragment shader GLSL code. */ + fs: string; + + /** Vertex shader GLSL code. */ + vs?: string; + + /** + * Textures to be passed to the shader. Textures need to be squared or will be + * automatically resized. + * + * Options default to: + * + * ```js + * { + * minFilter: LinearMipMapLinearFilter, + * magFilter: LinearFilter, + * wrapS: RepeatWrapping, + * wrapT: RepeatWrapping, + * } + * ``` + * + * See [textures in the docs](https://rysana.com/docs/react-shaders#textures) + * for details. + */ + textures?: TextureParams[]; + + /** + * Custom uniforms to be passed to the shader. + * + * See [custom uniforms in the + * docs](https://rysana.com/docs/react-shaders#custom-uniforms) for details. + */ + uniforms?: Uniforms; + + /** + * Color used when clearing the canvas. + * + * See [the WebGL + * docs](https://developer.mozilla.org/en-US/docs/Web/API/WebGLRenderingContext/clearColor) + * for details. + */ + clearColor?: Vector4; + + /** + * GLSL precision qualifier. Defaults to `'highp'`. Balance between + * performance and quality. + */ + precision?: 'highp' | 'lowp' | 'mediump'; + + /** Custom inline style for canvas. */ + style?: CSSStyleDeclaration; + + /** Customize WebGL context attributes. See [the WebGL docs](https://developer.mozilla.org/en-US/docs/Web/API/WebGLRenderingContext/getContextAttributes) for details. */ + contextAttributes?: Record; + + /** Lerp value for `iMouse` built-in uniform. Must be between 0 and 1. */ + lerp?: number; + + /** Device pixel ratio. */ + devicePixelRatio?: number; + + /** + * Callback for when the textures are done loading. Useful if you want to do + * something like e.g. hide the canvas until textures are done loading. + */ + onDoneLoadingTextures?: () => void; + + /** Custom callback to handle errors. Defaults to `console.error`. */ + onError?: (error: string) => void; + + /** Custom callback to handle warnings. Defaults to `console.warn`. */ + onWarning?: (warning: string) => void; +}; +export const Shader = ({ + fs, + vs = BASIC_VS, + textures = [], + uniforms: propUniforms, + clearColor = [0, 0, 0, 1], + precision = 'highp', + style, + contextAttributes = {}, + lerp = 1, + devicePixelRatio = 1, + onDoneLoadingTextures, + onError = console.error, + onWarning = console.warn, +}: Props) => { + // Refs for WebGL state + const canvasRef = useRef(null); + const glRef = useRef(null); + const squareVerticesBufferRef = useRef(null); + const shaderProgramRef = useRef(null); + const vertexPositionAttributeRef = useRef(undefined); + const animFrameIdRef = useRef(undefined); + const mousedownRef = useRef(false); + const canvasPositionRef = useRef(undefined); + const timerRef = useRef(0); + const lastMouseArrRef = useRef([0, 0]); + const texturesArrRef = useRef([]); + const lastTimeRef = useRef(0); + const resizeObserverRef = useRef(undefined); + const uniformsRef = useRef< + Record< + string, + { type: string; isNeeded: boolean; value?: number[] | number; arraySize?: string } + > + >({ + [UNIFORM_TIME]: { type: 'float', isNeeded: false, value: 0 }, + [UNIFORM_TIMEDELTA]: { type: 'float', isNeeded: false, value: 0 }, + [UNIFORM_DATE]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + [UNIFORM_MOUSE]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + [UNIFORM_RESOLUTION]: { type: 'vec2', isNeeded: false, value: [0, 0] }, + [UNIFORM_FRAME]: { type: 'int', isNeeded: false, value: 0 }, + [UNIFORM_DEVICEORIENTATION]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + }); + const propsUniformsRef = useRef(propUniforms); + + const setupChannelRes = ({ width, height }: Texture, id: number) => { + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iChannelResolution.value[id * 3] = width * devicePixelRatio; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iChannelResolution.value[id * 3 + 1] = height * devicePixelRatio; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iChannelResolution.value[id * 3 + 2] = 0; + }; + + const initWebGL = () => { + if (!canvasRef.current) return; + glRef.current = (canvasRef.current.getContext('webgl', contextAttributes) || + canvasRef.current.getContext( + 'experimental-webgl', + contextAttributes + )) as WebGLRenderingContext | null; + glRef.current?.getExtension('OES_standard_derivatives'); + glRef.current?.getExtension('EXT_shader_texture_lod'); + }; + + const initBuffers = () => { + const gl = glRef.current; + squareVerticesBufferRef.current = gl?.createBuffer() ?? null; + gl?.bindBuffer(gl.ARRAY_BUFFER, squareVerticesBufferRef.current); + const vertices = [1.0, 1.0, 0.0, -1.0, 1.0, 0.0, 1.0, -1.0, 0.0, -1.0, -1.0, 0.0]; + gl?.bufferData(gl.ARRAY_BUFFER, new Float32Array(vertices), gl.STATIC_DRAW); + }; + + const onDeviceOrientationChange = ({ alpha, beta, gamma }: DeviceOrientationEvent) => { + uniformsRef.current.iDeviceOrientation.value = [ + alpha ?? 0, + beta ?? 0, + gamma ?? 0, + window.orientation || 0, + ]; + }; + + const mouseDown = (e: MouseEvent | TouchEvent) => { + const [clientX, clientY] = latestPointerClientCoords(e); + const mouseX = clientX - (canvasPositionRef.current?.left ?? 0) - window.pageXOffset; + const mouseY = + (canvasPositionRef.current?.height ?? 0) - + clientY - + (canvasPositionRef.current?.top ?? 0) - + window.pageYOffset; + mousedownRef.current = true; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[2] = mouseX; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[3] = mouseY; + lastMouseArrRef.current[0] = mouseX; + lastMouseArrRef.current[1] = mouseY; + }; + + const mouseMove = (e: MouseEvent | TouchEvent) => { + canvasPositionRef.current = canvasRef.current?.getBoundingClientRect(); + const [clientX, clientY] = latestPointerClientCoords(e); + const mouseX = clientX - (canvasPositionRef.current?.left ?? 0); + const mouseY = + (canvasPositionRef.current?.height ?? 0) - clientY - (canvasPositionRef.current?.top ?? 0); + if (lerp !== 1) { + lastMouseArrRef.current[0] = mouseX; + lastMouseArrRef.current[1] = mouseY; + } else { + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[0] = mouseX; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[1] = mouseY; + } + }; + + const mouseUp = () => { + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[2] = 0; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[3] = 0; + }; + + const onResize = () => { + const gl = glRef.current; + if (!gl) return; + canvasPositionRef.current = canvasRef.current?.getBoundingClientRect(); + // Force pixel ratio to be one to avoid expensive calculus on retina display. + const realToCSSPixels = devicePixelRatio; + const displayWidth = Math.floor((canvasPositionRef.current?.width ?? 1) * realToCSSPixels); + const displayHeight = Math.floor((canvasPositionRef.current?.height ?? 1) * realToCSSPixels); + gl.canvas.width = displayWidth; + gl.canvas.height = displayHeight; + if (uniformsRef.current.iResolution?.isNeeded && shaderProgramRef.current) { + const rUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_RESOLUTION); + gl.uniform2fv(rUniform, [gl.canvas.width, gl.canvas.height]); + } + }; + + const createShader = (type: number, shaderCodeAsText: string) => { + const gl = glRef.current; + if (!gl) return null; + const shader = gl.createShader(type); + if (!shader) return null; + gl.shaderSource(shader, shaderCodeAsText); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + onWarning?.(log(`Error compiling the shader:\n${shaderCodeAsText}`)); + const compilationLog = gl.getShaderInfoLog(shader); + gl.deleteShader(shader); + onError?.(log(`Shader compiler log: ${compilationLog}`)); + } + return shader; + }; + + const initShaders = (fragmentShader: string, vertexShader: string) => { + const gl = glRef.current; + if (!gl) return; + const fragmentShaderObj = createShader(gl.FRAGMENT_SHADER, fragmentShader); + const vertexShaderObj = createShader(gl.VERTEX_SHADER, vertexShader); + shaderProgramRef.current = gl.createProgram(); + if (!shaderProgramRef.current || !vertexShaderObj || !fragmentShaderObj) return; + gl.attachShader(shaderProgramRef.current, vertexShaderObj); + gl.attachShader(shaderProgramRef.current, fragmentShaderObj); + gl.linkProgram(shaderProgramRef.current); + if (!gl.getProgramParameter(shaderProgramRef.current, gl.LINK_STATUS)) { + onError?.( + log( + `Unable to initialize the shader program: ${gl.getProgramInfoLog(shaderProgramRef.current)}` + ) + ); + return; + } + gl.useProgram(shaderProgramRef.current); + vertexPositionAttributeRef.current = gl.getAttribLocation( + shaderProgramRef.current, + 'aVertexPosition' + ); + gl.enableVertexAttribArray(vertexPositionAttributeRef.current); + }; + + const processCustomUniforms = () => { + if (propUniforms) { + for (const name of Object.keys(propUniforms)) { + const uniform = propUniforms[name]; + if (!uniform) return; + const { value, type } = uniform; + const glslType = uniformTypeToGLSLType(type); + if (!glslType) return; + const tempObject: { arraySize?: string } = {}; + if (isMatrixType(type, value)) { + const arrayLength = type.length; + const val = Number.parseInt(type.charAt(arrayLength - 3)); + const numberOfMatrices = Math.floor(value.length / (val * val)); + if (value.length > val * val) tempObject.arraySize = `[${numberOfMatrices}]`; + } else if (isVectorListType(type, value)) { + tempObject.arraySize = `[${Math.floor(value.length / Number.parseInt(type.charAt(0)))}]`; + } + uniformsRef.current[name] = { type: glslType, isNeeded: false, value, ...tempObject }; + } + } + }; + + const processTextures = () => { + const gl = glRef.current; + if (!gl) return; + if (textures && textures.length > 0) { + uniformsRef.current[`${UNIFORM_CHANNELRESOLUTION}`] = { + type: 'vec3', + isNeeded: false, + arraySize: `[${textures.length}]`, + value: [], + }; + const texturePromisesArr = textures.map((texture: TextureParams, id: number) => { + // Dynamically add textures uniforms. + uniformsRef.current[`${UNIFORM_CHANNEL}${id}`] = { + type: 'sampler2D', + isNeeded: false, + }; + // Initialize array with 0s: + // @ts-expect-error TODO: Deal with this. + setupChannelRes(texture, id); + texturesArrRef.current[id] = new Texture(gl); + return ( + texturesArrRef.current[id] + // @ts-expect-error TODO: Deal with this. + ?.load(texture, id) + .then((t: Texture) => { + setupChannelRes(t, id); + }) + ); + }); + Promise.all(texturePromisesArr) + .then(() => { + if (onDoneLoadingTextures) onDoneLoadingTextures(); + }) + .catch((e) => { + onError?.(e); + if (onDoneLoadingTextures) onDoneLoadingTextures(); + }); + } else if (onDoneLoadingTextures) onDoneLoadingTextures(); + }; + + const preProcessFragment = (fragment: string) => { + const isValidPrecision = PRECISIONS.includes(precision ?? 'highp'); + const precisionString = `precision ${isValidPrecision ? precision : PRECISIONS[1]} float;\n`; + if (!isValidPrecision) { + onWarning?.( + log( + `wrong precision type ${precision}, please make sure to pass one of a valid precision lowp, mediump, highp, by default you shader precision will be set to highp.` + ) + ); + } + let fragmentShader = precisionString + .concat(`#define DPR ${devicePixelRatio.toFixed(1)}\n`) + .concat(fragment.replace(/texture\(/g, 'texture2D(')); + for (const uniform of Object.keys(uniformsRef.current)) { + if (fragment.includes(uniform)) { + const u = uniformsRef.current[uniform]; + if (!u) continue; + fragmentShader = insertStringAtIndex( + fragmentShader, + `uniform ${u.type} ${uniform}${u.arraySize || ''}; \n`, + fragmentShader.lastIndexOf(precisionString) + precisionString.length + ); + u.isNeeded = true; + } + } + const isShadertoy = fragment.includes('mainImage'); + if (isShadertoy) fragmentShader = fragmentShader.concat(FS_MAIN_SHADER); + return fragmentShader; + }; + + const setUniforms = (timestamp: number) => { + const gl = glRef.current; + if (!gl || !shaderProgramRef.current) return; + const delta = lastTimeRef.current ? (timestamp - lastTimeRef.current) / 1000 : 0; + lastTimeRef.current = timestamp; + const propUniforms = propsUniformsRef.current; + if (propUniforms) { + for (const name of Object.keys(propUniforms)) { + const currentUniform = propUniforms[name]; + if (!currentUniform) return; + if (uniformsRef.current[name]?.isNeeded) { + if (!shaderProgramRef.current) return; + const customUniformLocation = gl.getUniformLocation(shaderProgramRef.current, name); + if (!customUniformLocation) return; + processUniform( + gl, + customUniformLocation, + currentUniform.type as UniformType, + currentUniform.value + ); + } + } + } + if (uniformsRef.current.iMouse?.isNeeded) { + const mouseUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_MOUSE); + gl.uniform4fv(mouseUniform, uniformsRef.current.iMouse.value as number[]); + } + if (uniformsRef.current.iChannelResolution?.isNeeded) { + const channelResUniform = gl.getUniformLocation( + shaderProgramRef.current, + UNIFORM_CHANNELRESOLUTION + ); + gl.uniform3fv(channelResUniform, uniformsRef.current.iChannelResolution.value as number[]); + } + if (uniformsRef.current.iDeviceOrientation?.isNeeded) { + const deviceOrientationUniform = gl.getUniformLocation( + shaderProgramRef.current, + UNIFORM_DEVICEORIENTATION + ); + gl.uniform4fv( + deviceOrientationUniform, + uniformsRef.current.iDeviceOrientation.value as number[] + ); + } + if (uniformsRef.current.iTime?.isNeeded) { + const timeUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_TIME); + gl.uniform1f(timeUniform, (timerRef.current += delta)); + } + if (uniformsRef.current.iTimeDelta?.isNeeded) { + const timeDeltaUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_TIMEDELTA); + gl.uniform1f(timeDeltaUniform, delta); + } + if (uniformsRef.current.iDate?.isNeeded) { + const d = new Date(); + const month = d.getMonth() + 1; + const day = d.getDate(); + const year = d.getFullYear(); + const time = + d.getHours() * 60 * 60 + d.getMinutes() * 60 + d.getSeconds() + d.getMilliseconds() * 0.001; + const dateUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_DATE); + gl.uniform4fv(dateUniform, [year, month, day, time]); + } + if (uniformsRef.current.iFrame?.isNeeded) { + const timeDeltaUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_FRAME); + gl.uniform1i(timeDeltaUniform, (uniformsRef.current.iFrame.value as number)++); + } + if (texturesArrRef.current.length > 0) { + for (let index = 0; index < texturesArrRef.current.length; index++) { + // TODO: Don't use this casting if possible: + const texture = texturesArrRef.current[index] as Texture | undefined; + if (!texture) return; + const { isVideo, _webglTexture, source, flipY, isLoaded } = texture; + if (!isLoaded || !_webglTexture || !source) return; + if (uniformsRef.current[`iChannel${index}`]?.isNeeded) { + if (!shaderProgramRef.current) return; + const iChannel = gl.getUniformLocation(shaderProgramRef.current, `iChannel${index}`); + // @ts-expect-error TODO: Fix. Can't index WebGL context with this dynamic value. + gl.activeTexture(gl[`TEXTURE${index}`]); + gl.bindTexture(gl.TEXTURE_2D, _webglTexture); + gl.uniform1i(iChannel, index); + if (isVideo) { + texture.updateTexture(_webglTexture, source as HTMLVideoElement, flipY); + } + } + } + } + }; + + const drawScene = (timestamp: number) => { + const gl = glRef.current; + if (!gl) return; + gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight); + gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); + gl.bindBuffer(gl.ARRAY_BUFFER, squareVerticesBufferRef.current); + gl.vertexAttribPointer(vertexPositionAttributeRef.current ?? 0, 3, gl.FLOAT, false, 0, 0); + setUniforms(timestamp); + gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); + if (uniformsRef.current.iMouse?.isNeeded && lerp !== 1) { + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[0] = lerpVal( + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[0], + lastMouseArrRef.current[0], + lerp + ); + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[1] = lerpVal( + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[1], + lastMouseArrRef.current[1], + lerp + ); + } + animFrameIdRef.current = requestAnimationFrame(drawScene); + }; + + const addEventListeners = () => { + const options = { passive: true }; + if (uniformsRef.current.iMouse?.isNeeded && canvasRef.current) { + canvasRef.current.addEventListener('mousemove', mouseMove, options); + canvasRef.current.addEventListener('mouseout', mouseUp, options); + canvasRef.current.addEventListener('mouseup', mouseUp, options); + canvasRef.current.addEventListener('mousedown', mouseDown, options); + canvasRef.current.addEventListener('touchmove', mouseMove, options); + canvasRef.current.addEventListener('touchend', mouseUp, options); + canvasRef.current.addEventListener('touchstart', mouseDown, options); + } + if (uniformsRef.current.iDeviceOrientation?.isNeeded) { + window.addEventListener('deviceorientation', onDeviceOrientationChange, options); + } + if (canvasRef.current) { + resizeObserverRef.current = new ResizeObserver(onResize); + resizeObserverRef.current.observe(canvasRef.current); + window.addEventListener('resize', onResize, options); + } + }; + + const removeEventListeners = () => { + const options = { passive: true } as EventListenerOptions; + if (uniformsRef.current.iMouse?.isNeeded && canvasRef.current) { + canvasRef.current.removeEventListener('mousemove', mouseMove, options); + canvasRef.current.removeEventListener('mouseout', mouseUp, options); + canvasRef.current.removeEventListener('mouseup', mouseUp, options); + canvasRef.current.removeEventListener('mousedown', mouseDown, options); + canvasRef.current.removeEventListener('touchmove', mouseMove, options); + canvasRef.current.removeEventListener('touchend', mouseUp, options); + canvasRef.current.removeEventListener('touchstart', mouseDown, options); + } + if (uniformsRef.current.iDeviceOrientation?.isNeeded) { + window.removeEventListener('deviceorientation', onDeviceOrientationChange, options); + } + if (resizeObserverRef.current) { + resizeObserverRef.current.disconnect(); + window.removeEventListener('resize', onResize, options); + } + }; + + useEffect(() => { + propsUniformsRef.current = propUniforms; + }, [propUniforms]); + + // Main effect for initialization and cleanup + useEffect(() => { + const textures = texturesArrRef.current; + + function init() { + initWebGL(); + const gl = glRef.current; + if (gl && canvasRef.current) { + gl.clearColor(...clearColor); + gl.clearDepth(1.0); + gl.enable(gl.DEPTH_TEST); + gl.depthFunc(gl.LEQUAL); + gl.viewport(0, 0, canvasRef.current.width, canvasRef.current.height); + console.log('canvasRef.current', canvasRef.current); + console.log('canvasRef.current.width', canvasRef.current.width); + console.log('canvasRef.current.height', canvasRef.current.height); + canvasRef.current.height = canvasRef.current.clientHeight; + canvasRef.current.width = canvasRef.current.clientWidth; + processCustomUniforms(); + processTextures(); + initShaders(preProcessFragment(fs || BASIC_FS), vs || BASIC_VS); + initBuffers(); + requestAnimationFrame(drawScene); + addEventListeners(); + onResize(); + } + } + + requestAnimationFrame(init); + + // Cleanup function + return () => { + const gl = glRef.current; + if (gl) { + gl.getExtension('WEBGL_lose_context')?.loseContext(); + gl.useProgram(null); + gl.deleteProgram(shaderProgramRef.current ?? null); + if (textures.length > 0) { + for (const texture of textures as Texture[]) { + gl.deleteTexture(texture._webglTexture); + } + } + shaderProgramRef.current = null; + } + removeEventListeners(); + cancelAnimationFrame(animFrameIdRef.current ?? 0); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); // Empty dependency array to run only once on mount + + return ( + + ); +}; diff --git a/components/livekit/react-shader/texture.ts b/components/livekit/react-shader/texture.ts new file mode 100644 index 000000000..19282c177 --- /dev/null +++ b/components/livekit/react-shader/texture.ts @@ -0,0 +1,197 @@ +import { log } from './logging'; + +export const LinearFilter = 9729; +export const NearestFilter = 9728; +export const LinearMipMapLinearFilter = 9987; +export const NearestMipMapLinearFilter = 9986; +export const LinearMipMapNearestFilter = 9985; +export const NearestMipMapNearestFilter = 9984; +export const MirroredRepeatWrapping = 33648; +export const ClampToEdgeWrapping = 33071; +export const RepeatWrapping = 10497; + +export class Texture { + gl: WebGLRenderingContext; + url?: string; + wrapS?: number; + wrapT?: number; + minFilter?: number; + magFilter?: number; + source?: HTMLImageElement | HTMLVideoElement; + pow2canvas?: HTMLCanvasElement; + isLoaded = false; + isVideo = false; + flipY = -1; + width = 0; + height = 0; + _webglTexture: WebGLTexture | null = null; + constructor(gl: WebGLRenderingContext) { + this.gl = gl; + } + updateTexture = (texture: WebGLTexture, video: HTMLVideoElement, flipY: number) => { + const { gl } = this; + const level = 0; + const internalFormat = gl.RGBA; + const srcFormat = gl.RGBA; + const srcType = gl.UNSIGNED_BYTE; + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, flipY); + gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, srcFormat, srcType, video); + }; + setupVideo = (url: string) => { + const video = document.createElement('video'); + let playing = false; + let timeupdate = false; + video.autoplay = true; + video.muted = true; + video.loop = true; + video.crossOrigin = 'anonymous'; + const checkReady = () => { + if (playing && timeupdate) { + this.isLoaded = true; + } + }; + video.addEventListener( + 'playing', + () => { + playing = true; + this.width = video.videoWidth || 0; + this.height = video.videoHeight || 0; + checkReady(); + }, + true + ); + video.addEventListener( + 'timeupdate', + () => { + timeupdate = true; + checkReady(); + }, + true + ); + video.src = url; + // video.play(); // Not sure why this is here nor commented out. From STR. + return video; + }; + makePowerOf2 = (image: T): T => { + if ( + image instanceof HTMLImageElement || + image instanceof HTMLCanvasElement || + image instanceof ImageBitmap + ) { + if (this.pow2canvas === undefined) this.pow2canvas = document.createElement('canvas'); + this.pow2canvas.width = 2 ** Math.floor(Math.log(image.width) / Math.LN2); + this.pow2canvas.height = 2 ** Math.floor(Math.log(image.height) / Math.LN2); + const context = this.pow2canvas.getContext('2d'); + context?.drawImage(image, 0, 0, this.pow2canvas.width, this.pow2canvas.height); + console.warn( + log( + `Image is not power of two ${image.width} x ${image.height}. Resized to ${this.pow2canvas.width} x ${this.pow2canvas.height};` + ) + ); + return this.pow2canvas as T; + } + return image; + }; + load = async ( + textureArgs: Texture + // channelId: number // Not sure why this is here nor commented out. From STR. + ) => { + const { gl } = this; + const { url, wrapS, wrapT, minFilter, magFilter, flipY = -1 }: Texture = textureArgs; + if (!url) { + return Promise.reject( + new Error(log('Missing url, please make sure to pass the url of your texture { url: ... }')) + ); + } + const isImage = /(\.jpg|\.jpeg|\.png|\.gif|\.bmp)$/i.exec(url); + const isVideo = /(\.mp4|\.3gp|\.webm|\.ogv)$/i.exec(url); + if (isImage === null && isVideo === null) { + return Promise.reject( + new Error(log(`Please upload a video or an image with a valid format (url: ${url})`)) + ); + } + Object.assign(this, { url, wrapS, wrapT, minFilter, magFilter, flipY }); + const level = 0; + const internalFormat = gl.RGBA; + const width = 1; + const height = 1; + const border = 0; + const srcFormat = gl.RGBA; + const srcType = gl.UNSIGNED_BYTE; + const pixel = new Uint8Array([255, 255, 255, 0]); + const texture = gl.createTexture(); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texImage2D( + gl.TEXTURE_2D, + level, + internalFormat, + width, + height, + border, + srcFormat, + srcType, + pixel + ); + if (isVideo) { + const video = this.setupVideo(url); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + this._webglTexture = texture; + this.source = video; + this.isVideo = true; + return video.play().then(() => this); + } + async function loadImage() { + return new Promise((resolve, reject) => { + const image = new Image(); + image.crossOrigin = 'anonymous'; + image.onload = () => { + resolve(image); + }; + image.onerror = () => { + reject(new Error(log(`failed loading url: ${url}`))); + }; + image.src = url ?? ''; + }); + } + let image = (await loadImage()) as HTMLImageElement; + let isPowerOf2 = + (image.width & (image.width - 1)) === 0 && (image.height & (image.height - 1)) === 0; + if ( + (textureArgs.wrapS !== ClampToEdgeWrapping || + textureArgs.wrapT !== ClampToEdgeWrapping || + (textureArgs.minFilter !== NearestFilter && textureArgs.minFilter !== LinearFilter)) && + !isPowerOf2 + ) { + image = this.makePowerOf2(image); + isPowerOf2 = true; + } + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, flipY); + gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, srcFormat, srcType, image); + if ( + isPowerOf2 && + textureArgs.minFilter !== NearestFilter && + textureArgs.minFilter !== LinearFilter + ) { + gl.generateMipmap(gl.TEXTURE_2D); + } + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, this.wrapS || RepeatWrapping); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, this.wrapT || RepeatWrapping); + gl.texParameteri( + gl.TEXTURE_2D, + gl.TEXTURE_MIN_FILTER, + this.minFilter || LinearMipMapLinearFilter + ); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, this.magFilter || LinearFilter); + this._webglTexture = texture; + this.source = image; + this.isVideo = false; + this.isLoaded = true; + this.width = image.width || 0; + this.height = image.height || 0; + return this; + }; +} diff --git a/components/livekit/react-shader/uniforms.ts b/components/livekit/react-shader/uniforms.ts new file mode 100644 index 000000000..39405a9a0 --- /dev/null +++ b/components/livekit/react-shader/uniforms.ts @@ -0,0 +1,145 @@ +import { log } from './logging'; + +export type Vector2 = [T, T]; +export type Vector3 = [T, T, T]; +export type Vector4 = [T, T, T, T]; +// biome-ignore format: +export type Matrix2 = [T, T, T, T]; +// biome-ignore format: +export type Matrix3 = [T, T, T, T, T, T, T, T, T]; +// biome-ignore format: +export type Matrix4 = [T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T]; +export type Uniforms = { + '1i': number; + '2i': Vector2; + '3i': Vector3; + '4i': Vector4; + '1f': number; + '2f': Vector2; + '3f': Vector3; + '4f': Vector4; + '1iv': Float32List; + '2iv': Float32List; + '3iv': Float32List; + '4iv': Float32List; + '1fv': Float32List; + '2fv': Float32List; + '3fv': Float32List; + '4fv': Float32List; + Matrix2fv: Float32List; + Matrix3fv: Float32List; + Matrix4fv: Float32List; +}; +export type UniformType = keyof Uniforms; + +export function isMatrixType(t: string, v: number[] | number): v is number[] { + return t.includes('Matrix') && Array.isArray(v); +} +export function isVectorListType(t: string, v: number[] | number): v is number[] { + return t.includes('v') && Array.isArray(v) && v.length > Number.parseInt(t.charAt(0)); +} +function isVectorType(t: string, v: number[] | number): v is Vector4 { + return !t.includes('v') && Array.isArray(v) && v.length > Number.parseInt(t.charAt(0)); +} +export const processUniform = ( + gl: WebGLRenderingContext, + location: WebGLUniformLocation, + t: T, + value: number | number[] +) => { + if (isVectorType(t, value)) { + switch (t) { + case '2f': + return gl.uniform2f(location, value[0], value[1]); + case '3f': + return gl.uniform3f(location, value[0], value[1], value[2]); + case '4f': + return gl.uniform4f(location, value[0], value[1], value[2], value[3]); + case '2i': + return gl.uniform2i(location, value[0], value[1]); + case '3i': + return gl.uniform3i(location, value[0], value[1], value[2]); + case '4i': + return gl.uniform4i(location, value[0], value[1], value[2], value[3]); + } + } + if (typeof value === 'number') { + switch (t) { + case '1i': + return gl.uniform1i(location, value); + default: + return gl.uniform1f(location, value); + } + } + switch (t) { + case '1iv': + return gl.uniform1iv(location, value); + case '2iv': + return gl.uniform2iv(location, value); + case '3iv': + return gl.uniform3iv(location, value); + case '4iv': + return gl.uniform4iv(location, value); + case '1fv': + return gl.uniform1fv(location, value); + case '2fv': + return gl.uniform2fv(location, value); + case '3fv': + return gl.uniform3fv(location, value); + case '4fv': + return gl.uniform4fv(location, value); + case 'Matrix2fv': + return gl.uniformMatrix2fv(location, false, value); + case 'Matrix3fv': + return gl.uniformMatrix3fv(location, false, value); + case 'Matrix4fv': + return gl.uniformMatrix4fv(location, false, value); + } +}; + +export const uniformTypeToGLSLType = (t: string) => { + switch (t) { + case '1f': + return 'float'; + case '2f': + return 'vec2'; + case '3f': + return 'vec3'; + case '4f': + return 'vec4'; + case '1i': + return 'int'; + case '2i': + return 'ivec2'; + case '3i': + return 'ivec3'; + case '4i': + return 'ivec4'; + case '1iv': + return 'int'; + case '2iv': + return 'ivec2'; + case '3iv': + return 'ivec3'; + case '4iv': + return 'ivec4'; + case '1fv': + return 'float'; + case '2fv': + return 'vec2'; + case '3fv': + return 'vec3'; + case '4fv': + return 'vec4'; + case 'Matrix2fv': + return 'mat2'; + case 'Matrix3fv': + return 'mat3'; + case 'Matrix4fv': + return 'mat4'; + default: + console.error( + log(`The uniform type "${t}" is not valid, please make sure your uniform type is valid`) + ); + } +}; diff --git a/hooks/useMicrophone.ts b/hooks/useMicrophone.ts new file mode 100644 index 000000000..bc6bfaa7b --- /dev/null +++ b/hooks/useMicrophone.ts @@ -0,0 +1,13 @@ +import { useEffect } from 'react'; +import { useLocalParticipant } from '@livekit/components-react'; +import { useConnection } from '@/hooks/useConnection'; + +export function useMicrophone() { + const { connect } = useConnection(); + const { localParticipant } = useLocalParticipant(); + + useEffect(() => { + connect(); + localParticipant.setMicrophoneEnabled(true, undefined); + }, [connect, localParticipant]); +} diff --git a/lib/components.tsx b/lib/components.tsx new file mode 100644 index 000000000..ce4038de2 --- /dev/null +++ b/lib/components.tsx @@ -0,0 +1,10 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +export function getComponentNames() { + const componentsDir = path.join(process.cwd(), 'components/demos'); + const componentNames = fs.readdirSync(componentsDir); + return componentNames + .filter((file) => file.endsWith('.tsx')) + .map((file) => file.replace('.tsx', '')); +} diff --git a/lib/utils.ts b/lib/utils.ts index e4484b9be..1b823f4d2 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -1,4 +1,11 @@ -import { cache } from 'react'; +import { + type CSSProperties, + Children, + type ReactNode, + cache, + cloneElement, + isValidElement, +} from 'react'; import { type ClassValue, clsx } from 'clsx'; import { twMerge } from 'tailwind-merge'; import { APP_CONFIG_DEFAULTS } from '@/app-config'; @@ -7,9 +14,6 @@ import type { AppConfig } from '@/app-config'; export const CONFIG_ENDPOINT = process.env.NEXT_PUBLIC_APP_CONFIG_ENDPOINT; export const SANDBOX_ID = process.env.SANDBOX_ID; -export const THEME_STORAGE_KEY = 'theme-mode'; -export const THEME_MEDIA_QUERY = '(prefers-color-scheme: dark)'; - export interface SandboxConfig { [key: string]: | { type: 'string'; value: string } @@ -89,3 +93,27 @@ export function getStyles(appConfig: AppConfig) { .filter(Boolean) .join('\n'); } + +export function cloneSingleChild( + children: ReactNode | ReactNode[], + props?: Record, + key?: unknown +) { + return Children.map(children, (child) => { + // Checking isValidElement is the safe way and avoids a typescript error too. + if (isValidElement(child) && Children.only(children)) { + const childProps = child.props as Record; + if (childProps.className) { + // make sure we retain classnames of both passed props and child + props ??= {}; + props.className = cn(childProps.className as string, props.className as string); + props.style = { + ...(childProps.style as CSSProperties), + ...(props.style as CSSProperties), + }; + } + return cloneElement(child, { ...props, key: key ? String(key) : undefined }); + } + return child; + }); +} diff --git a/styles/globals.css b/styles/globals.css index a1b5f7d4c..ba5e9d1b4 100644 --- a/styles/globals.css +++ b/styles/globals.css @@ -111,6 +111,10 @@ --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); --color-sidebar-border: var(--sidebar-border); --color-sidebar-ring: var(--sidebar-ring); + + /* LiveKit UI tokens */ + --audio-visualizer-idle: var(--color-muted); + --audio-visualizer-active: var(--color-foreground); } @layer base {