Update to use the agents SDK api (#27)

* Update to use the agents SDK api

* forgot to commit

* update taskfile

* fix up lint issues

Author: davidliu

app.json

@@ -49,8 +49,7 @@
           "backgroundColor": "#ffffff"
         }
       ],
-      "expo-font",
-      "expo-web-browser"
+      "expo-font"
     ],
     "experiments": {
       "typedRoutes": true

app/(start)/index.tsx

@@ -1,6 +1,6 @@
-import { ConnectionDetails, fetchToken } from '@/hooks/useConnectionDetails';
+import { useConnection } from '@/hooks/useConnection';
 import { useRouter } from 'expo-router';
-import { useEffect, useState } from 'react';
+import { useEffect } from 'react';
 import {
   StyleSheet,
   View,
@@ -12,37 +12,18 @@ import {
 
 export default function StartScreen() {
   const router = useRouter();
-
-  let [isConnecting, setConnecting] = useState(false);
-  let [connectionDetails, setConnectionDetails] = useState<
-    ConnectionDetails | undefined
-  >(undefined);
-
-  // Fetch token when we're connecting.
-  useEffect(() => {
-    if (isConnecting) {
-      fetchToken().then((details) => {
-        console.log(details);
-        setConnectionDetails(details);
-        if (!details) {
-          setConnecting(false);
-        }
-      });
-    }
-  }, [isConnecting]);
+  const { isConnectionActive, connect } = useConnection();
 
   // Navigate to Assistant screen when we have the connection details.
   useEffect(() => {
-    if (isConnecting && connectionDetails) {
-      setConnecting(false);
-      setConnectionDetails(undefined);
+    if (isConnectionActive) {
       router.navigate('../assistant');
     }
-  }, [isConnecting, router, connectionDetails]);
+  }, [isConnectionActive, router]);
 
   let connectText: string;
 
-  if (isConnecting) {
+  if (isConnectionActive) {
     connectText = 'Connecting';
   } else {
     connectText = 'Start Voice Assistant';
@@ -58,13 +39,13 @@ export default function StartScreen() {
 
       <TouchableOpacity
         onPress={() => {
-          setConnecting(true);
+          connect();
         }}
         style={styles.button}
         activeOpacity={0.7}
-        disabled={isConnecting} // Disable button while loading
+        disabled={isConnectionActive} // Disable button while loading
       >
-        {isConnecting ? (
+        {isConnectionActive ? (
           <ActivityIndicator
             size="small"
             color="#ffffff"

app/_layout.tsx

@@ -9,6 +9,7 @@ import 'react-native-reanimated';
 
 import { useColorScheme } from '@/hooks/useColorScheme';
 import { registerGlobals } from '@livekit/react-native';
+import { ConnectionProvider } from '@/hooks/useConnection';
 
 // Do required setup for LiveKit React-Native
 registerGlobals();
@@ -17,12 +18,14 @@ export default function RootLayout() {
   const colorScheme = useColorScheme();
 
   return (
-    <ThemeProvider value={colorScheme === 'dark' ? DarkTheme : DefaultTheme}>
-      <Stack>
-        <Stack.Screen name="(start)" options={{ headerShown: false }} />
-        <Stack.Screen name="assistant" options={{ headerShown: false }} />
-      </Stack>
-      <StatusBar style="auto" />
-    </ThemeProvider>
+    <ConnectionProvider>
+      <ThemeProvider value={colorScheme === 'dark' ? DarkTheme : DefaultTheme}>
+        <Stack>
+          <Stack.Screen name="(start)" options={{ headerShown: false }} />
+          <Stack.Screen name="assistant" options={{ headerShown: false }} />
+        </Stack>
+        <StatusBar style="auto" />
+      </ThemeProvider>
+    </ConnectionProvider>
   );
 }

app/assistant/index.tsx

@@ -10,22 +10,25 @@ import {
 import React, { useCallback, useEffect, useState } from 'react';
 import {
   AudioSession,
-  LiveKitRoom,
   useIOSAudioManagement,
   useLocalParticipant,
   useParticipantTracks,
   useRoomContext,
   VideoTrack,
 } from '@livekit/react-native';
-import { useConnectionDetails } from '@/hooks/useConnectionDetails';
 import { SafeAreaView } from 'react-native-safe-area-context';
 import { useRouter } from 'expo-router';
 import ControlBar from './ui/ControlBar';
 import ChatBar from './ui/ChatBar';
 import ChatLog from './ui/ChatLog';
 import AgentVisualization from './ui/AgentVisualization';
-import useDataStreamTranscriptions from '@/hooks/useDataStreamTranscriptions';
 import { Track } from 'livekit-client';
+import {
+  TrackReference,
+  useSessionMessages,
+  useTrackToggle,
+} from '@livekit/components-react';
+import { useConnection } from '@/hooks/useConnection';
 
 export default function AssistantScreen() {
   // Start the audio session first.
@@ -40,27 +43,18 @@ export default function AssistantScreen() {
     };
   }, []);
 
-  const connectionDetails = useConnectionDetails();
-
   return (
     <SafeAreaView>
-      <LiveKitRoom
-        serverUrl={connectionDetails?.url}
-        token={connectionDetails?.token}
-        connect={true}
-        audio={true}
-        video={false}
-      >
-        <RoomView />
-      </LiveKitRoom>
+      <RoomView />
     </SafeAreaView>
   );
 }
 
 const RoomView = () => {
   const router = useRouter();
-
+  const connection = useConnection();
   const room = useRoomContext();
+
   useIOSAudioManagement(room, true);
 
   const {
@@ -79,45 +73,41 @@ const RoomView = () => {
 
   const localVideoTrack =
     localCameraTrack && isCameraEnabled
-      ? {
+      ? ({
           participant: localParticipant,
           publication: localCameraTrack,
           source: Track.Source.Camera,
-        }
+        } satisfies TrackReference)
       : localScreenShareTrack.length > 0 && isScreenShareEnabled
       ? localScreenShareTrack[0]
       : null;
 
-  // Transcriptions
-  const transcriptionState = useDataStreamTranscriptions();
-  const addTranscription = transcriptionState.addTranscription;
+  // Messages
+  const { messages, send } = useSessionMessages();
   const [isChatEnabled, setChatEnabled] = useState(false);
   const [chatMessage, setChatMessage] = useState('');
 
   const onChatSend = useCallback(
     (message: string) => {
-      addTranscription(localParticipantIdentity, message);
+      send(message);
       setChatMessage('');
     },
-    [localParticipantIdentity, addTranscription, setChatMessage]
+    [setChatMessage, send]
   );
 
   // Control callbacks
-  const onMicClick = useCallback(() => {
-    localParticipant.setMicrophoneEnabled(!isMicrophoneEnabled);
-  }, [isMicrophoneEnabled, localParticipant]);
-  const onCameraClick = useCallback(() => {
-    localParticipant.setCameraEnabled(!isCameraEnabled);
-  }, [isCameraEnabled, localParticipant]);
-  const onScreenShareClick = useCallback(() => {
-    localParticipant.setScreenShareEnabled(!isScreenShareEnabled);
-  }, [isScreenShareEnabled, localParticipant]);
+  const micToggle = useTrackToggle({ source: Track.Source.Microphone });
+  const cameraToggle = useTrackToggle({ source: Track.Source.Camera });
+  const screenShareToggle = useTrackToggle({
+    source: Track.Source.ScreenShare,
+  });
   const onChatClick = useCallback(() => {
     setChatEnabled(!isChatEnabled);
   }, [isChatEnabled, setChatEnabled]);
   const onExitClick = useCallback(() => {
+    connection.disconnect();
     router.back();
-  }, [router]);
+  }, [connection, router]);
 
   // Layout positioning
   const [containerWidth, setContainerWidth] = useState(
@@ -159,10 +149,7 @@ const RoomView = () => {
       }}
     >
       <View style={styles.spacer} />
-      <ChatLog
-        style={styles.logContainer}
-        transcriptions={transcriptionState.transcriptions}
-      />
+      <ChatLog style={styles.logContainer} messages={messages} />
       <ChatBar
         style={styles.chatBar}
         value={chatMessage}
@@ -194,10 +181,10 @@ const RoomView = () => {
           isCameraEnabled,
           isScreenShareEnabled,
           isChatEnabled,
-          onMicClick,
-          onCameraClick,
+          onMicClick: micToggle.toggle,
+          onCameraClick: cameraToggle.toggle,
           onChatClick,
-          onScreenShareClick,
+          onScreenShareClick: screenShareToggle.toggle,
           onExitClick,
         }}
       />

app/assistant/ui/AgentVisualization.tsx

@@ -1,4 +1,4 @@
-import { useVoiceAssistant } from '@livekit/components-react';
+import { useAgent } from '@livekit/components-react';
 import { BarVisualizer, VideoTrack } from '@livekit/react-native';
 import React, { useCallback, useState } from 'react';
 import {
@@ -16,17 +16,19 @@ type AgentVisualizationProps = {
 const barSize = 0.2;
 
 export default function AgentVisualization({ style }: AgentVisualizationProps) {
-  const { state, audioTrack, videoTrack } = useVoiceAssistant();
+  const { state, microphoneTrack, cameraTrack } = useAgent();
   const [barWidth, setBarWidth] = useState(0);
   const [barBorderRadius, setBarBorderRadius] = useState(0);
+
   const layoutCallback = useCallback((event: LayoutChangeEvent) => {
     const { x, y, width, height } = event.nativeEvent.layout;
     console.log(x, y, width, height);
     setBarWidth(barSize * height);
     setBarBorderRadius(barSize * height);
   }, []);
-  let videoView = videoTrack ? (
-    <VideoTrack trackRef={videoTrack} style={styles.videoTrack} />
+
+  let videoView = cameraTrack ? (
+    <VideoTrack trackRef={cameraTrack} style={styles.videoTrack} />
   ) : null;
   return (
     <View style={[style, styles.container]}>
@@ -40,7 +42,7 @@ export default function AgentVisualization({ style }: AgentVisualizationProps) {
             barColor: '#FFFFFF',
             barBorderRadius: barBorderRadius,
           }}
-          trackRef={audioTrack}
+          trackRef={microphoneTrack}
           style={styles.barVisualizer}
         />
       </View>

app/assistant/ui/ChatLog.tsx

@@ -1,5 +1,7 @@
-import { Transcription } from '@/hooks/useDataStreamTranscriptions';
-import { useLocalParticipant } from '@livekit/components-react';
+import {
+  ReceivedMessage,
+  useLocalParticipant,
+} from '@livekit/components-react';
 import { useCallback } from 'react';
 import {
   ListRenderItemInfo,
@@ -14,28 +16,30 @@ import Animated, { LinearTransition } from 'react-native-reanimated';
 
 export type ChatLogProps = {
   style: StyleProp<ViewStyle>;
-  transcriptions: Transcription[];
+  messages: ReceivedMessage[];
 };
-export default function ChatLog({ style, transcriptions }: ChatLogProps) {
+export default function ChatLog({
+  style,
+  messages: transcriptions,
+}: ChatLogProps) {
   const { localParticipant } = useLocalParticipant();
-  const localParticipantIdentity = localParticipant.identity;
 
   const renderItem = useCallback(
-    ({ item }: ListRenderItemInfo<Transcription>) => {
-      const isLocalUser = item.identity === localParticipantIdentity;
+    ({ item }: ListRenderItemInfo<ReceivedMessage>) => {
+      const isLocalUser = item.from === localParticipant;
       if (isLocalUser) {
-        return <UserTranscriptionText text={item.segment.text} />;
+        return <UserTranscriptionText text={item.message} />;
       } else {
-        return <AgentTranscriptionText text={item.segment.text} />;
+        return <AgentTranscriptionText text={item.message} />;
       }
     },
-    [localParticipantIdentity]
+    [localParticipant]
   );
 
   return (
     <Animated.FlatList
       renderItem={renderItem}
-      data={transcriptions}
+      data={transcriptions.toReversed()}
       style={style}
       inverted={true}
       itemLayoutAnimation={LinearTransition}