Session api

build.yaml

@@ -19,6 +19,10 @@ targets:
         - example/**
     builders:
       json_serializable:
+        generate_for:
+          include:
+            - lib/src/json/**.dart
+            - lib/src/token_source/**.dart
         options:
           include_if_null: false
           explicit_to_json: true

lib/livekit_client.dart

@@ -29,6 +29,15 @@ export 'src/livekit.dart';
 export 'src/logger.dart';
 export 'src/managers/event.dart';
 export 'src/options.dart';
+export 'src/agent/agent.dart';
+export 'src/agent/session.dart';
+export 'src/agent/session_options.dart';
+export 'src/agent/chat/message.dart';
+export 'src/agent/chat/message_sender.dart';
+export 'src/agent/chat/message_receiver.dart';
+export 'src/agent/chat/text_message_sender.dart';
+export 'src/agent/chat/transcription_stream_receiver.dart';
+export 'src/agent/room_agent.dart';
 export 'src/participant/local.dart';
 export 'src/participant/participant.dart';
 export 'src/participant/remote.dart' hide ParticipantCreationResult;
@@ -48,7 +57,7 @@ export 'src/track/remote/audio.dart';
 export 'src/track/remote/remote.dart';
 export 'src/track/remote/video.dart';
 export 'src/track/track.dart';
-export 'src/types/attribute_typings.dart';
+export 'src/json/agent_attributes.dart';
 export 'src/types/data_stream.dart';
 export 'src/types/other.dart';
 export 'src/types/participant_permissions.dart';

lib/src/agent/agent.dart

@@ -0,0 +1,195 @@
+// Copyright 2025 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import 'package:flutter/foundation.dart';
+
+import 'package:collection/collection.dart';
+
+import '../json/agent_attributes.dart';
+import '../participant/participant.dart';
+import '../participant/remote.dart';
+import '../track/remote/audio.dart';
+import '../track/remote/video.dart';
+import '../types/other.dart';
+import 'constants.dart';
+
+/// Represents a LiveKit Agent.
+///
+/// The [Agent] class models the state of a LiveKit agent within a [Session].
+/// It exposes information about the agent's connection status, conversational
+/// state, and the media tracks that belong to the agent. Consumers should
+/// observe [Agent] to update their UI when the agent connects, disconnects,
+/// or transitions between conversational states such as listening, thinking,
+/// and speaking.
+///
+/// The associated [Participant]'s attributes are inspected to derive the
+/// agent-specific metadata (such as [agentState]). Audio and avatar video
+/// tracks are picked from the agent participant and its associated avatar
+/// worker (if any).
+class Agent extends ChangeNotifier {
+  Agent();
+
+  AgentFailure? get error => _error;
+  AgentFailure? _error;
+
+  /// The current conversational state of the agent.
+  AgentState? get agentState => _agentState;
+  AgentState? _agentState;
+
+  /// The agent's audio track, if available.
+  RemoteAudioTrack? get audioTrack => _audioTrack;
+  RemoteAudioTrack? _audioTrack;
+
+  /// The agent's avatar video track, if available.
+  RemoteVideoTrack? get avatarVideoTrack => _avatarVideoTrack;
+  RemoteVideoTrack? _avatarVideoTrack;
+
+  /// Indicates whether the agent is connected.
+  bool get isConnected => switch (_state) {
+        _AgentLifecycle.connected => true,
+        _AgentLifecycle.connecting => false,
+        _AgentLifecycle.disconnected => false,
+        _AgentLifecycle.failed => false,
+      };
+
+  /// Whether the agent is buffering audio prior to connecting.
+  bool get isBuffering => _state == _AgentLifecycle.connecting && _isBuffering;
+
+  _AgentLifecycle _state = _AgentLifecycle.disconnected;
+  bool _isBuffering = false;
+
+  /// Marks the agent as disconnected.
+  void disconnected() {
+    if (_state == _AgentLifecycle.disconnected &&
+        _agentState == null &&
+        _audioTrack == null &&
+        _avatarVideoTrack == null &&
+        _error == null) {
+      return;
+    }
+    _state = _AgentLifecycle.disconnected;
+    _isBuffering = false;
+    _agentState = null;
+    _audioTrack = null;
+    _avatarVideoTrack = null;
+    _error = null;
+    notifyListeners();
+  }
+
+  /// Marks the agent as connecting.
+  void connecting({required bool buffering}) {
+    _state = _AgentLifecycle.connecting;
+    _isBuffering = buffering;
+    _error = null;
+    notifyListeners();
+  }
+
+  /// Marks the agent as failed.
+  void failed(AgentFailure failure) {
+    _state = _AgentLifecycle.failed;
+    _isBuffering = false;
+    _error = failure;
+    notifyListeners();
+  }
+
+  /// Updates the agent with information from the connected [participant].
+  void connected(RemoteParticipant participant) {
+    final AgentState? nextAgentState = _readAgentState(participant);
+    final RemoteAudioTrack? nextAudioTrack = _resolveAudioTrack(participant);
+    final RemoteVideoTrack? nextAvatarTrack = _resolveAvatarVideoTrack(participant);
+
+    final bool shouldNotify = _state != _AgentLifecycle.connected ||
+        _agentState != nextAgentState ||
+        !identical(_audioTrack, nextAudioTrack) ||
+        !identical(_avatarVideoTrack, nextAvatarTrack) ||
+        _error != null ||
+        _isBuffering;
+
+    _state = _AgentLifecycle.connected;
+    _isBuffering = false;
+    _error = null;
+    _agentState = nextAgentState;
+    _audioTrack = nextAudioTrack;
+    _avatarVideoTrack = nextAvatarTrack;
+
+    if (shouldNotify) {
+      notifyListeners();
+    }
+  }
+
+  AgentState? _readAgentState(Participant participant) {
+    final rawState = participant.attributes[lkAgentStateAttributeKey];
+    if (rawState == null) {
+      return null;
+    }
+    switch (rawState) {
+      case 'idle':
+        return AgentState.IDLE;
+      case 'initializing':
+        return AgentState.INITIALIZING;
+      case 'listening':
+        return AgentState.LISTENING;
+      case 'speaking':
+        return AgentState.SPEAKING;
+      case 'thinking':
+        return AgentState.THINKING;
+      default:
+        return null;
+    }
+  }
+
+  RemoteAudioTrack? _resolveAudioTrack(RemoteParticipant participant) {
+    final publication = participant.audioTrackPublications.firstWhereOrNull(
+      (pub) => pub.source == TrackSource.microphone,
+    );
+    return publication?.track;
+  }
+
+  RemoteVideoTrack? _resolveAvatarVideoTrack(RemoteParticipant participant) {
+    final avatarWorker = _findAvatarWorker(participant);
+    if (avatarWorker == null) {
+      return null;
+    }
+    final publication = avatarWorker.videoTrackPublications.firstWhereOrNull(
+      (pub) => pub.source == TrackSource.camera,
+    );
+    return publication?.track;
+  }
+
+  RemoteParticipant? _findAvatarWorker(RemoteParticipant participant) {
+    final publishOnBehalf = participant.identity;
+    final room = participant.room;
+    return room.remoteParticipants.values.firstWhereOrNull(
+      (p) => p.attributes[lkPublishOnBehalfAttributeKey] == publishOnBehalf,
+    );
+  }
+}
+
+/// Describes why an [Agent] failed to connect.
+enum AgentFailure {
+  /// The agent did not connect within the allotted timeout.
+  timeout;
+
+  /// A human-readable error message.
+  String get message => switch (this) {
+        AgentFailure.timeout => 'Agent did not connect',
+      };
+}
+
+enum _AgentLifecycle {
+  disconnected,
+  connecting,
+  connected,
+  failed,
+}

lib/src/agent/chat/message.dart

@@ -0,0 +1,130 @@
+// Copyright 2025 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import 'package:flutter/foundation.dart';
+
+/// A message received from the agent.
+@immutable
+class ReceivedMessage {
+  const ReceivedMessage({
+    required this.id,
+    required this.timestamp,
+    required this.content,
+  });
+
+  final String id;
+  final DateTime timestamp;
+  final ReceivedMessageContent content;
+
+  ReceivedMessage copyWith({
+    String? id,
+    DateTime? timestamp,
+    ReceivedMessageContent? content,
+  }) {
+    return ReceivedMessage(
+      id: id ?? this.id,
+      timestamp: timestamp ?? this.timestamp,
+      content: content ?? this.content,
+    );
+  }
+
+  @override
+  bool operator ==(Object other) {
+    if (identical(this, other)) return true;
+    return other is ReceivedMessage && other.id == id && other.timestamp == timestamp && other.content == content;
+  }
+
+  @override
+  int get hashCode => Object.hash(id, timestamp, content);
+}
+
+/// Base class for message content types that can be received from the agent.
+sealed class ReceivedMessageContent {
+  const ReceivedMessageContent();
+
+  /// Textual representation of the content.
+  String get text;
+}
+
+/// A transcript emitted by the agent.
+class AgentTranscript extends ReceivedMessageContent {
+  const AgentTranscript(this.text);
+
+  @override
+  final String text;
+
+  @override
+  bool operator ==(Object other) => other is AgentTranscript && other.text == text;
+
+  @override
+  int get hashCode => text.hashCode;
+}
+
+/// A transcript emitted for the user (e.g., speech-to-text).
+class UserTranscript extends ReceivedMessageContent {
+  const UserTranscript(this.text);
+
+  @override
+  final String text;
+
+  @override
+  bool operator ==(Object other) => other is UserTranscript && other.text == text;
+
+  @override
+  int get hashCode => text.hashCode;
+}
+
+/// A message that originated from user input (loopback).
+class UserInput extends ReceivedMessageContent {
+  const UserInput(this.text);
+
+  @override
+  final String text;
+
+  @override
+  bool operator ==(Object other) => other is UserInput && other.text == text;
+
+  @override
+  int get hashCode => text.hashCode;
+}
+
+/// A message sent to the agent.
+@immutable
+class SentMessage {
+  const SentMessage({
+    required this.id,
+    required this.timestamp,
+    required this.content,
+  });
+
+  final String id;
+  final DateTime timestamp;
+  final SentMessageContent content;
+}
+
+/// Base class for message content types that can be sent to the agent.
+sealed class SentMessageContent {
+  const SentMessageContent();
+
+  /// Textual representation of the content.
+  String get text;
+}
+
+/// User-provided text input.
+class SentUserInput extends SentMessageContent {
+  const SentUserInput(this.text);
+
+  @override
+  final String text;
+}

lib/src/agent/chat/message_receiver.dart

@@ -0,0 +1,24 @@
+// Copyright 2025 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import 'dart:async';
+
+import 'message.dart';
+
+/// Receives messages produced by the agent.
+abstract class MessageReceiver {
+  Stream<ReceivedMessage> messages();
+
+  Future<void> dispose();
+}