From 757bcde875a3340d9930bffcb4d906a871d2c435 Mon Sep 17 00:00:00 2001 From: CloudWebRTC Date: Mon, 13 Jan 2025 19:43:30 +0800 Subject: [PATCH 1/4] feat: add TrackProcessor support. (#657) * feat: add TrackProcessor support. * update. * wip. * bump version for flutter-webrtc * import sorter. * fix analyze. * fix analyzer. * update. * add (T options) to restart. --- lib/livekit_client.dart | 1 + lib/src/core/engine.dart | 5 +++ lib/src/events.dart | 14 ++++++++ lib/src/participant/local.dart | 9 +++++ lib/src/track/local/audio.dart | 8 ++++- lib/src/track/local/local.dart | 57 ++++++++++++++++++++++++++++++++ lib/src/track/local/video.dart | 17 ++++++---- lib/src/track/options.dart | 11 ++++++ lib/src/track/processor.dart | 39 ++++++++++++++++++++++ lib/src/track/processor_web.dart | 14 ++++++++ pubspec.yaml | 4 +-- 11 files changed, 169 insertions(+), 10 deletions(-) create mode 100644 lib/src/track/processor.dart create mode 100644 lib/src/track/processor_web.dart diff --git a/lib/livekit_client.dart b/lib/livekit_client.dart index d9a71cc1c..d656f04ac 100644 --- a/lib/livekit_client.dart +++ b/lib/livekit_client.dart @@ -44,6 +44,7 @@ export 'src/track/remote/audio.dart'; export 'src/track/remote/remote.dart'; export 'src/track/remote/video.dart'; export 'src/track/track.dart'; +export 'src/track/processor.dart'; export 'src/types/other.dart'; export 'src/types/participant_permissions.dart'; export 'src/types/video_dimensions.dart'; diff --git a/lib/src/core/engine.dart b/lib/src/core/engine.dart index 0180b2752..9c73ab50b 100644 --- a/lib/src/core/engine.dart +++ b/lib/src/core/engine.dart @@ -133,6 +133,10 @@ class Engine extends Disposable with EventsEmittable { RegionUrlProvider? _regionUrlProvider; + lk_models.ServerInfo? _serverInfo; + + lk_models.ServerInfo? get serverInfo => _serverInfo; + void clearReconnectTimeout() { if (reconnectTimeout != null) { reconnectTimeout?.cancel(); @@ -893,6 +897,7 @@ class Engine extends Disposable with EventsEmittable { ..on((event) async { // create peer connections _subscriberPrimary = event.response.subscriberPrimary; + _serverInfo = event.response.serverInfo; var iceServersFromServer = event.response.iceServers.map((e) => e.toSDKType()).toList(); diff --git a/lib/src/events.dart b/lib/src/events.dart index cc1935403..aae527bb3 100644 --- a/lib/src/events.dart +++ b/lib/src/events.dart @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +import 'package:livekit_client/src/track/processor.dart'; import 'core/engine.dart'; import 'core/room.dart'; import 'core/signal_client.dart'; @@ -578,3 +579,16 @@ class AudioVisualizerEvent with TrackEvent { String toString() => '${runtimeType}' 'track: ${track})'; } + +class TrackProcessorUpdateEvent with TrackEvent { + final Track track; + final TrackProcessor? processor; + const TrackProcessorUpdateEvent({ + required this.track, + this.processor, + }); + + @override + String toString() => '${runtimeType}' + 'track: ${track})'; +} diff --git a/lib/src/participant/local.dart b/lib/src/participant/local.dart index 9b3282729..1731ff278 100644 --- a/lib/src/participant/local.dart +++ b/lib/src/participant/local.dart @@ -115,6 +115,8 @@ class LocalParticipant extends Participant { // did publish await track.onPublish(); + await track.processor?.onPublish(room); + await room.applyAudioSpeakerSettings(); var listener = track.createListener(); @@ -330,6 +332,7 @@ class LocalParticipant extends Participant { // did publish await track.onPublish(); + await track.processor?.onPublish(room); var listener = track.createListener(); listener.on((TrackEndedEvent event) { @@ -384,6 +387,12 @@ class LocalParticipant extends Participant { // did unpublish await track.onUnpublish(); + + if (track.processor != null) { + await track.processor?.onUnpublish(); + await track.stopProcessor(); + } + await room.applyAudioSpeakerSettings(); } diff --git a/lib/src/track/local/audio.dart b/lib/src/track/local/audio.dart index 90b91dd5d..5d86a0ca2 100644 --- a/lib/src/track/local/audio.dart +++ b/lib/src/track/local/audio.dart @@ -136,12 +136,18 @@ class LocalAudioTrack extends LocalTrack options ??= const AudioCaptureOptions(); final stream = await LocalTrack.createStream(options); - return LocalAudioTrack( + var track = LocalAudioTrack( TrackSource.microphone, stream, stream.getAudioTracks().first, options, enableVisualizer: enableVisualizer, ); + + if (options.processor != null) { + await track.setProcessor(options.processor); + } + + return track; } } diff --git a/lib/src/track/local/local.dart b/lib/src/track/local/local.dart index b64c62e80..3917e123f 100644 --- a/lib/src/track/local/local.dart +++ b/lib/src/track/local/local.dart @@ -31,6 +31,7 @@ import '../../support/native.dart'; import '../../support/platform.dart'; import '../../types/other.dart'; import '../options.dart'; +import '../processor.dart'; import '../remote/audio.dart'; import '../remote/video.dart'; import '../track.dart'; @@ -119,6 +120,10 @@ abstract class LocalTrack extends Track { bool _stopped = false; + TrackProcessor? _processor; + + TrackProcessor? get processor => _processor; + LocalTrack( TrackType kind, TrackSource source, @@ -253,6 +258,10 @@ abstract class LocalTrack extends Track { final newStream = await LocalTrack.createStream(currentOptions); final newTrack = newStream.getTracks().first; + var processor = _processor; + + await stopProcessor(); + // replace track on sender try { await sender?.replaceTrack(newTrack); @@ -267,6 +276,10 @@ abstract class LocalTrack extends Track { // set new stream & track to this object updateMediaStreamAndTrack(newStream, newTrack); + if (processor != null) { + await setProcessor(processor); + } + // mark as started await start(); @@ -277,6 +290,50 @@ abstract class LocalTrack extends Track { )); } + Future setProcessor(TrackProcessor? processor) async { + if (processor == null) { + return; + } + + if (_processor != null) { + await stopProcessor(); + } + + _processor = processor; + + var processorOptions = ProcessorOptions( + kind: kind, + track: mediaStreamTrack, + ); + + await _processor!.init(processorOptions); + + logger.fine('processor initialized'); + + events.emit(TrackProcessorUpdateEvent(track: this, processor: _processor)); + } + + @internal + Future stopProcessor({bool keepElement = false}) async { + if (_processor == null) return; + + logger.fine('stopping processor'); + await _processor?.destroy(); + _processor = null; + + if (!keepElement) { + // processorElement?.remove(); + // processorElement = null; + } + + // apply original track constraints in case the processor changed them + //await this._mediaStreamTrack.applyConstraints(this._constraints); + // force re-setting of the mediaStreamTrack on the sender + //await this.setMediaStreamTrack(this._mediaStreamTrack, true); + + events.emit(TrackProcessorUpdateEvent(track: this)); + } + @internal @mustCallSuper Future onPublish() async { diff --git a/lib/src/track/local/video.dart b/lib/src/track/local/video.dart index b7c0fc20f..44d3afc94 100644 --- a/lib/src/track/local/video.dart +++ b/lib/src/track/local/video.dart @@ -162,12 +162,9 @@ class LocalVideoTrack extends LocalTrack with VideoTrack { } // Private constructor - LocalVideoTrack._( - TrackSource source, - rtc.MediaStream stream, - rtc.MediaStreamTrack track, - this.currentOptions, - ) : super( + LocalVideoTrack._(TrackSource source, rtc.MediaStream stream, + rtc.MediaStreamTrack track, this.currentOptions) + : super( TrackType.VIDEO, source, stream, @@ -181,12 +178,18 @@ class LocalVideoTrack extends LocalTrack with VideoTrack { options ??= const CameraCaptureOptions(); final stream = await LocalTrack.createStream(options); - return LocalVideoTrack._( + var track = LocalVideoTrack._( TrackSource.camera, stream, stream.getVideoTracks().first, options, ); + + if (options.processor != null) { + await track.setProcessor(options.processor); + } + + return track; } /// Creates a LocalVideoTrack from the display. diff --git a/lib/src/track/options.dart b/lib/src/track/options.dart index 8483c891c..c5e8226f5 100644 --- a/lib/src/track/options.dart +++ b/lib/src/track/options.dart @@ -19,6 +19,7 @@ import '../support/platform.dart'; import '../track/local/audio.dart'; import '../track/local/video.dart'; import '../types/video_parameters.dart'; +import 'processor.dart'; /// A type that represents front or back of the camera. enum CameraPosition { @@ -60,10 +61,12 @@ class CameraCaptureOptions extends VideoCaptureOptions { double? maxFrameRate, VideoParameters params = VideoParametersPresets.h720_169, this.stopCameraCaptureOnMute = true, + TrackProcessor? processor, }) : super( params: params, deviceId: deviceId, maxFrameRate: maxFrameRate, + processor: processor, ); CameraCaptureOptions.from({required VideoCaptureOptions captureOptions}) @@ -217,10 +220,14 @@ abstract class VideoCaptureOptions extends LocalTrackOptions { // Limit the maximum frameRate of the capture device. final double? maxFrameRate; + /// A processor to apply to the video track. + final TrackProcessor? processor; + const VideoCaptureOptions({ this.params = VideoParametersPresets.h540_169, this.deviceId, this.maxFrameRate, + this.processor, }); @override @@ -269,6 +276,9 @@ class AudioCaptureOptions extends LocalTrackOptions { /// set to false to only toggle enabled instead of stop/replaceTrack for muting final bool stopAudioCaptureOnMute; + /// A processor to apply to the audio track. + final TrackProcessor? processor; + const AudioCaptureOptions({ this.deviceId, this.noiseSuppression = true, @@ -278,6 +288,7 @@ class AudioCaptureOptions extends LocalTrackOptions { this.voiceIsolation = true, this.typingNoiseDetection = true, this.stopAudioCaptureOnMute = true, + this.processor, }); @override diff --git a/lib/src/track/processor.dart b/lib/src/track/processor.dart new file mode 100644 index 000000000..f8a602cb3 --- /dev/null +++ b/lib/src/track/processor.dart @@ -0,0 +1,39 @@ +import 'package:flutter_webrtc/flutter_webrtc.dart'; + +import '../core/room.dart'; +import '../types/other.dart'; + +class ProcessorOptions { + T kind; + MediaStreamTrack track; + ProcessorOptions({ + required this.kind, + required this.track, + }); +} + +class AudioProcessorOptions extends ProcessorOptions { + AudioProcessorOptions({ + required MediaStreamTrack track, + }) : super(kind: TrackType.AUDIO, track: track); +} + +class VideoProcessorOptions extends ProcessorOptions { + VideoProcessorOptions({ + required MediaStreamTrack track, + }) : super(kind: TrackType.VIDEO, track: track); +} + +abstract class TrackProcessor { + String get name; + + Future init(T options); + + Future restart(T options); + + Future destroy(); + + Future onPublish(Room room); + + Future onUnpublish(); +} diff --git a/lib/src/track/processor_web.dart b/lib/src/track/processor_web.dart new file mode 100644 index 000000000..795d2e073 --- /dev/null +++ b/lib/src/track/processor_web.dart @@ -0,0 +1,14 @@ +import 'package:web/web.dart'; + +import 'processor.dart'; + +class AudioProcessorOptionsWeb extends AudioProcessorOptions { + AudioProcessorOptionsWeb({ + this.audioElement, + this.audioContext, + required super.track, + }); + + HTMLAudioElement? audioElement; + AudioContext? audioContext; +} diff --git a/pubspec.yaml b/pubspec.yaml index 3f625a4c8..2c68d330b 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -37,11 +37,11 @@ dependencies: uuid: '>=3.0.6' synchronized: ^3.0.0+3 protobuf: ^3.0.0 - flutter_webrtc: ^0.12.5+hotfix.2 + flutter_webrtc: ^0.12.4 device_info_plus: ^11.1.1 js: '>=0.6.4' platform_detect: ^2.0.7 - dart_webrtc: ^1.4.9 + dart_webrtc: ^1.4.10 sdp_transform: ^0.3.2 web: ^1.0.0 From 20b348c402379995de57dd4c1e684ae58f63b93a Mon Sep 17 00:00:00 2001 From: CloudWebRTC Date: Mon, 13 Jan 2025 23:36:24 +0800 Subject: [PATCH 2/4] fix issue 683. (#684) * fix issue 683. * fix issue #655. --- lib/src/hardware/hardware.dart | 23 +++++++++-------------- lib/src/track/track.dart | 3 +-- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/lib/src/hardware/hardware.dart b/lib/src/hardware/hardware.dart index 4b12a5549..dd44a01fb 100644 --- a/lib/src/hardware/hardware.dart +++ b/lib/src/hardware/hardware.dart @@ -76,10 +76,14 @@ class Hardware { MediaDevice? selectedVideoInput; - bool? speakerOn; + bool? _speakerOn; + + bool? get speakerOn => _speakerOn; bool _preferSpeakerOutput = false; + bool get preferSpeakerOutput => _preferSpeakerOutput; + Future> enumerateDevices({String? type}) async { var infos = await rtc.navigator.mediaDevices.enumerateDevices(); var devices = infos @@ -143,21 +147,12 @@ class Hardware { } } - bool get preferSpeakerOutput => _preferSpeakerOutput; - - bool get canSwitchSpeakerphone => - (lkPlatformIsMobile()) && - [AudioTrackState.localOnly, AudioTrackState.localAndRemote] - .contains(audioTrackState); + bool get canSwitchSpeakerphone => lkPlatformIsMobile(); Future setSpeakerphoneOn(bool enable) async { - if (lkPlatformIsMobile()) { - speakerOn = enable; - if (canSwitchSpeakerphone) { - await rtc.Helper.setSpeakerphoneOn(enable); - } else { - logger.warning('Can\'t switch speaker/earpiece'); - } + if (canSwitchSpeakerphone) { + _speakerOn = enable; + await rtc.Helper.setSpeakerphoneOn(enable); } else { logger.warning('setSpeakerphoneOn only support on iOS/Android'); } diff --git a/lib/src/track/track.dart b/lib/src/track/track.dart index 6ee7ef6d5..cbddaabc0 100644 --- a/lib/src/track/track.dart +++ b/lib/src/track/track.dart @@ -153,9 +153,8 @@ abstract class Track extends DisposableChangeNotifier logger .fine('$objectId.disable() disabling ${mediaStreamTrack.objectId}...'); try { - if (_active || !_muted) { + if (_active) { mediaStreamTrack.enabled = false; - if (!_muted) _muted = true; } } catch (_) { logger.warning( From 208b3816b7bc7fb7a28b6b74831eb0bb1418c2e9 Mon Sep 17 00:00:00 2001 From: Ben Cherry Date: Mon, 13 Jan 2025 07:36:38 -0800 Subject: [PATCH 3/4] E2EE instructions in the example readme (#681) --- example/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/example/README.md b/example/README.md index 14edda879..9b97bb29a 100644 --- a/example/README.md +++ b/example/README.md @@ -12,3 +12,19 @@ flutter pub get # you can autofill URL and TOKEN for first run in debug mode. flutter run --dart-define=URL=wss://${LIVEKIT_SERVER_IP_OR_DOMAIN} --dart-define=TOKEN=${YOUR_TOKEN} ``` + +## End-to-End Encryption (E2EE) + +The example app supports end-to-end encryption for audio and video tracks. To enable E2EE: + +1. Toggle the "E2EE" switch in the connect screen +2. Enter a shared key that will be used for encryption +3. All participants must use the same shared key to communicate + +For web support, you'll need to compile the E2EE web worker: + +```bash +dart compile js web/e2ee.worker.dart -o example/web/e2ee.worker.dart.js -m +``` + +Note: All participants in the room must have E2EE enabled and use the same shared key to see and hear each other. If the keys don't match, participants won't be able to decode each other's audio and video. From cf800fabfeec18c5eab158c37f185e1fac392483 Mon Sep 17 00:00:00 2001 From: CloudWebRTC Date: Tue, 14 Jan 2025 10:57:52 +0800 Subject: [PATCH 4/4] release: 2.3.5. (#685) --- CHANGELOG.md | 5 +++++ ios/livekit_client.podspec | 2 +- lib/src/livekit.dart | 2 +- macos/livekit_client.podspec | 2 +- pubspec.yaml | 2 +- 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cca35b55d..2badd6c08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # CHANGELOG +## 2.3.5 + +* feat: add TrackProcessor support. (#657) +* fix: bug for mute/unmute and speaker switch. (#684) + ## 2.3.4+hotfix.2 * fix: side effects for stop remote track. diff --git a/ios/livekit_client.podspec b/ios/livekit_client.podspec index dd5fa2c36..f9f7135ba 100644 --- a/ios/livekit_client.podspec +++ b/ios/livekit_client.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'livekit_client' - s.version = '2.3.4' + s.version = '2.3.5' s.summary = 'Open source platform for real-time audio and video.' s.description = 'Open source platform for real-time audio and video.' s.homepage = 'https://livekit.io/' diff --git a/lib/src/livekit.dart b/lib/src/livekit.dart index 76c5d1465..cabad489d 100644 --- a/lib/src/livekit.dart +++ b/lib/src/livekit.dart @@ -20,7 +20,7 @@ import 'support/native.dart'; /// Main entry point to connect to a room. /// {@category Room} class LiveKitClient { - static const version = '2.3.4'; + static const version = '2.3.5'; /// Initialize the WebRTC plugin. If this is not manually called, will be /// initialized with default settings. diff --git a/macos/livekit_client.podspec b/macos/livekit_client.podspec index 24bb3520b..ab11044e9 100644 --- a/macos/livekit_client.podspec +++ b/macos/livekit_client.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'livekit_client' - s.version = '2.3.4' + s.version = '2.3.5' s.summary = 'Open source platform for real-time audio and video.' s.description = 'Open source platform for real-time audio and video.' s.homepage = 'https://livekit.io/' diff --git a/pubspec.yaml b/pubspec.yaml index 2c68d330b..5ee94abe2 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -15,7 +15,7 @@ name: livekit_client description: Flutter Client SDK for LiveKit. Build real-time video and audio into your apps. Supports iOS, Android, and Web. -version: 2.3.4+hotfix.2 +version: 2.3.5 homepage: https://github.com/livekit/client-sdk-flutter environment: