From 8ece33dc024e21aa00a5cd06aa4c64fda238512d Mon Sep 17 00:00:00 2001 From: ronen barzel Date: Sat, 4 Nov 2023 23:02:07 +0000 Subject: [PATCH] Move `VideoCallClient` into its own crate (#142) * factor out connection, peer management, and packet handling code * clean up warnings about unused variables and results * make CameraEncoder::start() args consistent with the other encoders * Various cleanups and clippy * Pass VideoCallClient to encoders, remove on_packet callbacks * keep copy of aes in outer struct so no borrow() is needed to get it * Explicitly ensure peer before handling packet, rather than on-the-fly callback hell With some code cleanups along the way * Gratuitous cleanup: Better naming * Gratuitous cleanup: error enums & DRY some code * Gratuitous cleanup: factor out HashWithOrderedKeys * move yew-ui-src/model to videocall-client/src * Flesh out videocall-client crate definition, pull out dependencies Conform `use` statements in yew-ui to crate * cargo fmt * add wasm-unknown-unknown target to rust-toolchain.toml * clippy * Add documentation * Doc update: Mention that only Chrome is supported * Add do-not-edit note to README * Rebuild README * fix bad doc cross-references * move yew-ui-src/model to videocall-client/src * Flesh out videocall-client crate definition, pull out dependencies Conform `use` statements in yew-ui to crate * cargo fmt * add wasm-unknown-unknown target to rust-toolchain.toml * clippy * Add documentation * Doc update: Mention that only Chrome is supported * Add do-not-edit note to README * Rebuild README * fix bad doc cross-references --- Cargo.lock | 33 ++- Cargo.toml | 1 + videocall-client/Cargo.toml | 92 +++++++++ videocall-client/README.md | 93 +++++++++ videocall-client/README.tpl | 25 +++ videocall-client/rust-toolchain.toml | 3 + .../src}/client/mod.rs | 0 .../src}/client/video_call_client.rs | 65 +++++- .../src}/connection/connection.rs | 0 .../src}/connection/mod.rs | 1 + .../src}/connection/task.rs | 0 .../src}/connection/webmedia.rs | 0 .../src}/connection/websocket.rs | 0 .../src}/connection/webtransport.rs | 0 videocall-client/src/constants.rs | 26 +++ .../src/crypto/aes.rs | 0 .../src/crypto/mod.rs | 0 .../src/crypto/rsa.rs | 0 .../src}/decode/config.rs | 0 .../src}/decode/hash_map_with_ordered_keys.rs | 0 .../src}/decode/mod.rs | 0 .../src}/decode/peer_decode_manager.rs | 0 .../src}/decode/peer_decoder.rs | 0 .../src}/decode/video_decoder_with_buffer.rs | 0 .../src}/decode/video_decoder_wrapper.rs | 0 .../src}/encode/camera_encoder.rs | 44 +++- .../src}/encode/encoder_state.rs | 0 .../src}/encode/microphone_encoder.rs | 35 +++- .../src}/encode/mod.rs | 0 .../src}/encode/screen_encoder.rs | 27 ++- .../src}/encode/transform.rs | 0 videocall-client/src/lib.rs | 84 ++++++++ .../src}/media_devices/media_device_access.rs | 23 +++ .../src/media_devices/media_device_list.rs | 188 ++++++++++++++++++ .../src}/media_devices/mod.rs | 2 +- .../src}/wrappers.rs | 0 yew-ui/Cargo.toml | 73 +------ yew-ui/src/components/attendants.rs | 3 +- yew-ui/src/components/device_selector.rs | 2 +- yew-ui/src/components/host.rs | 6 +- yew-ui/src/constants.rs | 26 --- yew-ui/src/main.rs | 3 - .../model/media_devices/media_device_list.rs | 117 ----------- yew-ui/src/model/mod.rs | 8 - 44 files changed, 732 insertions(+), 248 deletions(-) create mode 100644 videocall-client/Cargo.toml create mode 100644 videocall-client/README.md create mode 100644 videocall-client/README.tpl create mode 100644 videocall-client/rust-toolchain.toml rename {yew-ui/src/model => videocall-client/src}/client/mod.rs (100%) rename {yew-ui/src/model => videocall-client/src}/client/video_call_client.rs (78%) rename {yew-ui/src/model => videocall-client/src}/connection/connection.rs (100%) rename {yew-ui/src/model => videocall-client/src}/connection/mod.rs (80%) rename {yew-ui/src/model => videocall-client/src}/connection/task.rs (100%) rename {yew-ui/src/model => videocall-client/src}/connection/webmedia.rs (100%) rename {yew-ui/src/model => videocall-client/src}/connection/websocket.rs (100%) rename {yew-ui/src/model => videocall-client/src}/connection/webtransport.rs (100%) create mode 100644 videocall-client/src/constants.rs rename {yew-ui => videocall-client}/src/crypto/aes.rs (100%) rename {yew-ui => videocall-client}/src/crypto/mod.rs (100%) rename {yew-ui => videocall-client}/src/crypto/rsa.rs (100%) rename {yew-ui/src/model => videocall-client/src}/decode/config.rs (100%) rename {yew-ui/src/model => videocall-client/src}/decode/hash_map_with_ordered_keys.rs (100%) rename {yew-ui/src/model => videocall-client/src}/decode/mod.rs (100%) rename {yew-ui/src/model => videocall-client/src}/decode/peer_decode_manager.rs (100%) rename {yew-ui/src/model => videocall-client/src}/decode/peer_decoder.rs (100%) rename {yew-ui/src/model => videocall-client/src}/decode/video_decoder_with_buffer.rs (100%) rename {yew-ui/src/model => videocall-client/src}/decode/video_decoder_wrapper.rs (100%) rename {yew-ui/src/model => videocall-client/src}/encode/camera_encoder.rs (77%) rename {yew-ui/src/model => videocall-client/src}/encode/encoder_state.rs (100%) rename {yew-ui/src/model => videocall-client/src}/encode/microphone_encoder.rs (77%) rename {yew-ui/src/model => videocall-client/src}/encode/mod.rs (100%) rename {yew-ui/src/model => videocall-client/src}/encode/screen_encoder.rs (80%) rename {yew-ui/src/model => videocall-client/src}/encode/transform.rs (100%) create mode 100644 videocall-client/src/lib.rs rename {yew-ui/src/model => videocall-client/src}/media_devices/media_device_access.rs (62%) create mode 100644 videocall-client/src/media_devices/media_device_list.rs rename {yew-ui/src/model => videocall-client/src}/media_devices/mod.rs (59%) rename {yew-ui/src/model => videocall-client/src}/wrappers.rs (100%) delete mode 100644 yew-ui/src/model/media_devices/media_device_list.rs delete mode 100644 yew-ui/src/model/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 12536eff..27e90d56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5709,6 +5709,30 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "videocall-client" +version = "0.1.0" +dependencies = [ + "aes", + "anyhow", + "cbc", + "gloo 0.8.1", + "gloo-timers", + "gloo-utils", + "js-sys", + "log", + "protobuf", + "rand 0.8.5", + "rsa", + "types", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "yew", + "yew-websocket", + "yew-webtransport", +] + [[package]] name = "vswhom" version = "0.1.0" @@ -6476,9 +6500,6 @@ dependencies = [ name = "yew-ui" version = "0.1.0" dependencies = [ - "aes", - "anyhow", - "cbc", "console_error_panic_hook", "console_log", "const_format", @@ -6486,25 +6507,21 @@ dependencies = [ "gloo 0.8.1", "gloo-timers", "gloo-utils", - "js-sys", "lazy_static", "log", "protobuf", - "rand 0.8.5", "reqwasm", - "rsa", "serde", "serde_derive", "types", "urlencoding", + "videocall-client", "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test", "web-sys", "yew", "yew-router", - "yew-websocket", - "yew-webtransport", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b3d566c2..20a59e4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,5 +5,6 @@ members = [ "bot", "src-tauri", "types", + "videocall-client", "yew-ui" ] diff --git a/videocall-client/Cargo.toml b/videocall-client/Cargo.toml new file mode 100644 index 00000000..ebb4d583 --- /dev/null +++ b/videocall-client/Cargo.toml @@ -0,0 +1,92 @@ +[package] +name = "videocall-client" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +aes = "0.8.3" +anyhow = "1" +cbc = { version = "0.1.2", features = ["alloc"] } +gloo = "0.8.0" +gloo-timers = "0.2.6" +gloo-utils = "0.1" +js-sys = "0.3" +log = "0.4.19" +protobuf = "3.2.0" +rand = { version = "0.8.5", features = ["std_rng", "small_rng"] } +rsa = "0.9.2" +types = { path= "../types"} +wasm-bindgen = "0.2.78" +wasm-bindgen-futures = "0.4.30" +yew = { version = "0.20" } +yew-websocket = "1.0.1" +yew-webtransport = "0.3.2" + +[dependencies.web-sys] +version = "0.3.60" +features = [ + "AudioTrack", + "AudioData", + "AudioEncoder", + "AudioEncoderInit", + "AudioEncoderConfig", + "AudioDecoder", + "AudioDecoderInit", + "AudioDecoderConfig", + "AudioContext", + "BaseAudioContext", + "GainOptions", + "GainNode", + "console", + "CodecState", + "CanvasRenderingContext2d", + "EncodedAudioChunk", + "EncodedAudioChunkInit", + "EncodedAudioChunkType", + "EncodedVideoChunk", + "EncodedVideoChunkInit", + "EncodedVideoChunkType", + "MediaStreamAudioDestinationNode", + "AudioDestinationNode", + "AudioContextOptions", + "AudioDataCopyToOptions", + "HtmlCanvasElement", + "HtmlImageElement", + "Navigator", + "MediaDevices", + "MediaStream", + "MediaStreamTrack", + "MediaTrackSettings", + "MediaStreamTrackProcessor", + "MediaStreamTrackProcessorInit", + "MediaStreamTrackGenerator", + "MediaStreamTrackGeneratorInit", + "WritableStream", + "WritableStreamDefaultWriter", + "MediaStreamAudioSourceNode", + "HtmlVideoElement", + "MediaStreamConstraints", + "ReadableStream", + "ReadableStreamGetReaderOptions", + "ReadableStreamDefaultReader", + "VideoEncoder", + "VideoEncoderInit", + "VideoEncoderConfig", + "VideoEncoderEncodeOptions", + "VideoFrame", + "VideoTrack", + "VideoDecoder", + "VideoDecoderConfig", + "VideoDecoderInit", + "LatencyMode", + "HtmlAudioElement", + "AudioDataInit", + "AudioSampleFormat", + "TransformStream", + "MediaDeviceInfo", + "MediaDeviceKind", + "MediaTrackConstraints", + "CanvasRenderingContext2d" +] diff --git a/videocall-client/README.md b/videocall-client/README.md new file mode 100644 index 00000000..63de216a --- /dev/null +++ b/videocall-client/README.md @@ -0,0 +1,93 @@ + + +# videocall-client + +Version: 0.1.0 + +--- + +This crate provides a client-side (browser) interface to the videocall protocol. The purpose is to +take care of the media I/O both for the encoding the current participant and for rendering the +media from the remote peers. It also provides tools for listing available media devices and +granting access. + +This crate intends to make no assumptions about the UI or the HTML of the client app. +The only DOM data it needs is the ID of the `HtmlVideoElement` for the participant's own video +display and the ID's of the `HtmlCanvasElement`s into which remote peer video should be renderered. + +In addition to its use by Rust UI apps (e.g. via yew), it is intended that this crate be +compiled to npm module that could be called from javascript, e.g. in an electron app. + +Currently, only the Chrome browser is supported, due to some of the Web APIs that are used. + +**NOTE:** This initial version is a slightly frankenstein result of piecemeal refactoring bits +from the original app and stitching them together. It could use cleaning up both the API the +internal design. + +## Outline of usage + +For more detailed documentation see the doc for each struct. + +### Client creation and connection: +```rust +let options = VideoCallClientOptions {...}; // set parameters and callbacks for various events +let client = VideoCallClient::new(options); + +client.connect(); +``` + +### Encoder creation: +```rust +let camera = CameraEncoder.new(client, video_element_id); +let microphone = MicrophoneEncoder.new(client); +let screen = ScreenEncoder.new(client); + +camera.select(video_device); +camera.start(); +camera.stop(); +microphone.select(video_device); +microphone.start(); +microphone.stop(); +screen.start(); +screen.stop(); +``` + +### Device access permission: + +```rust +let media_device_access = MediaDeviceAccess::new(); +media_device_access.on_granted = ...; // callback +media_device_access.on_denied = ...; // callback +media_device_access.request(); +``` + +#### Device query and listing: +```rust +let media_device_list = MediaDeviceList::new(); +media_device_list.audio_inputs.on_selected = ...; // callback +media_device_access.video_inputs.on_selected = ...; // callback + +media_device_list.load(); + +let microphones = media_device_list.audio_inputs.devices(); +let cameras = media_device_list.video_inputs.devices(); +media_device_list.audio_inputs.select(µphones[i].device_id); +media_device_list.video_inputs.select(&cameras[i].device_id); + +``` + +--- + +*Readme created using [cargo-readme](https://github.com/webern/cargo-readme) using [README.tpl](./README.tpl)* diff --git a/videocall-client/README.tpl b/videocall-client/README.tpl new file mode 100644 index 00000000..b333a4b9 --- /dev/null +++ b/videocall-client/README.tpl @@ -0,0 +1,25 @@ + + +# {{crate}} + +Version: {{version}} + +--- + +{{readme}} + +--- + +*Readme created using [cargo-readme](https://github.com/webern/cargo-readme) using [README.tpl](./README.tpl)* diff --git a/videocall-client/rust-toolchain.toml b/videocall-client/rust-toolchain.toml new file mode 100644 index 00000000..f4ed9c82 --- /dev/null +++ b/videocall-client/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "nightly-2023-01-27" +targets = [ "wasm32-unknown-unknown" ] diff --git a/yew-ui/src/model/client/mod.rs b/videocall-client/src/client/mod.rs similarity index 100% rename from yew-ui/src/model/client/mod.rs rename to videocall-client/src/client/mod.rs diff --git a/yew-ui/src/model/client/video_call_client.rs b/videocall-client/src/client/video_call_client.rs similarity index 78% rename from yew-ui/src/model/client/video_call_client.rs rename to videocall-client/src/client/video_call_client.rs index 77eecfcf..36c34a18 100644 --- a/yew-ui/src/model/client/video_call_client.rs +++ b/videocall-client/src/client/video_call_client.rs @@ -16,18 +16,44 @@ use types::protos::packet_wrapper::packet_wrapper::PacketType; use types::protos::packet_wrapper::PacketWrapper; use types::protos::rsa_packet::RsaPacket; use yew::prelude::Callback; + +/// Options struct for constructing a client via [VideoCallClient::new(options)][VideoCallClient::new] #[derive(Clone, Debug, PartialEq)] pub struct VideoCallClientOptions { + /// `true` to use end-to-end encription; `false` to send data unencrypted pub enable_e2ee: bool, + + /// `true` to use webtransport, `false` to use websocket pub enable_webtransport: bool, + + /// Callback will be called as `callback(peer_userid)` when a new peer is added pub on_peer_added: Callback, + + /// Callback will be called as `callback(peer_userid, media_type)` immediately after the first frame of a given peer & media type is decoded pub on_peer_first_frame: Callback<(String, MediaType)>, + + /// Callback will be called as `callback(peer_userid)` and must return the DOM id of the + /// `HtmlCanvasElement` into which the peer video should be rendered pub get_peer_video_canvas_id: Callback, + + /// Callback will be called as `callback(peer_userid)` and must return the DOM id of the + /// `HtmlCanvasElement` into which the peer screen image should be rendered pub get_peer_screen_canvas_id: Callback, + + /// The current client's userid. This userid will appear as this client's `peer_userid` in the + /// remote peers' clients. pub userid: String, + + /// The url to which WebSocket connections should be made pub websocket_url: String, + + /// The url to which WebTransport connections should be made pub webtransport_url: String, + + /// Callback will be called as `callback(())` after a new connection is made pub on_connected: Callback<()>, + + /// Callback will be called as `callback(())` if a connection gets dropped pub on_connection_lost: Callback<()>, } @@ -47,6 +73,12 @@ struct Inner { peer_decode_manager: PeerDecodeManager, } +/// The client struct for a video call connection. +/// +/// To use it, first construct the struct using [new(options)][Self::new]. Then when/if desired, +/// create the connection using [connect()][Self::connect]. Once connected, decoding of media from +/// remote peers will start immediately. +/// #[derive(Clone, Debug)] pub struct VideoCallClient { options: VideoCallClientOptions, @@ -61,6 +93,10 @@ impl PartialEq for VideoCallClient { } impl VideoCallClient { + /// Constructor for the client struct. + /// + /// See [VideoCallClientOptions] for description of the options. + /// pub fn new(options: VideoCallClientOptions) -> Self { let aes = Arc::new(Aes128State::new(options.enable_e2ee)); let inner = Rc::new(RefCell::new(Inner { @@ -81,6 +117,21 @@ impl VideoCallClient { } } + /// Initiates a connection to a videocall server. + /// + /// Initiates a connection using WebTransport (to + /// [`options.webtransport_url`](VideoCallClientOptions::webtransport_url)) or WebSocket (to + /// [`options.websocket_url`](VideoCallClientOptions::websocket_url)), based on the value of + /// [`options.enable_webtransport`](VideoCallClientOptions::enable_webtransport). + /// + /// Note that this method's success means only that it succesfully *attempted* initiation of the + /// connection. The connection cannot actually be considered to have been succesful until the + /// [`options.on_connected`](VideoCallClientOptions::on_connected) callback has been invoked. + /// + /// If the connection does not succeed, the + /// [`options.on_connection_lost`](VideoCallClientOptions::on_connection_lost) callback will be + /// invoked. + /// pub fn connect(&mut self) -> anyhow::Result<()> { let options = ConnectOptions { userid: self.options.userid.clone(), @@ -145,7 +196,7 @@ impl VideoCallClient { peer_decode_manager } - pub fn send_packet(&self, media: PacketWrapper) { + pub(crate) fn send_packet(&self, media: PacketWrapper) { match self.inner.try_borrow() { Ok(inner) => inner.send_packet(media), Err(_) => { @@ -154,6 +205,7 @@ impl VideoCallClient { } } + /// Returns `true` if the client is currently connected to a server. pub fn is_connected(&self) -> bool { if let Ok(inner) = self.inner.try_borrow() { if let Some(connection) = &inner.connection { @@ -163,6 +215,7 @@ impl VideoCallClient { false } + /// Returns a vector of the userids of the currently connected remote peers, sorted alphabetically. pub fn sorted_peer_keys(&self) -> Vec { match self.inner.try_borrow() { Ok(inner) => inner.peer_decode_manager.sorted_keys().to_vec(), @@ -170,6 +223,13 @@ impl VideoCallClient { } } + /// Hacky function that returns true if the given peer has yet to send a frame of screen share. + /// + /// No reason for this function to exist, it should be deducible from the + /// [`options.on_peer_first_frame(key, MediaType::Screen)`](VideoCallClientOptions::on_peer_first_frame) + /// callback. Or if polling is really necessary, instead of being hardwired for screen, it'd + /// be more elegant to at least pass a `MediaType`. + /// pub fn is_awaiting_peer_screen_frame(&self, key: &String) -> bool { if let Ok(inner) = self.inner.try_borrow() { if let Some(peer) = inner.peer_decode_manager.get(key) { @@ -179,10 +239,11 @@ impl VideoCallClient { false } - pub fn aes(&self) -> Arc { + pub(crate) fn aes(&self) -> Arc { self.aes.clone() } + /// Returns a reference to a copy of [`options.userid`](VideoCallClientOptions::userid) pub fn userid(&self) -> &String { &self.options.userid } diff --git a/yew-ui/src/model/connection/connection.rs b/videocall-client/src/connection/connection.rs similarity index 100% rename from yew-ui/src/model/connection/connection.rs rename to videocall-client/src/connection/connection.rs diff --git a/yew-ui/src/model/connection/mod.rs b/videocall-client/src/connection/mod.rs similarity index 80% rename from yew-ui/src/model/connection/mod.rs rename to videocall-client/src/connection/mod.rs index 03487c88..d559d1e3 100644 --- a/yew-ui/src/model/connection/mod.rs +++ b/videocall-client/src/connection/mod.rs @@ -1,3 +1,4 @@ +#[allow(clippy::module_inception)] mod connection; mod task; mod webmedia; diff --git a/yew-ui/src/model/connection/task.rs b/videocall-client/src/connection/task.rs similarity index 100% rename from yew-ui/src/model/connection/task.rs rename to videocall-client/src/connection/task.rs diff --git a/yew-ui/src/model/connection/webmedia.rs b/videocall-client/src/connection/webmedia.rs similarity index 100% rename from yew-ui/src/model/connection/webmedia.rs rename to videocall-client/src/connection/webmedia.rs diff --git a/yew-ui/src/model/connection/websocket.rs b/videocall-client/src/connection/websocket.rs similarity index 100% rename from yew-ui/src/model/connection/websocket.rs rename to videocall-client/src/connection/websocket.rs diff --git a/yew-ui/src/model/connection/webtransport.rs b/videocall-client/src/connection/webtransport.rs similarity index 100% rename from yew-ui/src/model/connection/webtransport.rs rename to videocall-client/src/connection/webtransport.rs diff --git a/videocall-client/src/constants.rs b/videocall-client/src/constants.rs new file mode 100644 index 00000000..6314e21e --- /dev/null +++ b/videocall-client/src/constants.rs @@ -0,0 +1,26 @@ +pub static AUDIO_CODEC: &str = "opus"; // https://www.w3.org/TR/webcodecs-codec-registry/#audio-codec-registry +pub static VIDEO_CODEC: &str = "vp09.00.10.08"; // profile 0,level 1.0, bit depth 8, + +// Commented out because it is not as fast as vp9. + +// pub static VIDEO_CODEC: &str = "av01.0.01M.08"; +// av01: AV1 +// 0 profile: main profile +// 01 level: level2.1 +// M tier: Main tier +// 08 bit depth = 8 bits + +pub const AUDIO_CHANNELS: u32 = 1u32; +pub const AUDIO_SAMPLE_RATE: u32 = 48000u32; +pub const AUDIO_BITRATE: f64 = 50000f64; + +// vga resolution +// pub const VIDEO_HEIGHT: i32 = 480i32; +// pub const VIDEO_WIDTH: i32 = 640i32; + +pub const VIDEO_HEIGHT: i32 = 720i32; +pub const VIDEO_WIDTH: i32 = 1280i32; +pub const SCREEN_HEIGHT: u32 = 1080u32; +pub const SCREEN_WIDTH: u32 = 1920u32; + +pub const RSA_BITS: usize = 1024; diff --git a/yew-ui/src/crypto/aes.rs b/videocall-client/src/crypto/aes.rs similarity index 100% rename from yew-ui/src/crypto/aes.rs rename to videocall-client/src/crypto/aes.rs diff --git a/yew-ui/src/crypto/mod.rs b/videocall-client/src/crypto/mod.rs similarity index 100% rename from yew-ui/src/crypto/mod.rs rename to videocall-client/src/crypto/mod.rs diff --git a/yew-ui/src/crypto/rsa.rs b/videocall-client/src/crypto/rsa.rs similarity index 100% rename from yew-ui/src/crypto/rsa.rs rename to videocall-client/src/crypto/rsa.rs diff --git a/yew-ui/src/model/decode/config.rs b/videocall-client/src/decode/config.rs similarity index 100% rename from yew-ui/src/model/decode/config.rs rename to videocall-client/src/decode/config.rs diff --git a/yew-ui/src/model/decode/hash_map_with_ordered_keys.rs b/videocall-client/src/decode/hash_map_with_ordered_keys.rs similarity index 100% rename from yew-ui/src/model/decode/hash_map_with_ordered_keys.rs rename to videocall-client/src/decode/hash_map_with_ordered_keys.rs diff --git a/yew-ui/src/model/decode/mod.rs b/videocall-client/src/decode/mod.rs similarity index 100% rename from yew-ui/src/model/decode/mod.rs rename to videocall-client/src/decode/mod.rs diff --git a/yew-ui/src/model/decode/peer_decode_manager.rs b/videocall-client/src/decode/peer_decode_manager.rs similarity index 100% rename from yew-ui/src/model/decode/peer_decode_manager.rs rename to videocall-client/src/decode/peer_decode_manager.rs diff --git a/yew-ui/src/model/decode/peer_decoder.rs b/videocall-client/src/decode/peer_decoder.rs similarity index 100% rename from yew-ui/src/model/decode/peer_decoder.rs rename to videocall-client/src/decode/peer_decoder.rs diff --git a/yew-ui/src/model/decode/video_decoder_with_buffer.rs b/videocall-client/src/decode/video_decoder_with_buffer.rs similarity index 100% rename from yew-ui/src/model/decode/video_decoder_with_buffer.rs rename to videocall-client/src/decode/video_decoder_with_buffer.rs diff --git a/yew-ui/src/model/decode/video_decoder_wrapper.rs b/videocall-client/src/decode/video_decoder_wrapper.rs similarity index 100% rename from yew-ui/src/model/decode/video_decoder_wrapper.rs rename to videocall-client/src/decode/video_decoder_wrapper.rs diff --git a/yew-ui/src/model/encode/camera_encoder.rs b/videocall-client/src/encode/camera_encoder.rs similarity index 77% rename from yew-ui/src/model/encode/camera_encoder.rs rename to videocall-client/src/encode/camera_encoder.rs index 8fc09dc1..c0d4cd2c 100644 --- a/yew-ui/src/model/encode/camera_encoder.rs +++ b/videocall-client/src/encode/camera_encoder.rs @@ -34,6 +34,15 @@ use crate::constants::VIDEO_CODEC; use crate::constants::VIDEO_HEIGHT; use crate::constants::VIDEO_WIDTH; +/// [CameraEncoder] encodes the video from a camera and sends it through a [`VideoCallClient`](crate::VideoCallClient) connection. +/// +/// To use this struct, the caller must first create an `HtmlVideoElement` DOM node, to which the +/// camera will be connected. +/// +/// See also: +/// * [MicrophoneEncoder](crate::MicrophoneEncoder) +/// * [ScreenEncoder](crate::ScreenEncoder) +/// pub struct CameraEncoder { client: VideoCallClient, video_elem_id: String, @@ -41,6 +50,14 @@ pub struct CameraEncoder { } impl CameraEncoder { + /// Construct a camera encoder, with arguments: + /// + /// * `client` - an instance of a [`VideoCallClient`](crate::VideoCallClient). It does not need to be currently connected. + /// + /// * `video_elem_id` - the the ID of an `HtmlVideoElement` to which the camera will be connected. It does not need to currently exist. + /// + /// The encoder is created in a disabled state, [`encoder.set_enabled(true)`](Self::set_enabled) must be called before it can start encoding. + /// The encoder is created without a camera selected, [`encoder.select(device_id)`](Self::select) must be called before it can start encoding. pub fn new(client: VideoCallClient, video_elem_id: &str) -> Self { Self { client, @@ -49,17 +66,38 @@ impl CameraEncoder { } } - // delegates to self.state + // The next three methods delegate to self.state + + /// Enables/disables the encoder. Returns true if the new value is different from the old value. + /// + /// The encoder starts disabled, [`encoder.set_enabled(true)`](Self::set_enabled) must be + /// called prior to starting encoding. + /// + /// Disabling encoding after it has started will cause it to stop. pub fn set_enabled(&mut self, value: bool) -> bool { self.state.set_enabled(value) } - pub fn select(&mut self, device: String) -> bool { - self.state.select(device) + + /// Selects a camera: + /// + /// * `device_id` - The value of `entry.device_id` for some entry in + /// [`media_device_list.video_inputs.devices()`](crate::MediaDeviceList::video_inputs) + /// + /// The encoder starts without a camera associated, + /// [`encoder.selected(device_id)`](Self::select) must be called prior to starting encoding. + pub fn select(&mut self, device_id: String) -> bool { + self.state.select(device_id) } + + /// Stops encoding after it has been started. pub fn stop(&mut self) { self.state.stop() } + /// Start encoding and sending the data to the client connection (if it's currently connected). + /// + /// This will not do anything if [`encoder.set_enabled(true)`](Self::set_enabled) has not been + /// called, or if [`encoder.select(device_id)`](Self::select) has not been called. pub fn start(&mut self) { // 1. Query the first device with a camera and a mic attached. // 2. setup WebCodecs, in particular diff --git a/yew-ui/src/model/encode/encoder_state.rs b/videocall-client/src/encode/encoder_state.rs similarity index 100% rename from yew-ui/src/model/encode/encoder_state.rs rename to videocall-client/src/encode/encoder_state.rs diff --git a/yew-ui/src/model/encode/microphone_encoder.rs b/videocall-client/src/encode/microphone_encoder.rs similarity index 77% rename from yew-ui/src/model/encode/microphone_encoder.rs rename to videocall-client/src/encode/microphone_encoder.rs index 274f31a6..9bae31c8 100644 --- a/yew-ui/src/model/encode/microphone_encoder.rs +++ b/videocall-client/src/encode/microphone_encoder.rs @@ -31,12 +31,24 @@ use crate::constants::AUDIO_CHANNELS; use crate::constants::AUDIO_CODEC; use crate::constants::AUDIO_SAMPLE_RATE; +/// [MicrophoneEncoder] encodes the audio from a microphone and sends it through a [`VideoCallClient`](crate::VideoCallClient) connection. +/// +/// See also: +/// * [CameraEncoder](crate::CameraEncoder) +/// * [ScreenEncoder](crate::ScreenEncoder) +/// pub struct MicrophoneEncoder { client: VideoCallClient, state: EncoderState, } impl MicrophoneEncoder { + /// Construct a microphone encoder: + /// + /// * `client` - an instance of a [`VideoCallClient`](crate::VideoCallClient). It does not need to be currently connected. + /// + /// The encoder is created in a disabled state, [`encoder.set_enabled(true)`](Self::set_enabled) must be called before it can start encoding. + /// The encoder is created without a camera selected, [`encoder.select(device_id)`](Self::select) must be called before it can start encoding. pub fn new(client: VideoCallClient) -> Self { Self { client, @@ -44,17 +56,38 @@ impl MicrophoneEncoder { } } - // delegates to self.state + // The next three methods delegate to self.state + + /// Enables/disables the encoder. Returns true if the new value is different from the old value. + /// + /// The encoder starts disabled, [`encoder.set_enabled(true)`](Self::set_enabled) must be + /// called prior to starting encoding. + /// + /// Disabling encoding after it has started will cause it to stop. pub fn set_enabled(&mut self, value: bool) -> bool { self.state.set_enabled(value) } + + /// Selects a microphone: + /// + /// * `device_id` - The value of `entry.device_id` for some entry in + /// [`media_device_list.audio_inputs.devices()`](crate::MediaDeviceList::audio_inputs) + /// + /// The encoder starts without a microphone associated, + /// [`encoder.selected(device_id)`](Self::select) must be called prior to starting encoding. pub fn select(&mut self, device: String) -> bool { self.state.select(device) } + + /// Stops encoding after it has been started. pub fn stop(&mut self) { self.state.stop() } + /// Start encoding and sending the data to the client connection (if it's currently connected). + /// + /// This will not do anything if [`encoder.set_enabled(true)`](Self::set_enabled) has not been + /// called, or if [`encoder.select(device_id)`](Self::select) has not been called. pub fn start(&mut self) { let device_id = if let Some(mic) = &self.state.selected { mic.to_string() diff --git a/yew-ui/src/model/encode/mod.rs b/videocall-client/src/encode/mod.rs similarity index 100% rename from yew-ui/src/model/encode/mod.rs rename to videocall-client/src/encode/mod.rs diff --git a/yew-ui/src/model/encode/screen_encoder.rs b/videocall-client/src/encode/screen_encoder.rs similarity index 80% rename from yew-ui/src/model/encode/screen_encoder.rs rename to videocall-client/src/encode/screen_encoder.rs index b62e0482..ebdf75d2 100644 --- a/yew-ui/src/model/encode/screen_encoder.rs +++ b/videocall-client/src/encode/screen_encoder.rs @@ -30,12 +30,23 @@ use crate::constants::SCREEN_HEIGHT; use crate::constants::SCREEN_WIDTH; use crate::constants::VIDEO_CODEC; +/// [ScreenEncoder] encodes the user's screen and sends it through a [`VideoCallClient`](crate::VideoCallClient) connection. +/// +/// See also: +/// * [CameraEncoder](crate::CameraEncoder) +/// * [MicrophoneEncoder](crate::MicrophoneEncoder) +/// pub struct ScreenEncoder { client: VideoCallClient, state: EncoderState, } impl ScreenEncoder { + /// Construct a screen encoder: + /// + /// * `client` - an instance of a [`VideoCallClient`](crate::VideoCallClient). It does not need to be currently connected. + /// + /// The encoder is created in a disabled state, [`encoder.set_enabled(true)`](Self::set_enabled) must be called before it can start encoding. pub fn new(client: VideoCallClient) -> Self { Self { client, @@ -43,14 +54,28 @@ impl ScreenEncoder { } } - // delegates to self.state + // The next two methods delegate to self.state + + /// Enables/disables the encoder. Returns true if the new value is different from the old value. + /// + /// The encoder starts disabled, [`encoder.set_enabled(true)`](Self::set_enabled) must be + /// called prior to starting encoding. + /// + /// Disabling encoding after it has started will cause it to stop. pub fn set_enabled(&mut self, value: bool) -> bool { self.state.set_enabled(value) } + + /// Stops encoding after it has been started. pub fn stop(&mut self) { self.state.stop() } + /// Start encoding and sending the data to the client connection (if it's currently connected). + /// The user is prompted by the browser to select which window or screen to encode. + /// + /// This will not do anything if [`encoder.set_enabled(true)`](Self::set_enabled) has not been + /// called. pub fn start(&mut self) { let EncoderState { enabled, destroy, .. diff --git a/yew-ui/src/model/encode/transform.rs b/videocall-client/src/encode/transform.rs similarity index 100% rename from yew-ui/src/model/encode/transform.rs rename to videocall-client/src/encode/transform.rs diff --git a/videocall-client/src/lib.rs b/videocall-client/src/lib.rs new file mode 100644 index 00000000..d4a2f226 --- /dev/null +++ b/videocall-client/src/lib.rs @@ -0,0 +1,84 @@ +//! This crate provides a client-side (browser) interface to the videocall protocol. The purpose is to +//! take care of the media I/O both for the encoding the current participant and for rendering the +//! media from the remote peers. It also provides tools for listing available media devices and +//! granting access. +//! +//! This crate intends to make no assumptions about the UI or the HTML of the client app. +//! The only DOM data it needs is the ID of the `HtmlVideoElement` for the participant's own video +//! display and the ID's of the `HtmlCanvasElement`s into which remote peer video should be renderered. +//! +//! In addition to its use by Rust UI apps (e.g. via yew), it is intended that this crate be +//! compiled to npm module that could be called from javascript, e.g. in an electron app. +//! +//! Currently, only the Chrome browser is supported, due to some of the Web APIs that are used. +//! +//! **NOTE:** This initial version is a slightly frankenstein result of piecemeal refactoring bits +//! from the original app and stitching them together. It could use cleaning up both the API the +//! internal design. +//! +//! # Outline of usage +//! +//! For more detailed documentation see the doc for each struct. +//! +//! ## Client creation and connection: +//! ``` +//! let options = VideoCallClientOptions {...}; // set parameters and callbacks for various events +//! let client = VideoCallClient::new(options); +//! +//! client.connect(); +//! ``` +//! +//! ## Encoder creation: +//! ``` +//! let camera = CameraEncoder.new(client, video_element_id); +//! let microphone = MicrophoneEncoder.new(client); +//! let screen = ScreenEncoder.new(client); +//! +//! camera.select(video_device); +//! camera.start(); +//! camera.stop(); +//! microphone.select(video_device); +//! microphone.start(); +//! microphone.stop(); +//! screen.start(); +//! screen.stop(); +//! ``` +//! +//! ## Device access permission: +//! +//! ``` +//! let media_device_access = MediaDeviceAccess::new(); +//! media_device_access.on_granted = ...; // callback +//! media_device_access.on_denied = ...; // callback +//! media_device_access.request(); +//! ``` +//! +//! ### Device query and listing: +//! ``` +//! let media_device_list = MediaDeviceList::new(); +//! media_device_list.audio_inputs.on_selected = ...; // callback +//! media_device_access.video_inputs.on_selected = ...; // callback +//! +//! media_device_list.load(); +//! +//! let microphones = media_device_list.audio_inputs.devices(); +//! let cameras = media_device_list.video_inputs.devices(); +//! media_device_list.audio_inputs.select(µphones[i].device_id); +//! media_device_list.video_inputs.select(&cameras[i].device_id); +//! +//! ``` + +#![feature(once_cell)] + +mod client; +mod connection; +mod constants; +mod crypto; +mod decode; +mod encode; +mod media_devices; +mod wrappers; + +pub use client::{VideoCallClient, VideoCallClientOptions}; +pub use encode::{CameraEncoder, MicrophoneEncoder, ScreenEncoder}; +pub use media_devices::{MediaDeviceAccess, MediaDeviceList, SelectableDevices}; diff --git a/yew-ui/src/model/media_devices/media_device_access.rs b/videocall-client/src/media_devices/media_device_access.rs similarity index 62% rename from yew-ui/src/model/media_devices/media_device_access.rs rename to videocall-client/src/media_devices/media_device_access.rs index 5709344d..ba229e10 100644 --- a/yew-ui/src/model/media_devices/media_device_access.rs +++ b/videocall-client/src/media_devices/media_device_access.rs @@ -6,13 +6,31 @@ use wasm_bindgen_futures::JsFuture; use web_sys::MediaStreamConstraints; use yew::prelude::Callback; +/// [MediaDeviceAccess] is a utility to request the user's permission to access the microphone and +/// camera. pub struct MediaDeviceAccess { granted: Arc, + + // Callback that is called when the user grants access permission pub on_granted: Callback<()>, + + // Callback that is called when the user fails to grant access permission pub on_denied: Callback<()>, } +#[allow(clippy::new_without_default)] impl MediaDeviceAccess { + /// Constructor for the device access struct. + /// + /// After construction, set the callbacks, then call the [`request()`](Self::request) method to request + /// access, e.g.: + /// + /// ``` + /// let media_device_access = MediaDeviceAccess::new(); + /// media_device_access.on_granted = ...; // callback + /// media_device_access.on_denied = ...; // callback + /// media_device_access.request(); + /// ``` pub fn new() -> Self { Self { granted: Arc::new(AtomicBool::new(false)), @@ -21,10 +39,15 @@ impl MediaDeviceAccess { } } + /// Returns true if permission has been granted pub fn is_granted(&self) -> bool { self.granted.load(Ordering::Acquire) } + /// Causes the browser to request the user's permission to access the microphone and camera. + /// + /// This function returns immediately. Eventually, either the [`on_granted`](Self::on_granted) + /// or [`on_denied`](Self::on_denied) callback will be called. pub fn request(&self) { let future = Self::request_permissions(); let on_granted = self.on_granted.clone(); diff --git a/videocall-client/src/media_devices/media_device_list.rs b/videocall-client/src/media_devices/media_device_list.rs new file mode 100644 index 00000000..44cce9f2 --- /dev/null +++ b/videocall-client/src/media_devices/media_device_list.rs @@ -0,0 +1,188 @@ +use gloo_utils::window; +use js_sys::Array; +use js_sys::Promise; +use std::cell::OnceCell; +use std::sync::Arc; +use wasm_bindgen::JsCast; +use wasm_bindgen_futures::JsFuture; +use web_sys::MediaDeviceInfo; +use web_sys::MediaDeviceKind; +use yew::prelude::Callback; + +/// A "smart" list of [web_sys::MediaDeviceInfo](web_sys::MediaDeviceInfo) items, used by [MediaDeviceList] +/// +/// The list keeps track of a currently selected device, supporting selection and a callback that +/// is triggered when a selection is made. +/// +pub struct SelectableDevices { + devices: Arc>>, + selected: Option, + + /// Callback that will be called as `callback(device_id)` whenever [`select(device_id)`](Self::select) is called with a valid `device_id` + pub on_selected: Callback, +} + +impl SelectableDevices { + fn new() -> Self { + Self { + devices: Arc::new(OnceCell::new()), + selected: None, + on_selected: Callback::noop(), + } + } + + /// Select a device: + /// + /// * `device_id` - The `device_id` field of an entry in [`devices()`](Self::devices) + /// + /// Triggers the [`on_selected(device_id)`](Self::on_selected) callback. + /// + /// Does nothing if the device_id is not in [`devices()`](Self::devices). + /// + /// **Note**: Selecting a device here does *not* automatically perform the corresponding + /// call to [`CameraEncoder::select(device_id)`](crate::CameraEncoder::select) or + /// [`MicrophoneEncoder::select(device_id)`](crate::MicrophoneEncoder::select) -- the expectation is + /// that the [`on_selected(device_id)`](Self::on_selected) callback will be set to a function + /// that calls the `select` method of the appropriate encoder. + pub fn select(&mut self, device_id: &str) { + if let Some(devices) = self.devices.get() { + for device in devices.iter() { + if device.device_id() == device_id { + self.selected = Some(device_id.to_string()); + self.on_selected.emit(device_id.to_string()); + } + } + } + } + + /// Returns a reference to an array of [MediaDeviceInfo] entries for the available devices. + pub fn devices(&self) -> &[MediaDeviceInfo] { + match self.devices.get() { + Some(devices) => devices, + None => &[], + } + } + + /// Returns the `device_id` of the currently selected device, or "" if there are no devices. + pub fn selected(&self) -> String { + match &self.selected { + Some(selected) => selected.to_string(), + // device 0 is the default selection + None => match self.devices().get(0) { + Some(device) => device.device_id(), + None => "".to_string(), + }, + } + } +} + +/// [MediaDeviceList] is a utility that queries the user's system for the currently +/// available audio and video input devices, and maintains a current selection for each. +/// +/// It does *not* have any explicit connection to [`CameraEncoder`](crate::CameraEncoder) or +/// [`MicrophoneEncoder`](crate::MicrophoneEncoder) -- the calling app is responsible for passing +/// the selection info from this utility to the encoders. +/// +/// Outline of usage is: +/// +/// ``` +/// let media_device_list = MediaDeviceList::new(); +/// media_device_list.audio_inputs.on_selected = ...; // callback +/// media_device_access.video_inputs.on_selected = ...; // callback +/// +/// media_device_list.load(); +/// +/// let microphones = media_device_list.audio_inputs.devices(); +/// let cameras = media_device_list.video_inputs.devices(); +/// media_device_list.audio_inputs.select(µphones[i].device_id); +/// media_device_list.video_inputs.select(&cameras[i].device_id); +/// +/// ``` +pub struct MediaDeviceList { + /// The list of audio input devices. This field is `pub` for access through it, but should be considerd "read-only". + pub audio_inputs: SelectableDevices, + + /// The list of video input devices. This field is `pub` for access through it, but should be considerd "read-only". + pub video_inputs: SelectableDevices, + + /// Callback that is called as `callback(())` after loading via [`load()`](Self::load) is complete. + pub on_loaded: Callback<()>, +} + +#[allow(clippy::new_without_default)] +impl MediaDeviceList { + /// Constructor for the media devices list struct. + /// + /// After constructing, the user should set the [`on_selected`](SelectableDevices::on_selected) + /// callbacks, e.g.: + /// + /// ``` + /// let media_device_list = MediaDeviceList::new(); + /// media_device_list.audio_inputs.on_selected = ...; // callback + /// media_device_access.video_inputs.on_selected = ...; // callback + /// ``` + /// + /// After constructing, [`load()`](Self::load) needs to be called to populate the lists. + pub fn new() -> Self { + Self { + audio_inputs: SelectableDevices::new(), + video_inputs: SelectableDevices::new(), + on_loaded: Callback::noop(), + } + } + + /// Queries the user's system to find the available audio and video input devices. + /// + /// This is an asynchronous operation; when it is complete the [`on_loaded`](Self::on_loaded) + /// callback will be triggered. Additionally, by default the first audio input device and + /// first video input device are automatically selected, and their + /// [`on_selected`](SelectableDevices::on_selected) callbacks will be triggered. + /// + /// After loading, the [`audio_inputs`](Self::audio_inputs) and [`video_inputs`](Self::video_inputs) lists + /// will be populated, and can be queried and selected. + pub fn load(&self) { + let on_loaded = self.on_loaded.clone(); + let on_audio_selected = self.audio_inputs.on_selected.clone(); + let on_video_selected = self.video_inputs.on_selected.clone(); + let audio_input_devices = Arc::clone(&self.audio_inputs.devices); + let video_input_devices = Arc::clone(&self.video_inputs.devices); + wasm_bindgen_futures::spawn_local(async move { + let navigator = window().navigator(); + let media_devices = navigator.media_devices().unwrap(); + + let promise: Promise = media_devices + .enumerate_devices() + .expect("enumerate devices"); + let future = JsFuture::from(promise); + let devices = future + .await + .expect("await devices") + .unchecked_into::(); + let devices = devices.to_vec(); + let devices = devices + .into_iter() + .map(|d| d.unchecked_into::()) + .collect::>(); + _ = audio_input_devices.set( + devices + .clone() + .into_iter() + .filter(|device| device.kind() == MediaDeviceKind::Audioinput) + .collect(), + ); + _ = video_input_devices.set( + devices + .into_iter() + .filter(|device| device.kind() == MediaDeviceKind::Videoinput) + .collect(), + ); + on_loaded.emit(()); + if let Some(device) = audio_input_devices.get().unwrap().get(0) { + on_audio_selected.emit(device.device_id()) + } + if let Some(device) = video_input_devices.get().unwrap().get(0) { + on_video_selected.emit(device.device_id()) + } + }); + } +} diff --git a/yew-ui/src/model/media_devices/mod.rs b/videocall-client/src/media_devices/mod.rs similarity index 59% rename from yew-ui/src/model/media_devices/mod.rs rename to videocall-client/src/media_devices/mod.rs index 957b0c25..eec87520 100644 --- a/yew-ui/src/model/media_devices/mod.rs +++ b/videocall-client/src/media_devices/mod.rs @@ -2,4 +2,4 @@ mod media_device_access; mod media_device_list; pub use media_device_access::MediaDeviceAccess; -pub use media_device_list::MediaDeviceList; +pub use media_device_list::{MediaDeviceList, SelectableDevices}; diff --git a/yew-ui/src/model/wrappers.rs b/videocall-client/src/wrappers.rs similarity index 100% rename from yew-ui/src/model/wrappers.rs rename to videocall-client/src/wrappers.rs diff --git a/yew-ui/Cargo.toml b/yew-ui/Cargo.toml index 4cdd7cbc..a6a739d1 100644 --- a/yew-ui/Cargo.toml +++ b/yew-ui/Cargo.toml @@ -8,98 +8,31 @@ edition = "2021" [dependencies] yew = { version = "0.20", features = ["csr"] } wasm-bindgen = "0.2.78" -wasm-bindgen-futures = "0.4.30" reqwasm = "0.5.0" types = { path= "../types"} +videocall-client = { path= "../videocall-client"} console_error_panic_hook = "0.1.7" console_log = "1.0.0" const_format="0.2.25" lazy_static = "1.4.0" log = "0.4.19" +gloo-timers = "0.2.6" gloo-utils = "0.1" yew-router = "0.17" -js-sys = "0.3" -yew-websocket = "1.0.1" -anyhow = "1" serde = "1" serde_derive = "1" protobuf = "3.2.0" gloo = "0.8.0" -gloo-timers = "0.2.6" urlencoding = "2.1.2" -yew-webtransport = "0.3.2" -aes = "0.8.3" -cbc = { version = "0.1.2", features = ["alloc"] } -rsa = "0.9.2" getrandom = { version = "0.2.10", features = ["js"] } -rand = { version = "0.8.5", features = ["std_rng", "small_rng"] } +wasm-bindgen-futures = "0.4.30" [dependencies.web-sys] version = "0.3.60" features = [ - "AudioTrack", - "AudioData", - "AudioEncoder", - "AudioEncoderInit", - "AudioEncoderConfig", - "AudioDecoder", - "AudioDecoderInit", - "AudioDecoderConfig", - "AudioContext", - "BaseAudioContext", - "GainOptions", - "GainNode", "HtmlDocument", "console", - "CodecState", - "CanvasRenderingContext2d", - "EncodedAudioChunk", - "EncodedAudioChunkInit", - "EncodedAudioChunkType", - "EncodedVideoChunk", - "EncodedVideoChunkInit", - "EncodedVideoChunkType", - "MediaStreamAudioDestinationNode", - "AudioDestinationNode", - "AudioContextOptions", - "AudioDataCopyToOptions", - "HtmlCanvasElement", - "HtmlImageElement", "Navigator", - "MediaDevices", - "MediaStream", - "MediaStreamTrack", - "MediaTrackSettings", - "MediaStreamTrackProcessor", - "MediaStreamTrackProcessorInit", - "MediaStreamTrackGenerator", - "MediaStreamTrackGeneratorInit", - "WritableStream", - "WritableStreamDefaultWriter", - "MediaStreamAudioSourceNode", - "HtmlVideoElement", - "MediaStreamConstraints", - "ReadableStream", - "ReadableStreamGetReaderOptions", - "ReadableStreamDefaultReader", - "VideoEncoder", - "VideoEncoderInit", - "VideoEncoderConfig", - "VideoEncoderEncodeOptions", - "VideoFrame", - "VideoTrack", - "VideoDecoder", - "VideoDecoderConfig", - "VideoDecoderInit", - "LatencyMode", - "HtmlAudioElement", - "AudioDataInit", - "AudioSampleFormat", - "TransformStream", - "MediaDeviceInfo", - "MediaDeviceKind", - "MediaTrackConstraints", - "CanvasRenderingContext2d" ] [dev-dependencies] diff --git a/yew-ui/src/components/attendants.rs b/yew-ui/src/components/attendants.rs index 9080845a..9fc360f5 100644 --- a/yew-ui/src/components/attendants.rs +++ b/yew-ui/src/components/attendants.rs @@ -1,11 +1,10 @@ use super::icons::push_pin::PushPinIcon; use crate::constants::{USERS_ALLOWED_TO_STREAM, WEBTRANSPORT_HOST}; -use crate::model::client::{VideoCallClient, VideoCallClientOptions}; -use crate::model::media_devices::MediaDeviceAccess; use crate::{components::host::Host, constants::ACTIX_WEBSOCKET}; use log::{error, warn}; use std::rc::Rc; use types::protos::media_packet::media_packet::MediaType; +use videocall_client::{MediaDeviceAccess, VideoCallClient, VideoCallClientOptions}; use wasm_bindgen::JsCast; use wasm_bindgen::JsValue; use web_sys::*; diff --git a/yew-ui/src/components/device_selector.rs b/yew-ui/src/components/device_selector.rs index 584750c9..64a0e666 100644 --- a/yew-ui/src/components/device_selector.rs +++ b/yew-ui/src/components/device_selector.rs @@ -1,4 +1,4 @@ -use crate::model::media_devices::MediaDeviceList; +use videocall_client::MediaDeviceList; use wasm_bindgen::JsCast; use web_sys::HtmlSelectElement; use yew::prelude::*; diff --git a/yew-ui/src/components/host.rs b/yew-ui/src/components/host.rs index 5975cbad..5126325c 100644 --- a/yew-ui/src/components/host.rs +++ b/yew-ui/src/components/host.rs @@ -1,14 +1,11 @@ -use crate::model::client::VideoCallClient; use gloo_timers::callback::Timeout; use log::debug; +use videocall_client::{CameraEncoder, MicrophoneEncoder, ScreenEncoder, VideoCallClient}; use std::fmt::Debug; use yew::prelude::*; use crate::components::device_selector::DeviceSelector; -use crate::model::encode::CameraEncoder; -use crate::model::encode::MicrophoneEncoder; -use crate::model::encode::ScreenEncoder; const VIDEO_ELEMENT_ID: &str = "webcam"; @@ -122,7 +119,6 @@ impl Component for Host { if !should_enable { return true; } - self.camera.start(); true } diff --git a/yew-ui/src/constants.rs b/yew-ui/src/constants.rs index f7831bf6..d6d55959 100644 --- a/yew-ui/src/constants.rs +++ b/yew-ui/src/constants.rs @@ -1,34 +1,8 @@ // This is read at compile time, please restart if you change this value. pub const LOGIN_URL: &str = std::env!("LOGIN_URL"); -pub static AUDIO_CODEC: &str = "opus"; // https://www.w3.org/TR/webcodecs-codec-registry/#audio-codec-registry -pub static VIDEO_CODEC: &str = "vp09.02.10.12"; // profile 0,level 1.0, bit depth 8, - -// Commented out because it is not as fast as vp9. - -// pub static VIDEO_CODEC: &str = "av01.0.01M.08"; -// av01: AV1 -// 0 profile: main profile -// 01 level: level2.1 -// M tier: Main tier -// 08 bit depth = 8 bits - -pub const AUDIO_CHANNELS: u32 = 1u32; -pub const AUDIO_SAMPLE_RATE: u32 = 48000u32; -pub const AUDIO_BITRATE: f64 = 50000f64; - -// vga resolution -// pub const VIDEO_HEIGHT: i32 = 480i32; -// pub const VIDEO_WIDTH: i32 = 640i32; - -pub const VIDEO_HEIGHT: i32 = 720i32; -pub const VIDEO_WIDTH: i32 = 1280i32; -pub const SCREEN_HEIGHT: u32 = 1080u32; -pub const SCREEN_WIDTH: u32 = 1920u32; pub const ACTIX_WEBSOCKET: &str = concat!(std::env!("ACTIX_UI_BACKEND_URL"), "/lobby"); pub const WEBTRANSPORT_HOST: &str = concat!(std::env!("WEBTRANSPORT_HOST"), "/lobby"); -pub const RSA_BITS: usize = 1024; - pub fn truthy(s: Option<&str>) -> bool { if let Some(s) = s { ["true".to_string(), "1".to_string()].contains(&s.to_lowercase()) diff --git a/yew-ui/src/main.rs b/yew-ui/src/main.rs index de7d7ffb..0cbfc435 100644 --- a/yew-ui/src/main.rs +++ b/yew-ui/src/main.rs @@ -1,10 +1,7 @@ #![feature(future_join)] -#![feature(once_cell)] #[allow(non_camel_case_types)] mod components; mod constants; -mod crypto; -mod model; mod pages; use constants::{truthy, E2EE_ENABLED, LOGIN_URL, WEBTRANSPORT_ENABLED}; diff --git a/yew-ui/src/model/media_devices/media_device_list.rs b/yew-ui/src/model/media_devices/media_device_list.rs deleted file mode 100644 index 41d1bed9..00000000 --- a/yew-ui/src/model/media_devices/media_device_list.rs +++ /dev/null @@ -1,117 +0,0 @@ -use gloo_utils::window; -use js_sys::Array; -use js_sys::Promise; -use std::cell::OnceCell; -use std::sync::Arc; -use wasm_bindgen::JsCast; -use wasm_bindgen_futures::JsFuture; -use web_sys::MediaDeviceInfo; -use web_sys::MediaDeviceKind; -use yew::prelude::Callback; - -pub struct SelectableDevices { - devices: Arc>>, - selected: Option, - pub on_selected: Callback, -} - -impl SelectableDevices { - fn new() -> Self { - Self { - devices: Arc::new(OnceCell::new()), - selected: None, - on_selected: Callback::noop(), - } - } - - pub fn select(&mut self, device_id: &str) { - if let Some(devices) = self.devices.get() { - for device in devices.iter() { - if device.device_id() == device_id { - self.selected = Some(device_id.to_string()); - self.on_selected.emit(device_id.to_string()); - } - } - } - } - - pub fn devices(&self) -> &[MediaDeviceInfo] { - match self.devices.get() { - Some(devices) => devices, - None => &[], - } - } - - pub fn selected(&self) -> String { - match &self.selected { - Some(selected) => selected.to_string(), - // device 0 is the default selection - None => match self.devices().get(0) { - Some(device) => device.device_id(), - None => "".to_string(), - }, - } - } -} - -pub struct MediaDeviceList { - pub audio_inputs: SelectableDevices, - pub video_inputs: SelectableDevices, - pub on_loaded: Callback<()>, -} - -impl MediaDeviceList { - pub fn new() -> Self { - Self { - audio_inputs: SelectableDevices::new(), - video_inputs: SelectableDevices::new(), - on_loaded: Callback::noop(), - } - } - - pub fn load(&self) { - let on_loaded = self.on_loaded.clone(); - let on_audio_selected = self.audio_inputs.on_selected.clone(); - let on_video_selected = self.video_inputs.on_selected.clone(); - let audio_input_devices = Arc::clone(&self.audio_inputs.devices); - let video_input_devices = Arc::clone(&self.video_inputs.devices); - wasm_bindgen_futures::spawn_local(async move { - let navigator = window().navigator(); - let media_devices = navigator.media_devices().unwrap(); - - let promise: Promise = media_devices - .enumerate_devices() - .expect("enumerate devices"); - let future = JsFuture::from(promise); - let devices = future - .await - .expect("await devices") - .unchecked_into::(); - let devices = devices.to_vec(); - let devices = devices - .into_iter() - .map(|d| d.unchecked_into::()) - .collect::>(); - _ = audio_input_devices.set( - devices - .clone() - .into_iter() - .filter(|device| device.kind() == MediaDeviceKind::Audioinput) - .collect(), - ); - _ = video_input_devices.set( - devices - .into_iter() - .filter(|device| device.kind() == MediaDeviceKind::Videoinput) - .collect(), - ); - if let Some(device) = audio_input_devices.get().unwrap().get(0) { - on_audio_selected.emit(device.device_id()) - } - if let Some(device) = video_input_devices.get().unwrap().get(0) { - on_video_selected.emit(device.device_id()) - } - on_loaded.emit(()); - }); - } -} diff --git a/yew-ui/src/model/mod.rs b/yew-ui/src/model/mod.rs deleted file mode 100644 index 290602a0..00000000 --- a/yew-ui/src/model/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub mod client; -pub mod connection; -pub mod decode; -pub mod encode; -pub mod media_devices; -pub mod wrappers; - -pub use client::{VideoCallClient, VideoCallClientOptions};