From bfe57433d12a8d4db52c2681bf4a463b96ff80cd Mon Sep 17 00:00:00 2001 From: Nathan Vasse Date: Mon, 20 Jan 2025 18:00:06 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(front)=20add=20blurring=20feature=20o?= =?UTF-8?q?n=20join?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds a button that opens a modal that allow user to enable video effects on join screen. --- .../src/features/home/routes/Home.tsx | 2 +- .../features/rooms/components/Conference.tsx | 17 +++- .../src/features/rooms/components/Join.tsx | 88 ++++++++++++++++++- .../blur/BackgroundBlurCustomProcessor.ts | 13 ++- .../BackgroundBlurTrackProcessorJsWrapper.ts | 13 ++- .../rooms/livekit/components/blur/index.ts | 23 ++++- .../livekit/components/effects/Effects.tsx | 11 ++- .../effects/EffectsConfiguration.tsx | 6 +- .../livekit/hooks/usePersistentUserChoices.ts | 56 ++++++++++++ .../livekit/prefabs/ControlBar/ControlBar.tsx | 2 +- .../src/features/rooms/routes/Room.tsx | 7 +- .../settings/components/tabs/AccountTab.tsx | 6 +- src/frontend/src/locales/en/rooms.json | 4 + src/frontend/src/locales/fr/rooms.json | 4 + 14 files changed, 232 insertions(+), 20 deletions(-) create mode 100644 src/frontend/src/features/rooms/livekit/hooks/usePersistentUserChoices.ts diff --git a/src/frontend/src/features/home/routes/Home.tsx b/src/frontend/src/features/home/routes/Home.tsx index a4903d59..113affc8 100644 --- a/src/frontend/src/features/home/routes/Home.tsx +++ b/src/frontend/src/features/home/routes/Home.tsx @@ -9,7 +9,6 @@ import { useUser, UserAware } from '@/features/auth' import { JoinMeetingDialog } from '../components/JoinMeetingDialog' import { ProConnectButton } from '@/components/ProConnectButton' import { useCreateRoom } from '@/features/rooms' -import { usePersistentUserChoices } from '@livekit/components-react' import { RiAddLine, RiLink } from '@remixicon/react' import { LaterMeetingDialog } from '@/features/home/components/LaterMeetingDialog' import { IntroSlider } from '@/features/home/components/IntroSlider' @@ -18,6 +17,7 @@ import { ReactNode, useState } from 'react' import { css } from '@/styled-system/css' import { menuRecipe } from '@/primitives/menuRecipe.ts' +import { usePersistentUserChoices } from '@/features/rooms/livekit/hooks/usePersistentUserChoices' const Columns = ({ children }: { children?: ReactNode }) => { return ( diff --git a/src/frontend/src/features/rooms/components/Conference.tsx b/src/frontend/src/features/rooms/components/Conference.tsx index 47638edb..fbde83a0 100644 --- a/src/frontend/src/features/rooms/components/Conference.tsx +++ b/src/frontend/src/features/rooms/components/Conference.tsx @@ -1,7 +1,7 @@ import { useEffect, useMemo, useState } from 'react' import { useQuery } from '@tanstack/react-query' import { useTranslation } from 'react-i18next' -import { LiveKitRoom, type LocalUserChoices } from '@livekit/components-react' +import { LiveKitRoom } from '@livekit/components-react' import { Room, RoomOptions } from 'livekit-client' import { keys } from '@/api/queryKeys' import { queryClient } from '@/api/queryClient' @@ -12,10 +12,11 @@ import { fetchRoom } from '../api/fetchRoom' import { ApiRoom } from '../api/ApiRoom' import { useCreateRoom } from '../api/createRoom' import { InviteDialog } from './InviteDialog' - import { VideoConference } from '../livekit/prefabs/VideoConference' import posthog from 'posthog-js' import { css } from '@/styled-system/css' +import { LocalUserChoices } from '../routes/Room' +import { BackgroundBlurFactory } from '../livekit/components/blur' export const Conference = ({ roomId, @@ -102,6 +103,8 @@ export const Conference = ({ peerConnectionTimeout: 60000, // Default: 15s. Extended for slow TURN/TLS negotiation } + console.log('ROOM', userConfig) + return ( @@ -111,7 +114,15 @@ export const Conference = ({ token={data?.livekit?.token} connect={true} audio={userConfig.audioEnabled} - video={userConfig.videoEnabled} + video={ + userConfig.videoEnabled + ? { + processor: BackgroundBlurFactory.deserializeProcessor( + userConfig.processorSerialized + ), + } + : false + } connectOptions={connectOptions} className={css({ backgroundColor: 'primaryDark.50 !important', diff --git a/src/frontend/src/features/rooms/components/Join.tsx b/src/frontend/src/features/rooms/components/Join.tsx index 674212da..140dde5d 100644 --- a/src/frontend/src/features/rooms/components/Join.tsx +++ b/src/frontend/src/features/rooms/components/Join.tsx @@ -1,8 +1,7 @@ import { useTranslation } from 'react-i18next' import { - usePersistentUserChoices, + ParticipantPlaceholder, usePreviewTracks, - type LocalUserChoices, } from '@livekit/components-react' import { css } from '@/styled-system/css' import { Screen } from '@/layout/Screen' @@ -13,6 +12,13 @@ import { SelectToggleDevice } from '../livekit/components/controls/SelectToggleD import { Field } from '@/primitives/Field' import { Form } from '@/primitives' import { HStack, VStack } from '@/styled-system/jsx' +import { Button, Dialog } from '@/primitives' +import { LocalUserChoices } from '../routes/Room' +import { Heading } from 'react-aria-components' +import { RiImageCircleAiFill } from '@remixicon/react' +import { EffectsConfiguration } from '../livekit/components/effects/EffectsConfiguration' +import { usePersistentUserChoices } from '../livekit/hooks/usePersistentUserChoices' +import { BackgroundBlurFactory } from '../livekit/components/blur' const onError = (e: Error) => console.error('ERROR', e) @@ -28,6 +34,7 @@ export const Join = ({ saveAudioInputDeviceId, saveVideoInputDeviceId, saveUsername, + saveProcessorSerialized, } = usePersistentUserChoices({}) const [audioDeviceId, setAudioDeviceId] = useState( @@ -37,6 +44,11 @@ export const Join = ({ initialUserChoices.videoDeviceId ) const [username, setUsername] = useState(initialUserChoices.username) + const [processor, setProcessor] = useState( + BackgroundBlurFactory.deserializeProcessor( + initialUserChoices.processorSerialized + ) + ) useEffect(() => { saveAudioInputDeviceId(audioDeviceId) @@ -49,6 +61,9 @@ export const Join = ({ useEffect(() => { saveUsername(username) }, [username, saveUsername]) + useEffect(() => { + saveProcessorSerialized(processor?.serialize()) + }, [processor, saveProcessorSerialized]) const [audioEnabled, setAudioEnabled] = useState(true) const [videoEnabled, setVideoEnabled] = useState(true) @@ -110,8 +125,48 @@ export const Join = ({ }) } + const [isEffectsOpen, setEffectsOpen] = useState(false) + + const openEffects = () => { + setEffectsOpen(true) + } + + // This hook is used to setup the persisted user choice processor on initialization. + // So it's on purpose that processor is not included in the deps. + // We just want to wait for the videoTrack to be loaded to apply the default processor. + useEffect(() => { + if (processor && videoTrack && !videoTrack.getProcessor()) { + videoTrack.setProcessor(processor) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [videoTrack]) + return ( + + + {t('effects.title')} + + { + setProcessor(processor) + }} + /> +
)} +
+
+ +
- { update(opts: BackgroundOptions): void options: BackgroundOptions clone(): BackgroundBlurProcessorInterface + serialize(): ProcessorSerialized +} + +export enum ProcessorType { + BLUR = 'blur', } export class BackgroundBlurFactory { @@ -21,13 +31,22 @@ export class BackgroundBlurFactory { ) } - static getProcessor(opts: BackgroundOptions) { + static getProcessor( + opts: BackgroundOptions + ): BackgroundBlurProcessorInterface | undefined { if (ProcessorWrapper.isSupported) { return new BackgroundBlurTrackProcessorJsWrapper(opts) } if (BackgroundBlurCustomProcessor.isSupported) { return new BackgroundBlurCustomProcessor(opts) } - return null + return undefined + } + + static deserializeProcessor(data?: ProcessorSerialized) { + if (data?.type === ProcessorType.BLUR) { + return BackgroundBlurFactory.getProcessor(data?.options) + } + return undefined } } diff --git a/src/frontend/src/features/rooms/livekit/components/effects/Effects.tsx b/src/frontend/src/features/rooms/livekit/components/effects/Effects.tsx index 3c2aee55..d8eb1278 100644 --- a/src/frontend/src/features/rooms/livekit/components/effects/Effects.tsx +++ b/src/frontend/src/features/rooms/livekit/components/effects/Effects.tsx @@ -2,17 +2,26 @@ import { useLocalParticipant } from '@livekit/components-react' import { LocalVideoTrack } from 'livekit-client' import { css } from '@/styled-system/css' import { EffectsConfiguration } from './EffectsConfiguration' +import { usePersistentUserChoices } from '../../hooks/usePersistentUserChoices' export const Effects = () => { const { cameraTrack } = useLocalParticipant() const localCameraTrack = cameraTrack?.track as LocalVideoTrack + const { saveProcessorSerialized } = usePersistentUserChoices() + return (
- + + saveProcessorSerialized(processor?.serialize()) + } + />
) } diff --git a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx index 235a31b8..83cb9bb0 100644 --- a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx +++ b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx @@ -1,4 +1,4 @@ -import { LocalVideoTrack, Track, TrackProcessor } from 'livekit-client' +import { LocalVideoTrack } from 'livekit-client' import { useEffect, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { @@ -33,7 +33,7 @@ export const EffectsConfiguration = ({ layout = 'horizontal', }: { videoTrack: LocalVideoTrack - onSubmit?: (processor?: TrackProcessor) => void + onSubmit?: (processor?: BackgroundBlurProcessorInterface) => void layout?: 'vertical' | 'horizontal' }) => { const videoRef = useRef(null) @@ -68,6 +68,8 @@ export const EffectsConfiguration = ({ onSubmit?.(newProcessor) } else { processor?.update({ blurRadius }) + // We want to trigger onSubmit when options changes so the parent component is aware of it. + onSubmit?.(processor) } } catch (error) { console.error('Error applying blur:', error) diff --git a/src/frontend/src/features/rooms/livekit/hooks/usePersistentUserChoices.ts b/src/frontend/src/features/rooms/livekit/hooks/usePersistentUserChoices.ts new file mode 100644 index 00000000..7e3846fb --- /dev/null +++ b/src/frontend/src/features/rooms/livekit/hooks/usePersistentUserChoices.ts @@ -0,0 +1,56 @@ +import { UsePersistentUserChoicesOptions } from '@livekit/components-react' +import React from 'react' +import { LocalUserChoices } from '../../routes/Room' +import { saveUserChoices, loadUserChoices } from '@livekit/components-core' +import { ProcessorSerialized } from '../components/blur' + +/** + * From @livekit/component-react + * + * A hook that provides access to user choices stored in local storage, such as + * selected media devices and their current state (on or off), as well as the user name. + * @alpha + */ +export function usePersistentUserChoices( + options: UsePersistentUserChoicesOptions = {} +) { + const [userChoices, setSettings] = React.useState( + loadUserChoices(options.defaults, options.preventLoad ?? false) + ) + + const saveAudioInputEnabled = React.useCallback((isEnabled: boolean) => { + setSettings((prev) => ({ ...prev, audioEnabled: isEnabled })) + }, []) + const saveVideoInputEnabled = React.useCallback((isEnabled: boolean) => { + setSettings((prev) => ({ ...prev, videoEnabled: isEnabled })) + }, []) + const saveAudioInputDeviceId = React.useCallback((deviceId: string) => { + setSettings((prev) => ({ ...prev, audioDeviceId: deviceId })) + }, []) + const saveVideoInputDeviceId = React.useCallback((deviceId: string) => { + setSettings((prev) => ({ ...prev, videoDeviceId: deviceId })) + }, []) + const saveUsername = React.useCallback((username: string) => { + setSettings((prev) => ({ ...prev, username: username })) + }, []) + const saveProcessorSerialized = React.useCallback( + (processorSerialized?: ProcessorSerialized) => { + setSettings((prev) => ({ ...prev, processorSerialized })) + }, + [] + ) + + React.useEffect(() => { + saveUserChoices(userChoices, options.preventSave ?? false) + }, [userChoices, options.preventSave]) + + return { + userChoices, + saveAudioInputEnabled, + saveVideoInputEnabled, + saveAudioInputDeviceId, + saveVideoInputDeviceId, + saveUsername, + saveProcessorSerialized, + } +} diff --git a/src/frontend/src/features/rooms/livekit/prefabs/ControlBar/ControlBar.tsx b/src/frontend/src/features/rooms/livekit/prefabs/ControlBar/ControlBar.tsx index c61161de..397ec3e4 100644 --- a/src/frontend/src/features/rooms/livekit/prefabs/ControlBar/ControlBar.tsx +++ b/src/frontend/src/features/rooms/livekit/prefabs/ControlBar/ControlBar.tsx @@ -1,11 +1,11 @@ import { Track } from 'livekit-client' import * as React from 'react' -import { usePersistentUserChoices } from '@livekit/components-react' import { MobileControlBar } from './MobileControlBar' import { DesktopControlBar } from './DesktopControlBar' import { SettingsDialogProvider } from '../../components/controls/SettingsDialogContext' import { useIsMobile } from '@/utils/useIsMobile' +import { usePersistentUserChoices } from '../../hooks/usePersistentUserChoices' /** @public */ export type ControlBarControls = { diff --git a/src/frontend/src/features/rooms/routes/Room.tsx b/src/frontend/src/features/rooms/routes/Room.tsx index 2424495e..c0c2cd7a 100644 --- a/src/frontend/src/features/rooms/routes/Room.tsx +++ b/src/frontend/src/features/rooms/routes/Room.tsx @@ -1,7 +1,7 @@ import { useEffect, useState } from 'react' import { usePersistentUserChoices, - type LocalUserChoices, + type LocalUserChoices as LocalUserChoicesLK, } from '@livekit/components-react' import { useParams } from 'wouter' import { ErrorScreen } from '@/components/ErrorScreen' @@ -9,6 +9,11 @@ import { useUser, UserAware } from '@/features/auth' import { Conference } from '../components/Conference' import { Join } from '../components/Join' import { useKeyboardShortcuts } from '@/features/shortcuts/useKeyboardShortcuts' +import { ProcessorSerialized } from '../livekit/components/blur' + +export type LocalUserChoices = LocalUserChoicesLK & { + processorSerialized?: ProcessorSerialized +} export const Room = () => { const { isLoggedIn } = useUser() diff --git a/src/frontend/src/features/settings/components/tabs/AccountTab.tsx b/src/frontend/src/features/settings/components/tabs/AccountTab.tsx index 89fb831c..77c9f655 100644 --- a/src/frontend/src/features/settings/components/tabs/AccountTab.tsx +++ b/src/frontend/src/features/settings/components/tabs/AccountTab.tsx @@ -1,15 +1,13 @@ import { A, Badge, Button, DialogProps, Field, H, P } from '@/primitives' import { Trans, useTranslation } from 'react-i18next' -import { - usePersistentUserChoices, - useRoomContext, -} from '@livekit/components-react' +import { useRoomContext } from '@livekit/components-react' import { logoutUrl, useUser } from '@/features/auth' import { css } from '@/styled-system/css' import { TabPanel, TabPanelProps } from '@/primitives/Tabs' import { HStack } from '@/styled-system/jsx' import { useState } from 'react' import { ProConnectButton } from '@/components/ProConnectButton' +import { usePersistentUserChoices } from '@/features/rooms/livekit/hooks/usePersistentUserChoices' export type AccountTabProps = Pick & Pick diff --git a/src/frontend/src/locales/en/rooms.json b/src/frontend/src/locales/en/rooms.json index 49b1bdff..b58ac8a3 100644 --- a/src/frontend/src/locales/en/rooms.json +++ b/src/frontend/src/locales/en/rooms.json @@ -18,6 +18,10 @@ "enable": "Enable microphone", "label": "Microphone" }, + "effects": { + "description": "Apply effects", + "title": "Effects" + }, "heading": "Join the meeting", "joinLabel": "Join", "joinMeeting": "Join meeting", diff --git a/src/frontend/src/locales/fr/rooms.json b/src/frontend/src/locales/fr/rooms.json index 1924b7eb..d99cae54 100644 --- a/src/frontend/src/locales/fr/rooms.json +++ b/src/frontend/src/locales/fr/rooms.json @@ -19,6 +19,10 @@ "label": "Microphone" }, "heading": "Rejoindre la réunion", + "effects": { + "description": "Appliquer des effets", + "title": "Effets" + }, "joinLabel": "Rejoindre", "joinMeeting": "Rejoindre la réjoindre", "toggleOff": "Cliquez pour désactiver",