✨(front) add blurring feature on join

This adds a button that opens a modal that allow user to enable
video effects on join screen.

src/frontend/src/features/home/routes/Home.tsx

@@ -9,7 +9,6 @@ import { useUser, UserAware } from '@/features/auth'
 import { JoinMeetingDialog } from '../components/JoinMeetingDialog'
 import { ProConnectButton } from '@/components/ProConnectButton'
 import { useCreateRoom } from '@/features/rooms'
-import { usePersistentUserChoices } from '@livekit/components-react'
 import { RiAddLine, RiLink } from '@remixicon/react'
 import { LaterMeetingDialog } from '@/features/home/components/LaterMeetingDialog'
 import { IntroSlider } from '@/features/home/components/IntroSlider'
@@ -18,6 +17,7 @@ import { ReactNode, useState } from 'react'
 
 import { css } from '@/styled-system/css'
 import { menuRecipe } from '@/primitives/menuRecipe.ts'
+import { usePersistentUserChoices } from '@/features/rooms/livekit/hooks/usePersistentUserChoices'
 
 const Columns = ({ children }: { children?: ReactNode }) => {
   return (

src/frontend/src/features/rooms/components/Conference.tsx

@@ -1,7 +1,7 @@
 import { useEffect, useMemo, useState } from 'react'
 import { useQuery } from '@tanstack/react-query'
 import { useTranslation } from 'react-i18next'
-import { LiveKitRoom, type LocalUserChoices } from '@livekit/components-react'
+import { LiveKitRoom } from '@livekit/components-react'
 import { Room, RoomOptions } from 'livekit-client'
 import { keys } from '@/api/queryKeys'
 import { queryClient } from '@/api/queryClient'
@@ -12,10 +12,11 @@ import { fetchRoom } from '../api/fetchRoom'
 import { ApiRoom } from '../api/ApiRoom'
 import { useCreateRoom } from '../api/createRoom'
 import { InviteDialog } from './InviteDialog'
-
 import { VideoConference } from '../livekit/prefabs/VideoConference'
 import posthog from 'posthog-js'
 import { css } from '@/styled-system/css'
+import { LocalUserChoices } from '../routes/Room'
+import { BackgroundBlurFactory } from '../livekit/components/blur'
 
 export const Conference = ({
   roomId,
@@ -102,6 +103,8 @@ export const Conference = ({
     peerConnectionTimeout: 60000, // Default: 15s. Extended for slow TURN/TLS negotiation
   }
 
+  console.log('ROOM', userConfig)
+
   return (
     <QueryAware status={isFetchError ? createStatus : fetchStatus}>
       <Screen header={false} footer={false}>
@@ -111,7 +114,15 @@ export const Conference = ({
           token={data?.livekit?.token}
           connect={true}
           audio={userConfig.audioEnabled}
-          video={userConfig.videoEnabled}
+          video={
+            userConfig.videoEnabled
+              ? {
+                  processor: BackgroundBlurFactory.deserializeProcessor(
+                    userConfig.processorSerialized
+                  ),
+                }
+              : false
+          }
           connectOptions={connectOptions}
           className={css({
             backgroundColor: 'primaryDark.50 !important',

src/frontend/src/features/rooms/components/Join.tsx

@@ -1,8 +1,7 @@
 import { useTranslation } from 'react-i18next'
 import {
-  usePersistentUserChoices,
+  ParticipantPlaceholder,
   usePreviewTracks,
-  type LocalUserChoices,
 } from '@livekit/components-react'
 import { css } from '@/styled-system/css'
 import { Screen } from '@/layout/Screen'
@@ -13,6 +12,13 @@ import { SelectToggleDevice } from '../livekit/components/controls/SelectToggleD
 import { Field } from '@/primitives/Field'
 import { Form } from '@/primitives'
 import { HStack, VStack } from '@/styled-system/jsx'
+import { Button, Dialog } from '@/primitives'
+import { LocalUserChoices } from '../routes/Room'
+import { Heading } from 'react-aria-components'
+import { RiImageCircleAiFill } from '@remixicon/react'
+import { EffectsConfiguration } from '../livekit/components/effects/EffectsConfiguration'
+import { usePersistentUserChoices } from '../livekit/hooks/usePersistentUserChoices'
+import { BackgroundBlurFactory } from '../livekit/components/blur'
 
 const onError = (e: Error) => console.error('ERROR', e)
 
@@ -28,6 +34,7 @@ export const Join = ({
     saveAudioInputDeviceId,
     saveVideoInputDeviceId,
     saveUsername,
+    saveProcessorSerialized,
   } = usePersistentUserChoices({})
 
   const [audioDeviceId, setAudioDeviceId] = useState<string>(
@@ -37,6 +44,11 @@ export const Join = ({
     initialUserChoices.videoDeviceId
   )
   const [username, setUsername] = useState<string>(initialUserChoices.username)
+  const [processor, setProcessor] = useState(
+    BackgroundBlurFactory.deserializeProcessor(
+      initialUserChoices.processorSerialized
+    )
+  )
 
   useEffect(() => {
     saveAudioInputDeviceId(audioDeviceId)
@@ -49,6 +61,9 @@ export const Join = ({
   useEffect(() => {
     saveUsername(username)
   }, [username, saveUsername])
+  useEffect(() => {
+    saveProcessorSerialized(processor?.serialize())
+  }, [processor, saveProcessorSerialized])
 
   const [audioEnabled, setAudioEnabled] = useState(true)
   const [videoEnabled, setVideoEnabled] = useState(true)
@@ -110,8 +125,48 @@ export const Join = ({
     })
   }
 
+  const [isEffectsOpen, setEffectsOpen] = useState(false)
+
+  const openEffects = () => {
+    setEffectsOpen(true)
+  }
+
+  // This hook is used to setup the persisted user choice processor on initialization.
+  // So it's on purpose that processor is not included in the deps.
+  // We just want to wait for the videoTrack to be loaded to apply the default processor.
+  useEffect(() => {
+    if (processor && videoTrack && !videoTrack.getProcessor()) {
+      videoTrack.setProcessor(processor)
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [videoTrack])
+
   return (
     <Screen footer={false}>
+      <Dialog
+        isOpen={isEffectsOpen}
+        onOpenChange={setEffectsOpen}
+        role="dialog"
+        type="flex"
+        size="large"
+      >
+        <Heading
+          slot="title"
+          level={1}
+          className={css({
+            textStyle: 'h1',
+            marginBottom: '1.5rem',
+          })}
+        >
+          {t('effects.title')}
+        </Heading>
+        <EffectsConfiguration
+          videoTrack={videoTrack}
+          onSubmit={(processor) => {
+            setProcessor(processor)
+          }}
+        />
+      </Dialog>
       <div
         className={css({
           display: 'flex',
@@ -204,8 +259,35 @@ export const Join = ({
                   </p>
                 </div>
               )}
+              <div
+                className={css({
+                  position: 'absolute',
+                  left: 0,
+                  right: 0,
+                  bottom: 0,
+                  height: '20%',
+                  backgroundImage:
+                    'linear-gradient(0deg, rgba(0,0,0,0.8) 0%, rgba(255,255,255,0) 100%)',
+                })}
+              ></div>
+              <div
+                className={css({
+                  position: 'absolute',
+                  right: 0,
+                  bottom: '0',
+                  padding: '1rem',
+                })}
+              >
+                <Button
+                  variant="whiteCircle"
+                  onPress={openEffects}
+                  tooltip={t('effects.description')}
+                  aria-label={t('effects.description')}
+                >
+                  <RiImageCircleAiFill size={24} />
+                </Button>
+              </div>
             </div>
-
             <HStack justify="center" padding={1.5}>
               <SelectToggleDevice
                 source={Track.Source.Microphone}

src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.ts

@@ -11,7 +11,11 @@ import {
   TIMEOUT_TICK,
   timerWorkerScript,
 } from './TimerWorker'
-import { BackgroundBlurProcessorInterface, BackgroundOptions } from '.'
+import {
+  BackgroundBlurProcessorInterface,
+  BackgroundOptions,
+  ProcessorType,
+} from '.'
 
 const PROCESSING_WIDTH = 256
 const PROCESSING_HEIGHT = 144
@@ -282,4 +286,11 @@ export class BackgroundBlurCustomProcessor
   clone() {
     return new BackgroundBlurCustomProcessor(this.options)
   }
+
+  serialize() {
+    return {
+      type: ProcessorType.BLUR,
+      options: this.options,
+    }
+  }
 }

src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts

@@ -4,7 +4,11 @@ import {
   ProcessorWrapper,
 } from '@livekit/track-processors'
 import { ProcessorOptions, Track } from 'livekit-client'
-import { BackgroundBlurProcessorInterface, BackgroundOptions } from '.'
+import {
+  BackgroundBlurProcessorInterface,
+  BackgroundOptions,
+  ProcessorType,
+} from '.'
 
 /**
  * This is simply a wrapper around track-processor-js Processor
@@ -54,4 +58,11 @@ export class BackgroundBlurTrackProcessorJsWrapper
       blurRadius: this.options!.blurRadius,
     })
   }
+
+  serialize() {
+    return {
+      type: ProcessorType.BLUR,
+      options: this.options,
+    }
+  }
 }

src/frontend/src/features/rooms/livekit/components/blur/index.ts

@@ -7,11 +7,21 @@ export type BackgroundOptions = {
   blurRadius?: number
 }
 
+export interface ProcessorSerialized {
+  type: ProcessorType
+  options: BackgroundOptions
+}
+
 export interface BackgroundBlurProcessorInterface
   extends TrackProcessor<Track.Kind> {
   update(opts: BackgroundOptions): void
   options: BackgroundOptions
   clone(): BackgroundBlurProcessorInterface
+  serialize(): ProcessorSerialized
+}
+
+export enum ProcessorType {
+  BLUR = 'blur',
 }
 
 export class BackgroundBlurFactory {
@@ -21,13 +31,22 @@ export class BackgroundBlurFactory {
     )
   }
 
-  static getProcessor(opts: BackgroundOptions) {
+  static getProcessor(
+    opts: BackgroundOptions
+  ): BackgroundBlurProcessorInterface | undefined {
     if (ProcessorWrapper.isSupported) {
       return new BackgroundBlurTrackProcessorJsWrapper(opts)
     }
     if (BackgroundBlurCustomProcessor.isSupported) {
       return new BackgroundBlurCustomProcessor(opts)
     }
-    return null
+    return undefined
+  }
+
+  static deserializeProcessor(data?: ProcessorSerialized) {
+    if (data?.type === ProcessorType.BLUR) {
+      return BackgroundBlurFactory.getProcessor(data?.options)
+    }
+    return undefined
   }
 }

src/frontend/src/features/rooms/livekit/components/effects/Effects.tsx

@@ -2,17 +2,26 @@ import { useLocalParticipant } from '@livekit/components-react'
 import { LocalVideoTrack } from 'livekit-client'
 import { css } from '@/styled-system/css'
 import { EffectsConfiguration } from './EffectsConfiguration'
+import { usePersistentUserChoices } from '../../hooks/usePersistentUserChoices'
 
 export const Effects = () => {
   const { cameraTrack } = useLocalParticipant()
   const localCameraTrack = cameraTrack?.track as LocalVideoTrack
+  const { saveProcessorSerialized } = usePersistentUserChoices()
+
   return (
     <div
       className={css({
         padding: '0 1.5rem',
       })}
     >
-      <EffectsConfiguration videoTrack={localCameraTrack} layout="vertical" />
+      <EffectsConfiguration
+        videoTrack={localCameraTrack}
+        layout="vertical"
+        onSubmit={(processor) =>
+          saveProcessorSerialized(processor?.serialize())
+        }
+      />
     </div>
   )
 }

src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx

@@ -1,4 +1,4 @@
-import { LocalVideoTrack, Track, TrackProcessor } from 'livekit-client'
+import { LocalVideoTrack } from 'livekit-client'
 import { useEffect, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import {
@@ -33,7 +33,7 @@ export const EffectsConfiguration = ({
   layout = 'horizontal',
 }: {
   videoTrack: LocalVideoTrack
-  onSubmit?: (processor?: TrackProcessor<Track.Kind.Video>) => void
+  onSubmit?: (processor?: BackgroundBlurProcessorInterface) => void
   layout?: 'vertical' | 'horizontal'
 }) => {
   const videoRef = useRef<HTMLVideoElement>(null)
@@ -68,6 +68,8 @@ export const EffectsConfiguration = ({
         onSubmit?.(newProcessor)
       } else {
         processor?.update({ blurRadius })
+        // We want to trigger onSubmit when options changes so the parent component is aware of it.
+        onSubmit?.(processor)
       }
     } catch (error) {
       console.error('Error applying blur:', error)