Merge pull request #6443 from mozilla/bitecs-mirror-button

BitECS mirror button

src/aframe-to-bit-components.js

@@ -8,7 +8,8 @@ import {
   NotRemoteHoverTarget,
   RemoveNetworkedEntityButton,
   DestroyAtExtremeDistance,
-  Billboard
+  Billboard,
+  AvatarPOVNode
 } from "./bit-components";
 
 [
@@ -20,7 +21,8 @@ import {
   ["is-not-remote-hover-target", NotRemoteHoverTarget],
   ["remove-networked-object-button", RemoveNetworkedEntityButton],
   ["destroy-at-extreme-distances", DestroyAtExtremeDistance],
-  ["billboard", Billboard]
+  ["billboard", Billboard],
+  ["avatar-pov-node", AvatarPOVNode]
 ].forEach(([aframeComponentName, bitecsComponent]) => {
   AFRAME.registerComponent(aframeComponentName, {
     init: function () {

src/app.ts

@@ -88,6 +88,7 @@ export class App {
   supplementaryAttenuation = new Map<ElOrEid, number>();
   clippingState = new Set<ElOrEid>();
   mutedState = new Set<ElOrEid>();
+  linkedMutedState = new Set<ElOrEid>();
   isAudioPaused = new Set<ElOrEid>();
   audioDebugPanelOverrides = new Map<SourceType, Partial<AudioSettings>>();
   sceneAudioDefaults = new Map<SourceType, Partial<AudioSettings>>();

src/bit-components.js

@@ -226,12 +226,17 @@ export const MediaPDF = defineComponent({
   pageNumber: Types.ui8
 });
 MediaPDF.map = new Map();
+export const MediaPDFUpdated = defineComponent({
+  pageNumber: Types.ui8
+});
 
 export const MediaVideo = defineComponent({
   ratio: Types.f32,
   flags: Types.ui8,
-  projection: Types.ui8
+  projection: Types.ui8,
+  lastUpdate: Types.ui32
 });
+export const MediaVideoUpdated = defineComponent();
 /**
  * @type {Map<EntityId, HTMLVideoElement}>}
  */
@@ -306,6 +311,27 @@ export const ObjectMenu = defineComponent({
   flags: Types.ui8
 });
 export const ObjectDropped = defineComponent();
+export const MediaMirrored = defineComponent({
+  linkedRef: Types.eid
+});
+export const MirroredMedia = defineComponent({
+  linkedRef: Types.eid
+});
+export const LinkedMedia = defineComponent({
+  linkedRef: Types.eid
+});
+export const FollowInFov = defineComponent({
+  offset: [Types.f32, 3],
+  angle: Types.f32,
+  speed: Types.f32,
+  started: Types.ui8
+});
+export const MirrorMenu = defineComponent({
+  closeRef: Types.eid,
+  mirrorTargetRef: Types.eid,
+  flags: Types.ui8
+});
+export const AvatarPOVNode = defineComponent();
 // TODO: Store this data elsewhere, since only one or two will ever exist.
 export const LinkHoverMenu = defineComponent({
   targetObjectRef: Types.eid,

src/bit-systems/audio-debug-system.ts

@@ -174,7 +174,11 @@ export function audioDebugSystem(world: HubsWorld) {
   if (isEnabled && uniforms) {
     let idx = 0;
     APP.audios.forEach((audio: AudioObject3D, audioEmitterId: ElOrEid) => {
-      if (APP.isAudioPaused.has(audioEmitterId) || APP.mutedState.has(audioEmitterId)) {
+      if (
+        APP.isAudioPaused.has(audioEmitterId) ||
+        APP.mutedState.has(audioEmitterId) ||
+        APP.linkedMutedState.has(audioEmitterId)
+      ) {
         return;
       }
       if (idx >= maxDebugEmitters) return;

src/bit-systems/follow-in-fov-system.ts

@@ -0,0 +1,58 @@
+import { defineQuery } from "bitecs";
+import { HubsWorld } from "../app";
+import { AvatarPOVNode, FollowInFov } from "../bit-components";
+import { Euler, MathUtils, Matrix4, Quaternion, Vector3 } from "three";
+import { anyEntityWith } from "../utils/bit-utils";
+
+const targetPos = new Vector3();
+const offset = new Vector3();
+const snappedRot = new Euler();
+const snappedQ = new Quaternion();
+const snappedXForm = new Matrix4();
+const snappedXFormWorld = new Matrix4();
+const tempVector = new Vector3();
+
+const followInFovQuery = defineQuery([FollowInFov]);
+export function followInFovSystem(world: HubsWorld) {
+  followInFovQuery(world).forEach(eid => {
+    const obj = world.eid2obj.get(eid)!;
+    if (!obj.visible) return;
+
+    const targetEid = anyEntityWith(world, AvatarPOVNode)!;
+    const targetObj = world.eid2obj.get(targetEid)!;
+    if (!targetObj) return;
+
+    offset.fromArray(FollowInFov.offset[eid]);
+
+    // Compute position + rotation by projecting offset along a downward ray in target space,
+    // and mask out Z rotation.
+    snappedRot.set(-FollowInFov.angle[eid] * MathUtils.DEG2RAD, targetObj.rotation.y, 0, targetObj.rotation.order);
+    snappedQ.setFromEuler(snappedRot);
+    snappedXForm.compose(targetObj.position, snappedQ, targetObj.scale);
+    snappedXFormWorld.multiplyMatrices(targetObj.parent!.matrixWorld, snappedXForm);
+
+    targetPos.copy(offset);
+    targetPos.applyMatrix4(snappedXFormWorld);
+
+    if (obj.parent) {
+      obj.parent.worldToLocal(targetPos);
+    }
+
+    if (!FollowInFov.started[eid]) {
+      obj.position.copy(targetPos);
+      FollowInFov.started[eid] = 1;
+    } else {
+      const speed = FollowInFov.speed[eid];
+      const t = speed * world.time.delta;
+
+      obj.position.set(
+        obj.position.x + (targetPos.x - obj.position.x) * t,
+        obj.position.y + (targetPos.y - obj.position.y) * t,
+        obj.position.z + (targetPos.z - obj.position.z) * t
+      );
+    }
+
+    snappedXFormWorld.decompose(tempVector, obj.quaternion, tempVector);
+    obj.matrixNeedsUpdate = true;
+  });
+}

src/bit-systems/linked-media-system.ts

@@ -0,0 +1,55 @@
+import { addComponent, defineQuery, enterQuery, entityExists, exitQuery, removeComponent } from "bitecs";
+import { HubsWorld } from "../app";
+import { MirroredMedia, MediaMirrored, LinkedMedia } from "../bit-components";
+import { cloneObject } from "./linked-menu-system";
+import { deleteTheDeletableAncestor } from "./delete-entity-system";
+import { anyEntityWith } from "../utils/bit-utils";
+import { EntityID } from "../utils/networking-types";
+
+export function linkMedia(world: HubsWorld, eid: EntityID, sourceMediaEid: EntityID) {
+  addComponent(world, LinkedMedia, eid);
+  LinkedMedia.linkedRef[eid] = sourceMediaEid;
+  addComponent(world, LinkedMedia, sourceMediaEid);
+  LinkedMedia.linkedRef[sourceMediaEid] = eid;
+}
+
+export function removeMirroredMedia(world: HubsWorld) {
+  const mirroredMediaEid = anyEntityWith(world, MirroredMedia);
+  if (mirroredMediaEid) {
+    deleteTheDeletableAncestor(world, mirroredMediaEid);
+  }
+}
+
+const mediaMirroredQuery = defineQuery([MediaMirrored]);
+const mediaMirroredEnterQuery = enterQuery(mediaMirroredQuery);
+const mediaMirroredExitQuery = exitQuery(mediaMirroredQuery);
+const mirroredMediaQuery = defineQuery([MirroredMedia]);
+const mirroredMediaEnterQuery = enterQuery(mirroredMediaQuery);
+const mirroredMediaExitQuery = exitQuery(mirroredMediaQuery);
+const linkedMediaQuery = defineQuery([LinkedMedia]);
+const linkedMediaExitQuery = exitQuery(linkedMediaQuery);
+export function linkedMediaSystem(world: HubsWorld) {
+  mirroredMediaExitQuery(world).forEach(eid => {
+    const mediaMirroredEid = MirroredMedia.linkedRef[eid];
+    if (entityExists(world, mediaMirroredEid)) {
+      removeComponent(world, MediaMirrored, mediaMirroredEid);
+    }
+  });
+  mirroredMediaEnterQuery(world).forEach(eid => {
+    const sourceMediaEid = MirroredMedia.linkedRef[eid];
+    MediaMirrored.linkedRef[sourceMediaEid] = eid;
+  });
+  mediaMirroredExitQuery(world).forEach(eid => removeMirroredMedia(world));
+  mediaMirroredEnterQuery(world).forEach(eid => {
+    const clonedEid = cloneObject(world, eid);
+    const clonedObj = world.eid2obj.get(clonedEid)!;
+    clonedObj.scale.set(0.75, 0.75, 0.75);
+    clonedObj.matrixNeedsUpdate = true;
+
+    addComponent(world, MirroredMedia, clonedEid);
+    MirroredMedia.linkedRef[clonedEid] = eid;
+  });
+  linkedMediaExitQuery(world).forEach(eid => {
+    removeComponent(world, LinkedMedia, LinkedMedia.linkedRef[eid]);
+  });
+}

src/bit-systems/linked-menu-system.ts

@@ -0,0 +1,63 @@
+import { defineQuery, enterQuery, exitQuery, hasComponent } from "bitecs";
+import type { HubsWorld } from "../app";
+import { Interacted, MirrorMenu, FollowInFov, MirroredMedia } from "../bit-components";
+import { anyEntityWith } from "../utils/bit-utils";
+import type { EntityID } from "../utils/networking-types";
+import { createMessageDatas } from "./networking";
+import { renderAsEntity } from "../utils/jsx-entity";
+import { LinkedMediaPrefab } from "../prefabs/linked-media";
+import { removeMirroredMedia } from "./linked-media-system";
+
+export function cloneObject(world: HubsWorld, sourceEid: EntityID): EntityID {
+  const { initialData } = createMessageDatas.get(sourceEid)!;
+  initialData.isObjectMenuTarget = false;
+  const clonedEid = renderAsEntity(world, LinkedMediaPrefab(initialData));
+  return clonedEid;
+}
+export const enum MirrorMenuFlags {
+  Visible = 1 << 0
+}
+
+function clicked(world: HubsWorld, eid: EntityID) {
+  return hasComponent(world, Interacted, eid);
+}
+
+function handleClicks(world: HubsWorld, menu: EntityID) {
+  if (clicked(world, MirrorMenu.closeRef[menu])) {
+    MirrorMenu.flags[menu] &= ~MirrorMenuFlags.Visible;
+    removeMirroredMedia(world);
+  }
+}
+
+function updateVisibility(world: HubsWorld, menu: EntityID) {
+  const obj = world.eid2obj.get(menu)!;
+  obj.visible = Boolean(MirrorMenu.flags[menu] & MirrorMenuFlags.Visible);
+}
+
+const mirroredMediaQuery = defineQuery([MirroredMedia]);
+const mirroredMediaEnterQuery = enterQuery(mirroredMediaQuery);
+const mirroredMediaExitQuery = exitQuery(mirroredMediaQuery);
+export function linkedMenuSystem(world: HubsWorld) {
+  const menu = anyEntityWith(world, MirrorMenu)!;
+
+  if (MirrorMenu.flags[menu] & MirrorMenuFlags.Visible) {
+    handleClicks(world, menu);
+  }
+
+  mirroredMediaExitQuery(world).forEach(eid => {
+    MirrorMenu.flags[menu] &= ~MirrorMenuFlags.Visible;
+  });
+
+  mirroredMediaEnterQuery(world).forEach(eid => {
+    FollowInFov.started[menu] = 0;
+
+    const mirrorTargetEid = MirrorMenu.mirrorTargetRef[menu];
+    const mirrorTargetObj = world.eid2obj.get(mirrorTargetEid);
+    const mirroredObj = world.eid2obj.get(eid)!;
+    mirrorTargetObj?.add(mirroredObj);
+
+    MirrorMenu.flags[menu] |= MirrorMenuFlags.Visible;
+  });
+
+  updateVisibility(world, menu);
+}

src/bit-systems/linked-pdf-system.ts

@@ -0,0 +1,36 @@
+import { addComponent, defineQuery, enterQuery, hasComponent } from "bitecs";
+import { HubsWorld } from "../app";
+import { MirroredMedia, LinkedMedia, MediaPDF, NetworkedPDF, MediaPDFUpdated } from "../bit-components";
+import { findAncestorWithComponent, findChildWithComponent } from "../utils/bit-utils";
+import { takeOwnership } from "../utils/take-ownership";
+import { linkMedia } from "./linked-media-system";
+
+const mediaPDFQuery = defineQuery([MediaPDF]);
+const mediaPDFEnterQuery = enterQuery(mediaPDFQuery);
+const updatedPDFQuery = defineQuery([MediaPDFUpdated]);
+const updatedPDFEnterQuery = enterQuery(updatedPDFQuery);
+export function linkedPDFSystem(world: HubsWorld) {
+  mediaPDFEnterQuery(world).forEach(eid => {
+    const mirroredMediaEid = findAncestorWithComponent(world, MirroredMedia, eid);
+    if (mirroredMediaEid) {
+      const mediaMirroredEid = MirroredMedia.linkedRef[mirroredMediaEid];
+      const sourceMediaEid = findChildWithComponent(world, MediaPDF, mediaMirroredEid)!;
+      if (sourceMediaEid) {
+        linkMedia(world, eid, sourceMediaEid);
+        addComponent(world, MediaPDFUpdated, eid);
+        MediaPDFUpdated.pageNumber[eid] = MediaPDF.pageNumber[sourceMediaEid];
+      }
+    }
+  });
+  updatedPDFEnterQuery(world).forEach(eid => {
+    if (!hasComponent(world, LinkedMedia, eid)) return;
+    const linked = LinkedMedia.linkedRef[eid];
+    if (MediaPDFUpdated.pageNumber[eid] !== MediaPDF.pageNumber[linked]) {
+      if (!hasComponent(world, NetworkedPDF, eid)) {
+        takeOwnership(world, linked);
+      }
+      addComponent(world, MediaPDFUpdated, linked);
+      MediaPDFUpdated.pageNumber[linked] = MediaPDFUpdated.pageNumber[eid];
+    }
+  });
+}

src/bit-systems/linked-video-system.ts

@@ -0,0 +1,79 @@
+import { addComponent, defineQuery, enterQuery, exitQuery, hasComponent } from "bitecs";
+import { HubsWorld } from "../app";
+import {
+  MirroredMedia,
+  LinkedMedia,
+  MediaVideo,
+  NetworkedVideo,
+  MediaVideoData,
+  MediaVideoUpdated,
+  AudioEmitter
+} from "../bit-components";
+import { findAncestorWithComponent, findChildWithComponent } from "../utils/bit-utils";
+import { takeOwnership } from "../utils/take-ownership";
+import { OUT_OF_SYNC_SEC } from "./video-system";
+import { linkMedia } from "./linked-media-system";
+import { updateAudioSettings } from "../update-audio-settings";
+
+const mediaVideoQuery = defineQuery([MediaVideo]);
+const mediaVideoEnterQuery = enterQuery(mediaVideoQuery);
+const updatedVideoQuery = defineQuery([MediaVideoUpdated]);
+const updatedVideoEnterQuery = enterQuery(updatedVideoQuery);
+const linkedMediaQuery = defineQuery([LinkedMedia]);
+const linkedMediaExitQuery = exitQuery(linkedMediaQuery);
+export function linkedVideoSystem(world: HubsWorld) {
+  mediaVideoEnterQuery(world).forEach(eid => {
+    const mirroredMediaEid = findAncestorWithComponent(world, MirroredMedia, eid);
+    if (mirroredMediaEid) {
+      const mediaMirroredEid = MirroredMedia.linkedRef[mirroredMediaEid];
+      const sourceMediaEid = findChildWithComponent(world, MediaVideo, mediaMirroredEid)!;
+      if (sourceMediaEid) {
+        linkMedia(world, eid, sourceMediaEid);
+        addComponent(world, MediaVideoUpdated, sourceMediaEid);
+        const sourceAudioEid = findChildWithComponent(world, AudioEmitter, sourceMediaEid);
+        if (sourceAudioEid) {
+          APP.linkedMutedState.add(sourceAudioEid);
+          const audio = APP.audios.get(sourceAudioEid);
+          if (audio) {
+            updateAudioSettings(sourceAudioEid, audio);
+          }
+        }
+      }
+    }
+  });
+  updatedVideoEnterQuery(world).forEach(eid => {
+    if (!hasComponent(world, LinkedMedia, eid)) return;
+    const linked = LinkedMedia.linkedRef[eid];
+    const video = MediaVideoData.get(eid)!;
+    const linkedVideo = MediaVideoData.get(linked)!;
+
+    if (video.paused !== linkedVideo.paused) {
+      if (!hasComponent(world, NetworkedVideo, eid)) {
+        takeOwnership(world, linked);
+      }
+      if (video.paused) {
+        linkedVideo.pause();
+      } else {
+        linkedVideo.play().catch(() => console.error("Error playing video."));
+      }
+    }
+    if (Math.abs(video.currentTime - linkedVideo.currentTime)) {
+      if (!hasComponent(world, NetworkedVideo, eid)) {
+        takeOwnership(world, linked);
+      }
+      linkedVideo.currentTime = video.currentTime;
+    }
+  });
+  linkedMediaExitQuery(world).forEach(eid => {
+    if (hasComponent(world, MediaVideo, eid)) {
+      const audioEid = findChildWithComponent(world, AudioEmitter, eid);
+      if (audioEid) {
+        APP.linkedMutedState.delete(audioEid);
+        const audio = APP.audios.get(audioEid);
+        if (audio) {
+          updateAudioSettings(audioEid, audio);
+        }
+      }
+    }
+  });
+}