Merge pull request #6389 from mozilla/bitecs-post-media-frame-alignme…

…nt-fixes Post Media Frame Updates
Hubs-Foundation · Nov 29, 2023 · 93116bd · 93116bd
2 parents fbf5d20 + dfa31bc
commit 93116bd
Show file tree

Hide file tree

Showing 5 changed files with 100 additions and 118 deletions.
diff --git a/src/bit-systems/media-loading.ts b/src/bit-systems/media-loading.ts
@@ -12,7 +12,6 @@ import {
 import { Box3, Group, Matrix4, Quaternion, Vector3 } from "three";
 import { HubsWorld } from "../app";
 import {
-  Deletable,
   LoadedByMediaLoader,
   MediaContentBounds,
   MediaImageLoaderData,
@@ -31,7 +30,7 @@ import { LoadingObject } from "../prefabs/loading-object";
 import { animate } from "../utils/animate";
 import { setNetworkedDataWithoutRoot } from "../utils/assign-network-ids";
 import { crNextFrame } from "../utils/coroutine";
-import { ClearFunction, JobRunner, withRollback } from "../utils/coroutine-utils";
+import { ClearFunction, JobRunner } from "../utils/coroutine-utils";
 import { easeOutQuadratic } from "../utils/easing";
 import { addObject3DComponent, renderAsEntity } from "../utils/jsx-entity";
 import { loadImage } from "../utils/load-image";
@@ -44,11 +43,9 @@ import { MediaType, mediaTypeName, resolveMediaInfo } from "../utils/media-utils
 import { EntityID } from "../utils/networking-types";
 import { LinkType, inflateLink } from "../inflators/link";
 import { inflateGrabbable } from "../inflators/grabbable";
-import { findAncestorWithComponents, findAncestorsWithComponent } from "../utils/bit-utils";
+import { findAncestorsWithComponent } from "../utils/bit-utils";
 import { setMatrixWorld } from "../utils/three-utils";
-import { Type, inflateRigidBody } from "../inflators/rigid-body";
-import { COLLISION_LAYERS } from "../constants";
-import { getScaleCoefficient } from "../utils/auto-box-collider";
+import { computeObjectAABB, getScaleCoefficient } from "../utils/auto-box-collider";
 
 export function* waitForMediaLoaded(world: HubsWorld, eid: EntityID) {
   while (hasComponent(world, MediaLoader, eid)) {
@@ -60,7 +57,8 @@ export const MEDIA_LOADER_FLAGS = {
   RECENTER: 1 << 0,
   RESIZE: 1 << 1,
   ANIMATE_LOAD: 1 << 2,
-  IS_OBJECT_MENU_TARGET: 1 << 3
+  IS_OBJECT_MENU_TARGET: 1 << 3,
+  MOVE_PARENT_NOT_OBJECT: 1 << 4
 };
 
 const origMat = new Matrix4();
@@ -74,46 +72,60 @@ const rootScale = new Vector3();
 function resizeAndRecenter(world: HubsWorld, mediaLoaderEid: EntityID, box: Box3) {
   const resize = MediaLoader.flags[mediaLoaderEid] & MEDIA_LOADER_FLAGS.RESIZE;
   const recenter = MediaLoader.flags[mediaLoaderEid] & MEDIA_LOADER_FLAGS.RECENTER;
+  const moveParentNotObject = MediaLoader.flags[mediaLoaderEid] & MEDIA_LOADER_FLAGS.MOVE_PARENT_NOT_OBJECT;
 
-  if (resize || recenter) {
-    const mediaLoaderObj = world.eid2obj.get(mediaLoaderEid)!;
-    const mediaObj = mediaLoaderObj.children.at(0)!;
+  const mediaLoaderObj = world.eid2obj.get(mediaLoaderEid)!;
+  const mediaObj = mediaLoaderObj.children.at(0)!;
 
-    mediaLoaderObj.updateMatrixWorld();
-    origMat.copy(mediaLoaderObj.matrixWorld);
+  mediaLoaderObj.updateMatrices();
+  origMat.copy(mediaLoaderObj.matrixWorld);
+
+  let scalar = 1;
+  if (recenter) {
     tmpMat.copy(origMat);
     invMat.copy(tmpMat).invert();
     tmpMat.multiply(invMat);
     setMatrixWorld(mediaLoaderObj, tmpMat);
     mediaLoaderObj.updateMatrixWorld();
     tmpMat.decompose(rootPosition, rootRotation, rootScale);
 
-    box.setFromObject(mediaObj);
-
-    // The AABB can be empty here for interactables that fetch media (ie. gltf with an empty that has a video component).
-    // If we don't return the interactable would be wrongly positioned at the (0,0,0).
+    computeObjectAABB(mediaObj, box, true);
     if (box.isEmpty()) return;
 
-    let scalar = 1;
     if (resize) {
-      const size = new Vector3();
-      box.getSize(size);
       scalar = getScaleCoefficient(0.5, box);
     }
 
-    if (recenter) {
-      const center = new Vector3();
-      center.addVectors(box.min, box.max).multiplyScalar(0.5);
-      diff.subVectors(rootPosition, center);
-      diff.multiplyScalar(scalar);
-      transformPosition.addVectors(rootPosition, diff);
-    }
+    const center = new Vector3();
+    center.addVectors(box.min, box.max).multiplyScalar(0.5);
+    diff.subVectors(rootPosition, center);
+    diff.multiplyScalar(scalar);
+    transformPosition.addVectors(rootPosition, diff);
 
     rootScale.set(scalar, scalar, scalar);
     tmpMat.compose(transformPosition, rootRotation, rootScale);
     setMatrixWorld(mediaObj, tmpMat);
     setMatrixWorld(mediaLoaderObj, origMat);
+  } else if (moveParentNotObject) {
+    origMat.decompose(rootPosition, rootRotation, rootScale);
+
+    computeObjectAABB(mediaObj, box, true);
+    if (box.isEmpty()) return;
+
+    const center = new Vector3();
+    center.addVectors(box.min, box.max).multiplyScalar(0.5);
+    diff.subVectors(rootPosition, center);
+    transformPosition.subVectors(rootPosition, diff);
+
+    tmpMat.compose(transformPosition, rootRotation, rootScale);
+    setMatrixWorld(mediaLoaderObj, tmpMat);
+    setMatrixWorld(mediaObj, origMat);
   }
+
+  addComponent(world, MediaContentBounds, mediaLoaderEid);
+  box.getSize(tmpVector);
+  tmpVector.multiplyScalar(scalar);
+  MediaContentBounds.bounds[mediaLoaderEid].set(tmpVector.toArray());
 }
 
 export function* animateScale(world: HubsWorld, mediaLoaderEid: EntityID) {
@@ -148,12 +160,6 @@ function* finish(world: HubsWorld, mediaLoaderEid: EntityID) {
       type: Shape.HULL,
       minHalfExtent: 0.04
     });
-
-    addComponent(world, MediaContentBounds, mediaLoaderEid);
-    const mediaLoaderObj = world.eid2obj.get(mediaLoaderEid)!;
-    box.setFromObject(mediaLoaderObj);
-    box.getSize(tmpVector);
-    MediaContentBounds.bounds[mediaLoaderEid].set(tmpVector.toArray());
   }
 }
 
@@ -317,16 +323,6 @@ export function mediaLoadingSystem(world: HubsWorld) {
       loadingCubes.set(eid, loadingObjEid);
     }
 
-    // If it's not deletable media, we need to add a rigid body to correctly position the physics shape
-    const deletableEid = findAncestorWithComponents(world, [MediaLoader, Deletable], eid);
-    if (!deletableEid) {
-      inflateRigidBody(world, eid, {
-        type: Type.STATIC,
-        collisionGroup: COLLISION_LAYERS.INTERACTABLES,
-        collisionMask: COLLISION_LAYERS.INTERACTABLES
-      });
-    }
-
     jobs.add(eid, clearRollbacks => loadAndAnimateMedia(world, eid, clearRollbacks));
   });
 

diff --git a/src/inflators/media-loader.ts b/src/inflators/media-loader.ts
@@ -5,17 +5,18 @@ import { MEDIA_LOADER_FLAGS } from "../bit-systems/media-loading";
 
 export type MediaLoaderParams = {
   src: string;
-  resize: boolean;
-  recenter: boolean;
+  resize?: boolean;
+  recenter?: boolean;
   animateLoad: boolean;
   fileId?: string;
   isObjectMenuTarget: boolean;
+  moveParentNotObject?: boolean;
 };
 
 export function inflateMediaLoader(
   world: HubsWorld,
   eid: number,
-  { src, recenter, resize, animateLoad, fileId, isObjectMenuTarget }: MediaLoaderParams
+  { src, recenter, resize, animateLoad, fileId, isObjectMenuTarget, moveParentNotObject }: MediaLoaderParams
 ) {
   addComponent(world, MediaLoader, eid);
   addComponent(world, MediaLoading, eid);
@@ -24,6 +25,7 @@ export function inflateMediaLoader(
   if (resize) flags |= MEDIA_LOADER_FLAGS.RESIZE;
   if (animateLoad) flags |= MEDIA_LOADER_FLAGS.ANIMATE_LOAD;
   if (isObjectMenuTarget) flags |= MEDIA_LOADER_FLAGS.IS_OBJECT_MENU_TARGET;
+  if (moveParentNotObject) flags |= MEDIA_LOADER_FLAGS.MOVE_PARENT_NOT_OBJECT;
   MediaLoader.flags[eid] = flags;
   if (fileId) {
     MediaLoader.fileId[eid] = APP.getSid(fileId)!;

diff --git a/src/inflators/spawner.ts b/src/inflators/spawner.ts
@@ -23,10 +23,11 @@ export interface SpawnerParams {
 export function inflateSpawner(world: HubsWorld, eid: number, props: SpawnerParams) {
   inflateMediaLoader(world, eid, {
     src: props.src,
-    recenter: true,
+    recenter: false,
     resize: false,
-    animateLoad: true,
-    isObjectMenuTarget: false
+    animateLoad: false,
+    isObjectMenuTarget: false,
+    moveParentNotObject: true
   });
 
   addComponent(world, HandCollisionTarget, eid);
@@ -42,8 +43,7 @@ export function inflateSpawner(world: HubsWorld, eid: number, props: SpawnerPara
   inflateRigidBody(world, eid, {
     mass: 0,
     type: Type.STATIC,
-    collisionGroup: COLLISION_LAYERS.DEFAULT_SPAWNER,
-    collisionMask: COLLISION_LAYERS.INTERACTABLES,
-    disableCollision: true
+    collisionGroup: COLLISION_LAYERS.INTERACTABLES,
+    collisionMask: COLLISION_LAYERS.DEFAULT_SPAWNER
   });
 }
diff --git a/src/systems/bit-media-frames.js b/src/systems/bit-media-frames.js
@@ -29,25 +29,14 @@ import {
   Rigidbody
 } from "../bit-components";
 import { MediaType } from "../utils/media-utils";
-import { cloneObject3D, createPlaneBufferGeometry, disposeNode, setMatrixWorld } from "../utils/three-utils";
+import { cloneObject3D, disposeNode, setMatrixWorld } from "../utils/three-utils";
 import { takeOwnership } from "../utils/take-ownership";
 import { takeSoftOwnership } from "../utils/take-soft-ownership";
 import { findAncestorWithComponent, findChildWithComponent } from "../utils/bit-utils";
-import { TEXTURES_FLIP_Y } from "../loaders/HubsTextureLoader";
 import { addObject3DComponent } from "../utils/jsx-entity";
 import { updateMaterials } from "../utils/material-utils";
 import { MEDIA_FRAME_FLAGS, AxisAlignType } from "../inflators/media-frame";
-import {
-  DoubleSide,
-  Group,
-  Matrix4,
-  Mesh,
-  MeshBasicMaterial,
-  NormalBlending,
-  Quaternion,
-  RGBAFormat,
-  Vector3
-} from "three";
+import { Matrix4, NormalBlending, Quaternion, RGBAFormat, Vector3 } from "three";
 
 const EMPTY_COLOR = 0x6fc0fd;
 const HOVER_COLOR = 0x2f80ed;
@@ -203,62 +192,23 @@ const setMatrixScale = (() => {
   };
 })();
 
-const videoGeometry = createPlaneBufferGeometry(1, 1, 1, 1, TEXTURES_FLIP_Y);
-const previewMaterial = new MeshBasicMaterial();
-previewMaterial.side = DoubleSide;
-previewMaterial.transparent = true;
-previewMaterial.opacity = 0.5;
 function createPreview(world, capturableEid) {
   // Source object to copy
   const capturableObj = world.eid2obj.get(capturableEid);
-  // Copied object to use as preview
-  let previewObj;
-  let videoObj;
-  let aspectRatio;
-  if (hasComponent(world, AEntity, capturableEid)) {
-    const video = capturableObj.el.components["media-video"];
-    if (video) {
-      aspectRatio =
-        (video.videoTexture.image.videoHeight || video.videoTexture.image.height) /
-        (video.videoTexture.image.videoWidth || video.videoTexture.image.width);
-      videoObj = capturableObj.el.getObject3D("mesh");
-    }
-  } else {
-    const mediaEid = findChildWithComponent(world, MediaLoaded, capturableEid);
-    if (hasComponent(world, MediaVideo, mediaEid)) {
-      aspectRatio = MediaVideo.ratio[mediaEid];
-      videoObj = world.eid2obj.get(mediaEid);
-    }
-  }
-
-  // Videos can't be cloned so we take a different path for them
-  if (videoObj) {
-    // Video mesh will be scaled to the aspect ratio and vertical orientation of the video
-    // this is then placed inside a Group to keep the downstream snap logic simpler
-    const videoMesh = new Mesh(videoGeometry, previewMaterial);
-    videoMesh.material.map = videoObj.material.map;
-    videoMesh.material.needsUpdate = true;
-    // Preview mesh UVs are set to accommodate textureLoader default, but video textures don't match this
-    videoMesh.scale.setY(TEXTURES_FLIP_Y !== videoMesh.material.map.flipY ? -aspectRatio : aspectRatio);
-    videoMesh.matrixNeedsUpdate = true;
-    previewObj = new Group();
-    previewObj.add(videoMesh);
-  } else {
-    previewObj = cloneObject3D(capturableObj, false);
-    previewObj.traverse(node => {
-      updateMaterials(node, function (srcMat) {
-        const mat = srcMat.clone();
-        mat.transparent = true;
-        mat.opacity = 0.5;
-        mat.format = RGBAFormat;
-        mat.blending = NormalBlending;
-        node.material = mat;
-        return mat;
-      });
+  const previewObj = cloneObject3D(capturableObj, false);
+  previewObj.traverse(node => {
+    updateMaterials(node, function (srcMat) {
+      const mat = srcMat.clone();
+      mat.transparent = true;
+      mat.opacity = 0.5;
+      mat.format = RGBAFormat;
+      mat.blending = NormalBlending;
+      node.material = mat;
+      return mat;
     });
-    // Copy the target's transform to preserve scale for some of the media frame options
-    setMatrixWorld(previewObj, capturableObj.matrixWorld);
-  }
+  });
+  // }
+  setMatrixWorld(previewObj, capturableObj.matrixWorld);
   world.scene.add(previewObj);
   return previewObj;
 }

diff --git a/src/utils/three-utils.js b/src/utils/three-utils.js
@@ -1,4 +1,4 @@
-import { removeEntity } from "bitecs";
+import { hasComponent, removeEntity } from "bitecs";
 import { forEachMaterial } from "./material-utils";
 
 const tempVector3 = new THREE.Vector3();
@@ -365,7 +365,9 @@ export function createPlaneBufferGeometry(width, height, widthSegments, heightSe
 }
 
 import { Layers } from "../camera-layers";
-import { Box3, Vector3 } from "three";
+import { Box3, DoubleSide, Mesh, MeshBasicMaterial, Object3D, Vector3 } from "three";
+import { TEXTURES_FLIP_Y } from "../loaders/HubsTextureLoader";
+import { MediaVideo } from "../bit-components";
 
 // This code is from three-vrm. We will likely be using that in the future and this inlined code can go away
 function excludeTriangles(triangles, bws, skinIndex, exclude) {
@@ -566,3 +568,35 @@ export function setFromObject(box, object, onlyVisible = true, precise = false)
   box.makeEmpty();
   expandByObject(box, object, onlyVisible, precise);
 }
+
+const videoGeometry = createPlaneBufferGeometry(1, 1, 1, 1, TEXTURES_FLIP_Y);
+const previewMaterial = new MeshBasicMaterial();
+previewMaterial.side = DoubleSide;
+previewMaterial.transparent = true;
+previewMaterial.opacity = 0.5;
+THREE.Object3D.prototype._clone = THREE.Object3D.prototype.clone;
+THREE.Object3D.prototype.clone = (function () {
+  return function clone() {
+    if (this.type === "Audio") {
+      console.log("Audio clone not supported");
+      return new Object3D();
+    } else if (hasComponent(APP.world, MediaVideo, this.eid)) {
+      const videoMesh = new Mesh(videoGeometry, previewMaterial);
+      videoMesh.material.map = this.material.map;
+      videoMesh.material.needsUpdate = true;
+      // Preview mesh UVs are set to accommodate textureLoader default, but video textures don't match this
+      const aspectRatio = MediaVideo.ratio[this.eid];
+      videoMesh.scale.setY(TEXTURES_FLIP_Y !== videoMesh.material.map.flipY ? -aspectRatio : aspectRatio);
+      videoMesh.matrixNeedsUpdate = true;
+
+      for (let i = 0; i < this.children.length; i++) {
+        const child = this.children[i];
+        videoMesh.add(child.clone());
+      }
+
+      return videoMesh;
+    } else {
+      return this._clone();
+    }
+  };
+})();