From dfa31bcc7e9c173a678aed77647660af2cb77a4b Mon Sep 17 00:00:00 2001 From: Manuel Martin Date: Wed, 29 Nov 2023 10:17:01 +0100 Subject: [PATCH] Post Media Frame Updates --- src/bit-systems/media-loading.ts | 80 +++++++++++++++----------------- src/inflators/media-loader.ts | 8 ++-- src/inflators/spawner.ts | 12 ++--- src/systems/bit-media-frames.js | 80 ++++++-------------------------- src/utils/three-utils.js | 38 ++++++++++++++- 5 files changed, 100 insertions(+), 118 deletions(-) diff --git a/src/bit-systems/media-loading.ts b/src/bit-systems/media-loading.ts index c865a624d0..44df8c28f5 100644 --- a/src/bit-systems/media-loading.ts +++ b/src/bit-systems/media-loading.ts @@ -12,7 +12,6 @@ import { import { Box3, Group, Matrix4, Quaternion, Vector3 } from "three"; import { HubsWorld } from "../app"; import { - Deletable, LoadedByMediaLoader, MediaContentBounds, MediaImageLoaderData, @@ -31,7 +30,7 @@ import { LoadingObject } from "../prefabs/loading-object"; import { animate } from "../utils/animate"; import { setNetworkedDataWithoutRoot } from "../utils/assign-network-ids"; import { crNextFrame } from "../utils/coroutine"; -import { ClearFunction, JobRunner, withRollback } from "../utils/coroutine-utils"; +import { ClearFunction, JobRunner } from "../utils/coroutine-utils"; import { easeOutQuadratic } from "../utils/easing"; import { addObject3DComponent, renderAsEntity } from "../utils/jsx-entity"; import { loadImage } from "../utils/load-image"; @@ -44,11 +43,9 @@ import { MediaType, mediaTypeName, resolveMediaInfo } from "../utils/media-utils import { EntityID } from "../utils/networking-types"; import { LinkType, inflateLink } from "../inflators/link"; import { inflateGrabbable } from "../inflators/grabbable"; -import { findAncestorWithComponents, findAncestorsWithComponent } from "../utils/bit-utils"; +import { findAncestorsWithComponent } from "../utils/bit-utils"; import { setMatrixWorld } from "../utils/three-utils"; -import { Type, inflateRigidBody } from "../inflators/rigid-body"; -import { COLLISION_LAYERS } from "../constants"; -import { getScaleCoefficient } from "../utils/auto-box-collider"; +import { computeObjectAABB, getScaleCoefficient } from "../utils/auto-box-collider"; export function* waitForMediaLoaded(world: HubsWorld, eid: EntityID) { while (hasComponent(world, MediaLoader, eid)) { @@ -60,7 +57,8 @@ export const MEDIA_LOADER_FLAGS = { RECENTER: 1 << 0, RESIZE: 1 << 1, ANIMATE_LOAD: 1 << 2, - IS_OBJECT_MENU_TARGET: 1 << 3 + IS_OBJECT_MENU_TARGET: 1 << 3, + MOVE_PARENT_NOT_OBJECT: 1 << 4 }; const origMat = new Matrix4(); @@ -74,13 +72,16 @@ const rootScale = new Vector3(); function resizeAndRecenter(world: HubsWorld, mediaLoaderEid: EntityID, box: Box3) { const resize = MediaLoader.flags[mediaLoaderEid] & MEDIA_LOADER_FLAGS.RESIZE; const recenter = MediaLoader.flags[mediaLoaderEid] & MEDIA_LOADER_FLAGS.RECENTER; + const moveParentNotObject = MediaLoader.flags[mediaLoaderEid] & MEDIA_LOADER_FLAGS.MOVE_PARENT_NOT_OBJECT; - if (resize || recenter) { - const mediaLoaderObj = world.eid2obj.get(mediaLoaderEid)!; - const mediaObj = mediaLoaderObj.children.at(0)!; + const mediaLoaderObj = world.eid2obj.get(mediaLoaderEid)!; + const mediaObj = mediaLoaderObj.children.at(0)!; - mediaLoaderObj.updateMatrixWorld(); - origMat.copy(mediaLoaderObj.matrixWorld); + mediaLoaderObj.updateMatrices(); + origMat.copy(mediaLoaderObj.matrixWorld); + + let scalar = 1; + if (recenter) { tmpMat.copy(origMat); invMat.copy(tmpMat).invert(); tmpMat.multiply(invMat); @@ -88,32 +89,43 @@ function resizeAndRecenter(world: HubsWorld, mediaLoaderEid: EntityID, box: Box3 mediaLoaderObj.updateMatrixWorld(); tmpMat.decompose(rootPosition, rootRotation, rootScale); - box.setFromObject(mediaObj); - - // The AABB can be empty here for interactables that fetch media (ie. gltf with an empty that has a video component). - // If we don't return the interactable would be wrongly positioned at the (0,0,0). + computeObjectAABB(mediaObj, box, true); if (box.isEmpty()) return; - let scalar = 1; if (resize) { - const size = new Vector3(); - box.getSize(size); scalar = getScaleCoefficient(0.5, box); } - if (recenter) { - const center = new Vector3(); - center.addVectors(box.min, box.max).multiplyScalar(0.5); - diff.subVectors(rootPosition, center); - diff.multiplyScalar(scalar); - transformPosition.addVectors(rootPosition, diff); - } + const center = new Vector3(); + center.addVectors(box.min, box.max).multiplyScalar(0.5); + diff.subVectors(rootPosition, center); + diff.multiplyScalar(scalar); + transformPosition.addVectors(rootPosition, diff); rootScale.set(scalar, scalar, scalar); tmpMat.compose(transformPosition, rootRotation, rootScale); setMatrixWorld(mediaObj, tmpMat); setMatrixWorld(mediaLoaderObj, origMat); + } else if (moveParentNotObject) { + origMat.decompose(rootPosition, rootRotation, rootScale); + + computeObjectAABB(mediaObj, box, true); + if (box.isEmpty()) return; + + const center = new Vector3(); + center.addVectors(box.min, box.max).multiplyScalar(0.5); + diff.subVectors(rootPosition, center); + transformPosition.subVectors(rootPosition, diff); + + tmpMat.compose(transformPosition, rootRotation, rootScale); + setMatrixWorld(mediaLoaderObj, tmpMat); + setMatrixWorld(mediaObj, origMat); } + + addComponent(world, MediaContentBounds, mediaLoaderEid); + box.getSize(tmpVector); + tmpVector.multiplyScalar(scalar); + MediaContentBounds.bounds[mediaLoaderEid].set(tmpVector.toArray()); } export function* animateScale(world: HubsWorld, mediaLoaderEid: EntityID) { @@ -148,12 +160,6 @@ function* finish(world: HubsWorld, mediaLoaderEid: EntityID) { type: Shape.HULL, minHalfExtent: 0.04 }); - - addComponent(world, MediaContentBounds, mediaLoaderEid); - const mediaLoaderObj = world.eid2obj.get(mediaLoaderEid)!; - box.setFromObject(mediaLoaderObj); - box.getSize(tmpVector); - MediaContentBounds.bounds[mediaLoaderEid].set(tmpVector.toArray()); } } @@ -317,16 +323,6 @@ export function mediaLoadingSystem(world: HubsWorld) { loadingCubes.set(eid, loadingObjEid); } - // If it's not deletable media, we need to add a rigid body to correctly position the physics shape - const deletableEid = findAncestorWithComponents(world, [MediaLoader, Deletable], eid); - if (!deletableEid) { - inflateRigidBody(world, eid, { - type: Type.STATIC, - collisionGroup: COLLISION_LAYERS.INTERACTABLES, - collisionMask: COLLISION_LAYERS.INTERACTABLES - }); - } - jobs.add(eid, clearRollbacks => loadAndAnimateMedia(world, eid, clearRollbacks)); }); diff --git a/src/inflators/media-loader.ts b/src/inflators/media-loader.ts index 120dd41edf..00d041c7b8 100644 --- a/src/inflators/media-loader.ts +++ b/src/inflators/media-loader.ts @@ -5,17 +5,18 @@ import { MEDIA_LOADER_FLAGS } from "../bit-systems/media-loading"; export type MediaLoaderParams = { src: string; - resize: boolean; - recenter: boolean; + resize?: boolean; + recenter?: boolean; animateLoad: boolean; fileId?: string; isObjectMenuTarget: boolean; + moveParentNotObject?: boolean; }; export function inflateMediaLoader( world: HubsWorld, eid: number, - { src, recenter, resize, animateLoad, fileId, isObjectMenuTarget }: MediaLoaderParams + { src, recenter, resize, animateLoad, fileId, isObjectMenuTarget, moveParentNotObject }: MediaLoaderParams ) { addComponent(world, MediaLoader, eid); addComponent(world, MediaLoading, eid); @@ -24,6 +25,7 @@ export function inflateMediaLoader( if (resize) flags |= MEDIA_LOADER_FLAGS.RESIZE; if (animateLoad) flags |= MEDIA_LOADER_FLAGS.ANIMATE_LOAD; if (isObjectMenuTarget) flags |= MEDIA_LOADER_FLAGS.IS_OBJECT_MENU_TARGET; + if (moveParentNotObject) flags |= MEDIA_LOADER_FLAGS.MOVE_PARENT_NOT_OBJECT; MediaLoader.flags[eid] = flags; if (fileId) { MediaLoader.fileId[eid] = APP.getSid(fileId)!; diff --git a/src/inflators/spawner.ts b/src/inflators/spawner.ts index b6b929b441..78c6be0310 100644 --- a/src/inflators/spawner.ts +++ b/src/inflators/spawner.ts @@ -23,10 +23,11 @@ export interface SpawnerParams { export function inflateSpawner(world: HubsWorld, eid: number, props: SpawnerParams) { inflateMediaLoader(world, eid, { src: props.src, - recenter: true, + recenter: false, resize: false, - animateLoad: true, - isObjectMenuTarget: false + animateLoad: false, + isObjectMenuTarget: false, + moveParentNotObject: true }); addComponent(world, HandCollisionTarget, eid); @@ -42,8 +43,7 @@ export function inflateSpawner(world: HubsWorld, eid: number, props: SpawnerPara inflateRigidBody(world, eid, { mass: 0, type: Type.STATIC, - collisionGroup: COLLISION_LAYERS.DEFAULT_SPAWNER, - collisionMask: COLLISION_LAYERS.INTERACTABLES, - disableCollision: true + collisionGroup: COLLISION_LAYERS.INTERACTABLES, + collisionMask: COLLISION_LAYERS.DEFAULT_SPAWNER }); } diff --git a/src/systems/bit-media-frames.js b/src/systems/bit-media-frames.js index e949f4eba4..b2d2a55de4 100644 --- a/src/systems/bit-media-frames.js +++ b/src/systems/bit-media-frames.js @@ -29,25 +29,14 @@ import { Rigidbody } from "../bit-components"; import { MediaType } from "../utils/media-utils"; -import { cloneObject3D, createPlaneBufferGeometry, disposeNode, setMatrixWorld } from "../utils/three-utils"; +import { cloneObject3D, disposeNode, setMatrixWorld } from "../utils/three-utils"; import { takeOwnership } from "../utils/take-ownership"; import { takeSoftOwnership } from "../utils/take-soft-ownership"; import { findAncestorWithComponent, findChildWithComponent } from "../utils/bit-utils"; -import { TEXTURES_FLIP_Y } from "../loaders/HubsTextureLoader"; import { addObject3DComponent } from "../utils/jsx-entity"; import { updateMaterials } from "../utils/material-utils"; import { MEDIA_FRAME_FLAGS, AxisAlignType } from "../inflators/media-frame"; -import { - DoubleSide, - Group, - Matrix4, - Mesh, - MeshBasicMaterial, - NormalBlending, - Quaternion, - RGBAFormat, - Vector3 -} from "three"; +import { Matrix4, NormalBlending, Quaternion, RGBAFormat, Vector3 } from "three"; const EMPTY_COLOR = 0x6fc0fd; const HOVER_COLOR = 0x2f80ed; @@ -203,62 +192,23 @@ const setMatrixScale = (() => { }; })(); -const videoGeometry = createPlaneBufferGeometry(1, 1, 1, 1, TEXTURES_FLIP_Y); -const previewMaterial = new MeshBasicMaterial(); -previewMaterial.side = DoubleSide; -previewMaterial.transparent = true; -previewMaterial.opacity = 0.5; function createPreview(world, capturableEid) { // Source object to copy const capturableObj = world.eid2obj.get(capturableEid); - // Copied object to use as preview - let previewObj; - let videoObj; - let aspectRatio; - if (hasComponent(world, AEntity, capturableEid)) { - const video = capturableObj.el.components["media-video"]; - if (video) { - aspectRatio = - (video.videoTexture.image.videoHeight || video.videoTexture.image.height) / - (video.videoTexture.image.videoWidth || video.videoTexture.image.width); - videoObj = capturableObj.el.getObject3D("mesh"); - } - } else { - const mediaEid = findChildWithComponent(world, MediaLoaded, capturableEid); - if (hasComponent(world, MediaVideo, mediaEid)) { - aspectRatio = MediaVideo.ratio[mediaEid]; - videoObj = world.eid2obj.get(mediaEid); - } - } - - // Videos can't be cloned so we take a different path for them - if (videoObj) { - // Video mesh will be scaled to the aspect ratio and vertical orientation of the video - // this is then placed inside a Group to keep the downstream snap logic simpler - const videoMesh = new Mesh(videoGeometry, previewMaterial); - videoMesh.material.map = videoObj.material.map; - videoMesh.material.needsUpdate = true; - // Preview mesh UVs are set to accommodate textureLoader default, but video textures don't match this - videoMesh.scale.setY(TEXTURES_FLIP_Y !== videoMesh.material.map.flipY ? -aspectRatio : aspectRatio); - videoMesh.matrixNeedsUpdate = true; - previewObj = new Group(); - previewObj.add(videoMesh); - } else { - previewObj = cloneObject3D(capturableObj, false); - previewObj.traverse(node => { - updateMaterials(node, function (srcMat) { - const mat = srcMat.clone(); - mat.transparent = true; - mat.opacity = 0.5; - mat.format = RGBAFormat; - mat.blending = NormalBlending; - node.material = mat; - return mat; - }); + const previewObj = cloneObject3D(capturableObj, false); + previewObj.traverse(node => { + updateMaterials(node, function (srcMat) { + const mat = srcMat.clone(); + mat.transparent = true; + mat.opacity = 0.5; + mat.format = RGBAFormat; + mat.blending = NormalBlending; + node.material = mat; + return mat; }); - // Copy the target's transform to preserve scale for some of the media frame options - setMatrixWorld(previewObj, capturableObj.matrixWorld); - } + }); + // } + setMatrixWorld(previewObj, capturableObj.matrixWorld); world.scene.add(previewObj); return previewObj; } diff --git a/src/utils/three-utils.js b/src/utils/three-utils.js index f9ed830eec..dffc35bd50 100644 --- a/src/utils/three-utils.js +++ b/src/utils/three-utils.js @@ -1,4 +1,4 @@ -import { removeEntity } from "bitecs"; +import { hasComponent, removeEntity } from "bitecs"; import { forEachMaterial } from "./material-utils"; const tempVector3 = new THREE.Vector3(); @@ -365,7 +365,9 @@ export function createPlaneBufferGeometry(width, height, widthSegments, heightSe } import { Layers } from "../camera-layers"; -import { Box3, Vector3 } from "three"; +import { Box3, DoubleSide, Mesh, MeshBasicMaterial, Object3D, Vector3 } from "three"; +import { TEXTURES_FLIP_Y } from "../loaders/HubsTextureLoader"; +import { MediaVideo } from "../bit-components"; // This code is from three-vrm. We will likely be using that in the future and this inlined code can go away function excludeTriangles(triangles, bws, skinIndex, exclude) { @@ -566,3 +568,35 @@ export function setFromObject(box, object, onlyVisible = true, precise = false) box.makeEmpty(); expandByObject(box, object, onlyVisible, precise); } + +const videoGeometry = createPlaneBufferGeometry(1, 1, 1, 1, TEXTURES_FLIP_Y); +const previewMaterial = new MeshBasicMaterial(); +previewMaterial.side = DoubleSide; +previewMaterial.transparent = true; +previewMaterial.opacity = 0.5; +THREE.Object3D.prototype._clone = THREE.Object3D.prototype.clone; +THREE.Object3D.prototype.clone = (function () { + return function clone() { + if (this.type === "Audio") { + console.log("Audio clone not supported"); + return new Object3D(); + } else if (hasComponent(APP.world, MediaVideo, this.eid)) { + const videoMesh = new Mesh(videoGeometry, previewMaterial); + videoMesh.material.map = this.material.map; + videoMesh.material.needsUpdate = true; + // Preview mesh UVs are set to accommodate textureLoader default, but video textures don't match this + const aspectRatio = MediaVideo.ratio[this.eid]; + videoMesh.scale.setY(TEXTURES_FLIP_Y !== videoMesh.material.map.flipY ? -aspectRatio : aspectRatio); + videoMesh.matrixNeedsUpdate = true; + + for (let i = 0; i < this.children.length; i++) { + const child = this.children[i]; + videoMesh.add(child.clone()); + } + + return videoMesh; + } else { + return this._clone(); + } + }; +})();