BitECS snap object menu support

src/bit-components.js

@@ -278,6 +278,7 @@ export const VideoMenu = defineComponent({
   headRef: Types.eid,
   playIndicatorRef: Types.eid,
   pauseIndicatorRef: Types.eid,
+  snapRef: Types.eid,
   clearTargetTimer: Types.f64
 });
 export const AudioEmitter = defineComponent({
@@ -352,12 +353,14 @@ export const PDFMenu = defineComponent({
   prevButtonRef: Types.eid,
   nextButtonRef: Types.eid,
   pageLabelRef: Types.eid,
+  snapRef: Types.eid,
   targetRef: Types.eid,
   clearTargetTimer: Types.f64
 });
 export const ObjectMenuTarget = defineComponent({
   flags: Types.ui8
 });
+export const MediaSnapped = defineComponent();
 export const NetworkDebug = defineComponent();
 export const NetworkDebugRef = defineComponent({
   ref: Types.eid

src/bit-systems/object-menu-transform-system.ts

@@ -17,6 +17,7 @@ const tmpMat42 = new Matrix4();
 const aabb = new Box3();
 const sphere = new Sphere();
 const yVector = new Vector3(0, 1, 0);
+const UNIT_V3 = new Vector3(1, 1, 1);
 
 // Calculate the AABB without accounting for the root object rotation
 function getAABB(obj: Object3D, box: Box3, onlyVisible: boolean = false) {
@@ -83,8 +84,9 @@ function transformMenu(world: HubsWorld, menu: EntityID) {
     // For now we are defaulting to the current AFrame behavior.
   } else {
     targetObj.updateMatrices(true, true);
-    tmpMat4.copy(targetObj.matrixWorld);
-    tmpMat4.decompose(tmpVec1, tmpQuat1, tmpVec2);
+    targetObj.matrixWorld.decompose(tmpVec1, tmpQuat1, tmpVec2);
+    tmpMat42.compose(tmpVec1, tmpQuat1, UNIT_V3);
+    tmpMat4.copy(tmpMat42);
 
     const isFacing = isFacingCamera(targetObj);
     if (!isFacing) {

src/bit-systems/pdf-menu-system.ts

@@ -10,6 +10,7 @@ import {
   MediaLoader,
   MediaPDF,
   MediaPDFUpdated,
+  MediaSnapped,
   NetworkedPDF,
   ObjectMenuTransform,
   PDFMenu
@@ -25,6 +26,7 @@ function setCursorRaycastable(world: HubsWorld, menu: EntityID, enable: boolean)
   change(world, CursorRaycastable, menu);
   change(world, CursorRaycastable, PDFMenu.prevButtonRef[menu]);
   change(world, CursorRaycastable, PDFMenu.nextButtonRef[menu]);
+  change(world, CursorRaycastable, PDFMenu.snapRef[menu]);
 }
 
 function clicked(world: HubsWorld, eid: EntityID) {
@@ -83,6 +85,9 @@ function handleClicks(world: HubsWorld, menu: EntityID) {
   } else if (clicked(world, PDFMenu.prevButtonRef[menu])) {
     const pdf = PDFMenu.targetRef[menu];
     setPage(world, pdf, MediaPDF.pageNumber[pdf] - 1);
+  } else if (clicked(world, PDFMenu.snapRef[menu])) {
+    const pdf = PDFMenu.targetRef[menu];
+    addComponent(world, MediaSnapped, pdf);
   }
 }
 
@@ -107,13 +112,6 @@ function flushToObject3Ds(world: HubsWorld, menu: EntityID, frozen: boolean) {
     ObjectMenuTransform.flags[menu] &= ~ObjectMenuTransformFlags.Enabled;
   }
 
-  [PDFMenu.prevButtonRef[menu], PDFMenu.nextButtonRef[menu]].forEach(buttonRef => {
-    const buttonObj = world.eid2obj.get(buttonRef)!;
-    // Parent visibility doesn't block raycasting, so we must set each button to be invisible
-    // TODO: Ensure that children of invisible entities aren't raycastable
-    buttonObj.visible = visible;
-  });
-
   if (target) {
     const numPages = PDFResourcesMap.get(target)!.pdf.numPages;
     (world.eid2obj.get(PDFMenu.pageLabelRef[menu]) as Text).text = `${MediaPDF.pageNumber[target]} / ${numPages}`;

src/bit-systems/snap-media-system.ts

@@ -0,0 +1,64 @@
+import { defineQuery, enterQuery, entityExists, exitQuery, hasComponent, removeComponent } from "bitecs";
+import { HubsWorld } from "../app";
+import { MediaPDF, MediaSnapped, MediaVideo, MediaVideoData } from "../bit-components";
+import { SOUND_CAMERA_TOOL_TOOK_SNAPSHOT } from "../systems/sound-effects-system";
+import { JobRunner } from "../utils/coroutine-utils";
+import { EntityID } from "../utils/networking-types";
+import { PDFResourcesMap } from "./pdf-system";
+import { spawnFromFileList } from "../load-media-on-paste-or-drop";
+
+const TYPE_IMG_PNG = { type: "image/png" };
+
+export function* snapMedia(world: HubsWorld, eid: EntityID) {
+  let canvas: HTMLCanvasElement | undefined;
+  if (hasComponent(world, MediaPDF, eid)) {
+    const res = PDFResourcesMap.get(eid);
+    canvas = res?.canvas;
+  } else if (hasComponent(world, MediaVideo, eid)) {
+    const video = MediaVideoData.get(eid)!;
+    if (video) {
+      canvas = document.createElement("canvas");
+      canvas.width = video.videoWidth;
+      canvas.height = video.videoHeight;
+      canvas.getContext("2d")!.drawImage(video, 0, 0, canvas.width, canvas.height);
+    }
+  }
+
+  if (canvas) {
+    const blob = new Promise((resolve, reject) => {
+      if (canvas) {
+        canvas.toBlob(resolve);
+      } else {
+        reject();
+      }
+    });
+    if (blob) {
+      const file = new File([yield blob], "snap.png", TYPE_IMG_PNG);
+      spawnFromFileList([file] as any);
+    } else {
+      console.error("Snapped image creation error");
+    }
+  }
+
+  if (entityExists(world, eid)) {
+    removeComponent(world, MediaSnapped, eid);
+  }
+}
+
+const jobs = new JobRunner();
+const snappedMediaQuery = defineQuery([MediaSnapped]);
+const snappedEnterQuery = enterQuery(snappedMediaQuery);
+const snappedExitQuery = exitQuery(snappedMediaQuery);
+export function snapMediaSystem(world: HubsWorld, sfxSystem: any) {
+  snappedExitQuery(world).forEach(eid => {
+    jobs.stop(eid);
+  });
+  snappedEnterQuery(world).forEach(eid => {
+    sfxSystem.playSoundOneShot(SOUND_CAMERA_TOOL_TOOK_SNAPSHOT);
+
+    if (!jobs.has(eid)) {
+      jobs.add(eid, () => snapMedia(world, eid));
+    }
+  });
+  jobs.tick();
+}

src/bit-systems/video-menu-system.ts

@@ -12,6 +12,7 @@ import {
   HoveredRemoteRight,
   Interacted,
   MediaLoader,
+  MediaSnapped,
   MediaVideo,
   MediaVideoData,
   MediaVideoUpdated,
@@ -38,6 +39,7 @@ function setCursorRaycastable(world: HubsWorld, menu: number, enable: boolean) {
   change(world, CursorRaycastable, VideoMenu.trackRef[menu]);
   change(world, CursorRaycastable, VideoMenu.playIndicatorRef[menu]);
   change(world, CursorRaycastable, VideoMenu.pauseIndicatorRef[menu]);
+  change(world, CursorRaycastable, VideoMenu.snapRef[menu]);
 }
 
 const intersectInThePlaneOf = (() => {
@@ -113,7 +115,6 @@ function flushToObject3Ds(world: HubsWorld, menu: EntityID, frozen: boolean) {
   } else {
     obj.removeFromParent();
     setCursorRaycastable(world, menu, false);
-
     ObjectMenuTransform.flags[menu] &= ~ObjectMenuTransformFlags.Enabled;
   }
 }
@@ -142,6 +143,9 @@ function handleClicks(world: HubsWorld, menu: EntityID) {
       addComponent(world, EntityStateDirty, videoEid);
     }
     addComponent(world, MediaVideoUpdated, videoEid);
+  } else if (clicked(world, VideoMenu.snapRef[menu])) {
+    const video = VideoMenu.videoRef[menu];
+    addComponent(world, MediaSnapped, video);
   }
 }
 

src/hub.js

@@ -192,6 +192,7 @@ import { renderAsEntity } from "./utils/jsx-entity";
 import { VideoMenuPrefab, loadVideoMenuButtonIcons } from "./prefabs/video-menu";
 import { loadObjectMenuButtonIcons, ObjectMenuPrefab } from "./prefabs/object-menu";
 import { loadMirrorMenuButtonIcons, MirrorMenuPrefab } from "./prefabs/mirror-menu";
+import { loadPDFMenuButtonIcons } from "./prefabs/pdf-menu";
 import { LinkHoverMenuPrefab } from "./prefabs/link-hover-menu";
 import { PDFMenuPrefab } from "./prefabs/pdf-menu";
 import { loadWaypointPreviewModel, WaypointPreview } from "./prefabs/waypoint-preview";
@@ -206,7 +207,7 @@ function addToScene(entityDef, visible) {
     obj.visible = !!visible;
   });
 }
-preload(addToScene(PDFMenuPrefab(), false));
+preload(loadPDFMenuButtonIcons().then(() => addToScene(PDFMenuPrefab(), false)));
 preload(loadObjectMenuButtonIcons().then(() => addToScene(ObjectMenuPrefab(), false)));
 preload(loadMirrorMenuButtonIcons().then(() => addToScene(MirrorMenuPrefab(), false)));
 preload(addToScene(LinkHoverMenuPrefab(), false));

src/inflators/pdf.ts

@@ -11,6 +11,7 @@ import { createPlaneBufferGeometry } from "../utils/three-utils";
 export interface PDFResources {
   pdf: PDFDocumentProxy;
   material: MeshBasicMaterial;
+  canvas: HTMLCanvasElement;
   canvasContext: CanvasRenderingContext2D;
 }
 

src/prefabs/pdf-menu.tsx

@@ -3,10 +3,17 @@ import { Color } from "three";
 import { ArrayVec3, Attrs, createElementEntity, createRef } from "../utils/jsx-entity";
 import { Button3D, BUTTON_TYPES } from "./button3D";
 import { Label } from "./camera-tool";
+import { loadTexture, loadTextureFromCache } from "../utils/load-texture";
+import snapIconSrc from "../assets/spawn_message.png";
 
 const BUTTON_HEIGHT = 0.2;
-const BUTTON_SCALE: ArrayVec3 = [0.4, 0.4, 0.4];
-const BUTTON_WIDTH = 0.3;
+const BUTTON_SCALE: ArrayVec3 = [0.6, 0.6, 0.6];
+const BIG_BUTTON_SCALE: ArrayVec3 = [0.8, 0.8, 0.8];
+const BUTTON_WIDTH = 0.2;
+
+export async function loadPDFMenuButtonIcons() {
+  return Promise.all([loadTexture(snapIconSrc, 1, "image/png")]);
+}
 
 interface PDFPageButtonProps extends Attrs {
   text: string;
@@ -16,7 +23,7 @@ function PDFPageButton(props: PDFPageButtonProps) {
   return (
     <Button3D
       name={props.name}
-      scale={BUTTON_SCALE}
+      scale={BIG_BUTTON_SCALE}
       width={BUTTON_WIDTH}
       height={BUTTON_HEIGHT}
       type={BUTTON_TYPES.ACTION}
@@ -25,25 +32,44 @@ function PDFPageButton(props: PDFPageButtonProps) {
   );
 }
 
+function SnapButton(props: Attrs) {
+  const { texture, cacheKey } = loadTextureFromCache(snapIconSrc, 1);
+  return (
+    <Button3D
+      name="Remove Button"
+      scale={BUTTON_SCALE}
+      width={BUTTON_HEIGHT}
+      height={BUTTON_WIDTH}
+      type={BUTTON_TYPES.ACTION}
+      icon={{ texture, cacheKey, scale: [0.165, 0.165, 0.165] }}
+      {...props}
+    />
+  );
+}
+
 const UI_Z = 0.001;
-const POSITION_PREV: ArrayVec3 = [-0.45, 0.0, UI_Z];
-const POSITION_NEXT: ArrayVec3 = [0.45, 0.0, UI_Z];
-const POSITION_LABEL: ArrayVec3 = [0.0, -0.35, UI_Z];
+const POSITION_PREV: ArrayVec3 = [-0.35, 0.0, UI_Z];
+const POSITION_NEXT: ArrayVec3 = [0.35, 0.0, UI_Z];
+const POSITION_LABEL: ArrayVec3 = [0.0, -0.45, UI_Z];
+const POSITION_SNAP: ArrayVec3 = [0.0, 0.45, UI_Z];
 const PAGE_LABEL_COLOR = new Color(0.1, 0.1, 0.1);
 export function PDFMenuPrefab() {
   const refPrev = createRef();
   const refNext = createRef();
   const refLabel = createRef();
+  const refSnap = createRef();
   return (
     <entity
       name="PDF Menu"
       objectMenuTransform={{ center: false }}
       pdfMenu={{
         prevButtonRef: refPrev,
         nextButtonRef: refNext,
-        pageLabelRef: refLabel
+        pageLabelRef: refLabel,
+        snapRef: refSnap
       }}
     >
+      <SnapButton name="Snap Button" ref={refSnap} position={POSITION_SNAP} />
       <PDFPageButton name="Previous Page Button" text="<" ref={refPrev} position={POSITION_PREV} />
       <PDFPageButton name="Next Page Button" text=">" ref={refNext} position={POSITION_NEXT} />
       <Label ref={refLabel} position={POSITION_LABEL} text={{ color: PAGE_LABEL_COLOR }} />

src/prefabs/video-menu.tsx

@@ -1,17 +1,26 @@
 /** @jsx createElementEntity */
 import { BoxBufferGeometry, Mesh, MeshBasicMaterial, PlaneBufferGeometry } from "three";
 import { Label } from "../prefabs/camera-tool";
-import { Attrs, createElementEntity, createRef } from "../utils/jsx-entity";
+import { ArrayVec3, Attrs, createElementEntity, createRef } from "../utils/jsx-entity";
 import playImageUrl from "../assets/images/sprites/notice/play.png";
 import pauseImageUrl from "../assets/images/sprites/notice/pause.png";
 import { BUTTON_TYPES, Button3D } from "./button3D";
 import { loadTexture, loadTextureFromCache } from "../utils/load-texture";
+import snapIconSrc from "../assets/spawn_message.png";
 
 export async function loadVideoMenuButtonIcons() {
-  return Promise.all([loadTexture(playImageUrl, 1, "image/png"), loadTexture(pauseImageUrl, 1, "image/png")]);
+  return Promise.all([
+    loadTexture(playImageUrl, 1, "image/png"),
+    loadTexture(pauseImageUrl, 1, "image/png"),
+    loadTexture(snapIconSrc, 1, "image/png")
+  ]);
 }
 
 const uiZ = 0.001;
+const BUTTON_HEIGHT = 0.2;
+const BIG_BUTTON_SCALE: ArrayVec3 = [0.8, 0.8, 0.8];
+const BUTTON_SCALE: ArrayVec3 = [0.6, 0.6, 0.6];
+const BUTTON_WIDTH = 0.2;
 
 function Slider({ trackRef, headRef, ...props }: any) {
   return (
@@ -44,31 +53,47 @@ function VideoActionButton({ buttonIcon, ...props }: VideoButtonProps) {
   const { texture, cacheKey } = loadTextureFromCache(buttonIcon, 1);
   return (
     <Button3D
-      position={[0, 0, uiZ]}
-      scale={[1, 1, 1]}
-      width={0.2}
-      height={0.2}
+      position={[0, -0.1, uiZ]}
+      scale={BIG_BUTTON_SCALE}
+      width={BUTTON_HEIGHT}
+      height={BUTTON_WIDTH}
       type={BUTTON_TYPES.DEFAULT}
       icon={{ texture, cacheKey, scale: [0.165, 0.165, 0.165] }}
       {...props}
     />
   );
 }
 
+function SnapButton(props: Attrs) {
+  const { texture, cacheKey } = loadTextureFromCache(snapIconSrc, 1);
+  return (
+    <Button3D
+      name="Remove Button"
+      scale={BUTTON_SCALE}
+      width={BUTTON_HEIGHT}
+      height={BUTTON_WIDTH}
+      type={BUTTON_TYPES.ACTION}
+      icon={{ texture, cacheKey, scale: [0.165, 0.165, 0.165] }}
+      {...props}
+    />
+  );
+}
+
 export function VideoMenuPrefab() {
   const timeLabelRef = createRef();
   const sliderRef = createRef();
   const headRef = createRef();
   const trackRef = createRef();
   const playIndicatorRef = createRef();
   const pauseIndicatorRef = createRef();
+  const snapRef = createRef();
   const halfHeight = 9 / 16 / 2;
 
   return (
     <entity
       name="Video Menu"
       objectMenuTransform={{ center: false }}
-      videoMenu={{ sliderRef, timeLabelRef, headRef, trackRef, playIndicatorRef, pauseIndicatorRef }}
+      videoMenu={{ sliderRef, timeLabelRef, headRef, trackRef, playIndicatorRef, pauseIndicatorRef, snapRef }}
     >
       <Label
         name="Time Label"
@@ -77,6 +102,7 @@ export function VideoMenuPrefab() {
         scale={[0.5, 0.5, 0.5]}
         position={[0.5 - 0.02, halfHeight - 0.02, uiZ]}
       />
+      <SnapButton name="Snap Button" ref={snapRef} position={[0.0, 0.2, uiZ]} />
       <Slider ref={sliderRef} trackRef={trackRef} headRef={headRef} position={[0, -halfHeight + 0.025, uiZ]} />
       <VideoActionButton ref={playIndicatorRef} name={"Play Button"} buttonIcon={playImageUrl} />
       <VideoActionButton ref={pauseIndicatorRef} name={"Pause Button"} buttonIcon={pauseImageUrl} />

src/systems/hubs-systems.ts

@@ -91,6 +91,7 @@ import { linkedMediaSystem } from "../bit-systems/linked-media-system";
 import { linkedVideoSystem } from "../bit-systems/linked-video-system";
 import { linkedPDFSystem } from "../bit-systems/linked-pdf-system";
 import { inspectSystem } from "../bit-systems/inspect-system";
+import { snapMediaSystem } from "../bit-systems/snap-media-system";
 
 declare global {
   interface Window {
@@ -298,6 +299,7 @@ export function mainTick(xrFrame: XRFrame, renderer: WebGLRenderer, scene: Scene
   audioDebugSystem(world);
 
   bitPenCompatSystem(world, aframeSystems["pen-tools"]);
+  snapMediaSystem(world, aframeSystems["hubs-systems"].soundEffectsSystem);
 
   deleteEntitySystem(world, aframeSystems.userinput);
   destroyAtExtremeDistanceSystem(world);