Refactor sprite mask system

packages/core/src/2d/index.ts

@@ -1,5 +1,4 @@
 export { SpriteMaskInteraction } from "./enums/SpriteMaskInteraction";
-export { SpriteMaskLayer } from "./enums/SpriteMaskLayer";
 export { TextHorizontalAlignment, TextVerticalAlignment } from "./enums/TextAlignment";
 export { OverflowMode } from "./enums/TextOverflow";
 export { FontStyle } from "./enums/FontStyle";

packages/core/src/2d/sprite/SpriteMask.ts

@@ -1,4 +1,5 @@
 import { BoundingBox } from "@galacean/engine-math";
+import { Camera } from "../../Camera";
 import { Entity } from "../../Entity";
 import { RenderQueueFlags } from "../../RenderPipeline/BasicRenderPipeline";
 import { BatchUtils } from "../../RenderPipeline/BatchUtils";
@@ -8,10 +9,10 @@ import { RenderElement } from "../../RenderPipeline/RenderElement";
 import { SubPrimitiveChunk } from "../../RenderPipeline/SubPrimitiveChunk";
 import { SubRenderElement } from "../../RenderPipeline/SubRenderElement";
 import { Renderer, RendererUpdateFlags } from "../../Renderer";
+import { SpriteMaskLayer } from "../../SpriteMaskLayer";
 import { assignmentClone, ignoreClone } from "../../clone/CloneManager";
 import { ShaderProperty } from "../../shader/ShaderProperty";
 import { SimpleSpriteAssembler } from "../assembler/SimpleSpriteAssembler";
-import { SpriteMaskLayer } from "../enums/SpriteMaskLayer";
 import { SpriteModifyFlags } from "../enums/SpriteModifyFlags";
 import { Sprite } from "./Sprite";
 
@@ -200,6 +201,13 @@ export class SpriteMask extends Renderer {
     target.sprite = this._sprite;
   }
 
+  /**
+   * @inheritdoc
+   */
+  override _isCulledByCamera(camera: Camera): boolean {
+    return false;
+  }
+
   /**
    * @internal
    */

packages/core/src/2d/sprite/SpriteRenderer.ts

@@ -8,7 +8,6 @@ import { SubRenderElement } from "../../RenderPipeline/SubRenderElement";
 import { Renderer, RendererUpdateFlags } from "../../Renderer";
 import { assignmentClone, deepClone, ignoreClone } from "../../clone/CloneManager";
 import { ShaderProperty } from "../../shader/ShaderProperty";
-import { CompareFunction } from "../../shader/enums/CompareFunction";
 import { ISpriteAssembler } from "../assembler/ISpriteAssembler";
 import { SimpleSpriteAssembler } from "../assembler/SimpleSpriteAssembler";
 import { SlicedSpriteAssembler } from "../assembler/SlicedSpriteAssembler";
@@ -257,7 +256,6 @@ export class SpriteRenderer extends Renderer {
 
   set maskInteraction(value: SpriteMaskInteraction) {
     if (this._maskInteraction !== value) {
-      this._updateStencilState(this._maskInteraction, value);
       this._maskInteraction = value;
     }
   }
@@ -334,7 +332,7 @@ export class SpriteRenderer extends Renderer {
     }
     // @todo: This question needs to be raised rather than hidden.
     if (material.destroyed) {
-      material = this._engine._spriteDefaultMaterials[this._maskInteraction];
+      material = this._engine._spriteDefaultMaterial;
     }
 
     // Update position
@@ -395,28 +393,6 @@ export class SpriteRenderer extends Renderer {
     this._dirtyUpdateFlag &= ~SpriteRendererUpdateFlags.AutomaticSize;
   }
 
-  private _updateStencilState(from: SpriteMaskInteraction, to: SpriteMaskInteraction): void {
-    const material = this.getMaterial();
-    const { _spriteDefaultMaterials: spriteDefaultMaterials } = this._engine;
-    if (material === spriteDefaultMaterials[from]) {
-      this.setMaterial(spriteDefaultMaterials[to]);
-    } else {
-      const { stencilState } = material.renderState;
-      if (to === SpriteMaskInteraction.None) {
-        stencilState.enabled = false;
-        stencilState.writeMask = 0xff;
-        stencilState.referenceValue = 0;
-        stencilState.compareFunctionFront = stencilState.compareFunctionBack = CompareFunction.Always;
-      } else {
-        stencilState.enabled = true;
-        stencilState.writeMask = 0x00;
-        stencilState.referenceValue = 1;
-        stencilState.compareFunctionFront = stencilState.compareFunctionBack =
-          to === SpriteMaskInteraction.VisibleInsideMask ? CompareFunction.LessEqual : CompareFunction.Greater;
-      }
-    }
-  }
-
   @ignoreClone
   private _onSpriteChange(type: SpriteModifyFlags): void {
     switch (type) {

packages/core/src/2d/text/TextRenderer.ts

@@ -10,7 +10,6 @@ import { Renderer } from "../../Renderer";
 import { TransformModifyFlags } from "../../Transform";
 import { assignmentClone, deepClone, ignoreClone } from "../../clone/CloneManager";
 import { ShaderData, ShaderProperty } from "../../shader";
-import { CompareFunction } from "../../shader/enums/CompareFunction";
 import { ShaderDataGroup } from "../../shader/enums/ShaderDataGroup";
 import { Texture2D } from "../../texture";
 import { FontStyle } from "../enums/FontStyle";
@@ -250,7 +249,6 @@ export class TextRenderer extends Renderer {
   set maskInteraction(value: SpriteMaskInteraction) {
     if (this._maskInteraction !== value) {
       this._maskInteraction = value;
-      this._setDirtyFlagTrue(DirtyFlag.MaskInteraction);
     }
   }
 
@@ -394,11 +392,6 @@ export class TextRenderer extends Renderer {
       return;
     }
 
-    if (this._isContainDirtyFlag(DirtyFlag.MaskInteraction)) {
-      this._updateStencilState();
-      this._setDirtyFlagFalse(DirtyFlag.MaskInteraction);
-    }
-
     if (this._isContainDirtyFlag(DirtyFlag.SubFont)) {
       this._resetSubFont();
       this._setDirtyFlagFalse(DirtyFlag.SubFont);
@@ -437,29 +430,6 @@ export class TextRenderer extends Renderer {
     camera._renderPipeline.pushRenderElement(context, renderElement);
   }
 
-  private _updateStencilState(): void {
-    const material = this.getInstanceMaterial();
-    const stencilState = material.renderState.stencilState;
-    const maskInteraction = this._maskInteraction;
-
-    if (maskInteraction === SpriteMaskInteraction.None) {
-      stencilState.enabled = false;
-      stencilState.writeMask = 0xff;
-      stencilState.referenceValue = 0;
-      stencilState.compareFunctionFront = stencilState.compareFunctionBack = CompareFunction.Always;
-    } else {
-      stencilState.enabled = true;
-      stencilState.writeMask = 0x00;
-      stencilState.referenceValue = 1;
-      const compare =
-        maskInteraction === SpriteMaskInteraction.VisibleInsideMask
-          ? CompareFunction.LessEqual
-          : CompareFunction.Greater;
-      stencilState.compareFunctionFront = compare;
-      stencilState.compareFunctionBack = compare;
-    }
-  }
-
   private _resetSubFont(): void {
     const font = this._font;
     this._subFont = font._getSubFont(this.fontSize, this.fontStyle);
@@ -762,7 +732,6 @@ enum DirtyFlag {
   LocalPositionBounds = 0x2,
   WorldPosition = 0x4,
   WorldBounds = 0x8,
-  MaskInteraction = 0x10,
   Color = 0x20,
 
   Position = LocalPositionBounds | WorldPosition | WorldBounds,

packages/core/src/Engine.ts

@@ -7,7 +7,6 @@ import {
   IXRDevice
 } from "@galacean/engine-design";
 import { Color } from "@galacean/engine-math";
-import { SpriteMaskInteraction } from "./2d";
 import { CharRenderInfo } from "./2d/text/CharRenderInfo";
 import { Font } from "./2d/text/Font";
 import { BasicResources } from "./BasicResources";
@@ -30,7 +29,6 @@ import { Material } from "./material/Material";
 import { ParticleBufferUtils } from "./particle/ParticleBufferUtils";
 import { PhysicsScene } from "./physics/PhysicsScene";
 import { ColliderShape } from "./physics/shape/ColliderShape";
-import { CompareFunction } from "./shader";
 import { Shader } from "./shader/Shader";
 import { ShaderMacro } from "./shader/ShaderMacro";
 import { ShaderMacroCollection } from "./shader/ShaderMacroCollection";
@@ -94,8 +92,6 @@ export class Engine extends EventDispatcher {
   _basicResources: BasicResources;
   /* @internal */
   _spriteDefaultMaterial: Material;
-  /** @internal */
-  _spriteDefaultMaterials: Material[] = [];
   /* @internal */
   _textDefaultMaterial: Material;
   /* @internal */
@@ -239,16 +235,7 @@ export class Engine extends EventDispatcher {
 
     this._canvas = canvas;
 
-    const { _spriteDefaultMaterials: spriteDefaultMaterials } = this;
-    this._spriteDefaultMaterial = spriteDefaultMaterials[SpriteMaskInteraction.None] = this._createSpriteMaterial(
-      SpriteMaskInteraction.None
-    );
-    spriteDefaultMaterials[SpriteMaskInteraction.VisibleInsideMask] = this._createSpriteMaterial(
-      SpriteMaskInteraction.VisibleInsideMask
-    );
-    spriteDefaultMaterials[SpriteMaskInteraction.VisibleOutsideMask] = this._createSpriteMaterial(
-      SpriteMaskInteraction.VisibleOutsideMask
-    );
+    this._spriteDefaultMaterial = this._createSpriteMaterial();
     this._textDefaultMaterial = this._createTextMaterial();
     this._spriteMaskDefaultMaterial = this._createSpriteMaskMaterial();
     this._textDefaultFont = Font.createFromOS(this, "Arial");
@@ -581,7 +568,7 @@ export class Engine extends EventDispatcher {
     return Promise.all(initializePromises).then(() => this);
   }
 
-  private _createSpriteMaterial(maskInteraction: SpriteMaskInteraction): Material {
+  private _createSpriteMaterial(): Material {
     const material = new Material(this, Shader.find("Sprite"));
     const renderState = material.renderState;
     const target = renderState.blendState.targetBlendState;
@@ -591,18 +578,6 @@ export class Engine extends EventDispatcher {
     target.sourceAlphaBlendFactor = BlendFactor.One;
     target.destinationAlphaBlendFactor = BlendFactor.OneMinusSourceAlpha;
     target.colorBlendOperation = target.alphaBlendOperation = BlendOperation.Add;
-    if (maskInteraction !== SpriteMaskInteraction.None) {
-      const stencilState = renderState.stencilState;
-      stencilState.enabled = true;
-      stencilState.writeMask = 0x00;
-      stencilState.referenceValue = 1;
-      const compare =
-        maskInteraction === SpriteMaskInteraction.VisibleInsideMask
-          ? CompareFunction.LessEqual
-          : CompareFunction.Greater;
-      stencilState.compareFunctionFront = compare;
-      stencilState.compareFunctionBack = compare;
-    }
     renderState.depthState.writeEnabled = false;
     renderState.rasterState.cullMode = CullMode.Off;
     renderState.renderQueueType = RenderQueueType.Transparent;
@@ -617,7 +592,6 @@ export class Engine extends EventDispatcher {
     renderState.rasterState.cullMode = CullMode.Off;
     renderState.stencilState.enabled = true;
     renderState.depthState.enabled = false;
-    renderState.renderQueueType = RenderQueueType.Transparent;
     material.isGCIgnored = true;
     return material;
   }

packages/core/src/RenderPipeline/BasicRenderPipeline.ts

@@ -169,6 +169,7 @@ export class BasicRenderPipeline {
       rhi.clearRenderTarget(camera.engine, clearFlags, color);
     }
 
+    scene._maskManager.preMaskLayer = 0;
     opaqueQueue.render(context, PipelineStage.Forward);
     alphaTestQueue.render(context, PipelineStage.Forward);
     if (clearFlags & CameraClearFlags.Color) {
@@ -338,7 +339,7 @@ export class BasicRenderPipeline {
       const renderer = elements[i];
 
       // Filter by camera culling mask
-      if (!(camera.cullingMask & renderer._entity.layer)) {
+      if (renderer._isCulledByCamera(camera)) {
         continue;
       }
 

packages/core/src/RenderPipeline/MaskManager.ts

@@ -54,8 +54,10 @@ export class MaskManager {
         }
 
         if (influenceLayers & curMaskLayer) {
+          mask.getMaterial().renderState.renderQueueType = incrementMaskQueue.renderQueueType;
           incrementMaskQueue.pushRenderElement(mask._renderElement);
         } else if (influenceLayers & reduceLayer) {
+          mask.getMaterial().renderState.renderQueueType = decrementMaskQueue.renderQueueType;
           decrementMaskQueue.pushRenderElement(mask._renderElement);
         }
       }

packages/core/src/RenderPipeline/RenderQueue.ts

@@ -1,6 +1,6 @@
 import { SpriteMaskInteraction } from "../2d/enums/SpriteMaskInteraction";
 import { Utils } from "../Utils";
-import { RenderQueueType, Shader, StencilOperation } from "../shader";
+import { CompareFunction, RenderQueueType, Shader, StencilOperation, StencilState } from "../shader";
 import { ShaderMacroCollection } from "../shader/ShaderMacroCollection";
 import { BatcherManager } from "./BatcherManager";
 import { MaskManager } from "./MaskManager";
@@ -12,6 +12,8 @@ import { SubRenderElement } from "./SubRenderElement";
  * @internal
  */
 export class RenderQueue {
+  private static _tempStencilState: StencilState = new StencilState();
+
   static compareForOpaque(a: RenderElement, b: RenderElement): number {
     return a.priority - b.priority || a.distanceForSort - b.distanceForSort;
   }
@@ -37,7 +39,7 @@ export class RenderQueue {
   render(
     context: RenderContext,
     pipelineStageTagValue: string,
-    maskType: RenderQueueMaskType = RenderQueueMaskType.No
+    stencilOperation: StencilOperation = StencilOperation.Keep
   ): void {
     const batchedSubElements = this.batchedSubElements;
     const length = batchedSubElements.length;
@@ -52,7 +54,7 @@ export class RenderQueue {
     const rhi = engine._hardwareRenderer;
     const pipelineStageKey = RenderContext.pipelineStageKey;
     const renderQueueType = this.renderQueueType;
-    scene._maskManager.preMaskLayer = 0;
+    const tempStencilState = RenderQueue._tempStencilState;
 
     for (let i = 0; i < length; i++) {
       const subElement = batchedSubElements[i];
@@ -71,8 +73,9 @@ export class RenderQueue {
         renderer._updateTransformShaderData(context, true, batched);
       }
 
-      renderer._maskInteraction !== SpriteMaskInteraction.None &&
-        this._drawMask(context, pipelineStageTagValue, subElement);
+      const maskInteraction = renderer._maskInteraction;
+      const maskInteractionNotNone = maskInteraction !== SpriteMaskInteraction.None;
+      maskInteractionNotNone && this._drawMask(context, pipelineStageTagValue, subElement);
 
       const compileMacros = Shader._compileMacros;
       const { primitive, material, shaderPasses, shaderData: renderElementShaderData } = subElement;
@@ -82,16 +85,10 @@ export class RenderQueue {
       // Union render global macro and material self macro
       ShaderMacroCollection.unionCollection(renderer._globalShaderMacro, materialData._macroCollection, compileMacros);
 
-      // TODO: Mask should not modify material's render state, will delete this code after mask refactor
-      if (maskType !== RenderQueueMaskType.No) {
-        const operation =
-          maskType === RenderQueueMaskType.Increment
-            ? StencilOperation.IncrementSaturate
-            : StencilOperation.DecrementSaturate;
-
+      if (stencilOperation !== StencilOperation.Keep) {
         const { stencilState } = material.renderState;
-        stencilState.passOperationFront = operation;
-        stencilState.passOperationBack = operation;
+        stencilState.passOperationFront = stencilOperation;
+        stencilState.passOperationBack = stencilOperation;
       }
 
       for (let j = 0, m = shaderPasses.length; j < m; j++) {
@@ -164,6 +161,19 @@ export class RenderQueue {
         }
 
         const renderState = shaderPass._renderState ?? renderStates[j];
+        const stencilState = renderState.stencilState;
+        if (maskInteractionNotNone) {
+          // Copy origin stencil state to temp stencil state
+          this._copyStencilState(stencilState, tempStencilState);
+          // Set stencil state for renderer
+          stencilState.enabled = true;
+          stencilState.writeMask = 0x00;
+          stencilState.referenceValue = 1;
+          stencilState.compareFunctionFront = stencilState.compareFunctionBack =
+            maskInteraction === SpriteMaskInteraction.VisibleInsideMask
+              ? CompareFunction.LessEqual
+              : CompareFunction.Greater;
+        }
         renderState._applyStates(
           engine,
           renderer.entity.transform._isFrontFaceInvert(),
@@ -172,6 +182,8 @@ export class RenderQueue {
         );
 
         rhi.drawPrimitive(primitive, subElement.subPrimitive, program);
+        // Restore the previous stencil state from temp stencil state
+        maskInteractionNotNone && this._copyStencilState(tempStencilState, stencilState);
       }
     }
   }
@@ -205,14 +217,16 @@ export class RenderQueue {
     camera.scene._maskManager.buildMaskRenderElement(master, incrementMaskQueue, decrementMaskQueue);
 
     incrementMaskQueue._batch(engine._batcherManager);
-    incrementMaskQueue.render(context, pipelineStageTagValue, RenderQueueMaskType.Increment);
+    incrementMaskQueue.render(context, pipelineStageTagValue, StencilOperation.IncrementSaturate);
     decrementMaskQueue._batch(engine._batcherManager);
-    decrementMaskQueue.render(context, pipelineStageTagValue, RenderQueueMaskType.Decrement);
+    decrementMaskQueue.render(context, pipelineStageTagValue, StencilOperation.DecrementSaturate);
   }
-}
 
-enum RenderQueueMaskType {
-  No,
-  Increment,
-  Decrement
+  private _copyStencilState(scrStencilState: StencilState, dstStencilState: StencilState): void {
+    dstStencilState.enabled = scrStencilState.enabled;
+    dstStencilState.writeMask = scrStencilState.writeMask;
+    dstStencilState.referenceValue = scrStencilState.referenceValue;
+    dstStencilState.compareFunctionFront = scrStencilState.compareFunctionFront;
+    dstStencilState.compareFunctionBack = scrStencilState.compareFunctionBack;
+  }
 }