Depth overlay updates : only use the stencil mask approach when the shader exports depth

renderdoc/driver/d3d11/d3d11_overlay.cpp

@@ -1274,6 +1274,7 @@ ResourceId D3D11Replay::RenderOverlay(ResourceId texid, FloatVector clearCol, De
       ID3D11DepthStencilView *dsNewView = NULL;
 
       D3D11_DEPTH_STENCIL_DESC d = dsDesc;
+      bool useDepthWriteStencilPass = false;
 
       if(overlay == DebugOverlay::Depth)
       {
@@ -1320,6 +1321,21 @@ ResourceId D3D11Replay::RenderOverlay(ResourceId texid, FloatVector clearCol, De
       UINT prevNumClassInstances = 0;
       m_pImmediateContext->PSGetShader(&prevPS, prevClassInstances, &prevNumClassInstances);
 
+      if(overlay == DebugOverlay::Depth)
+      {
+        WrappedShader *wrappedPS =
+            (WrappedShader *)(WrappedID3D11Shader<ID3D11PixelShader> *)(prevPS);
+        if(wrappedPS)
+        {
+          ShaderReflection &reflection = wrappedPS->GetDetails();
+          for(SigParameter &output : reflection.outputSignature)
+          {
+            if(output.systemValue == ShaderBuiltin::DepthOutput)
+              useDepthWriteStencilPass = true;
+          }
+        }
+      }
+
       {
         float failColour[] = {1.0f, 0.0f, 0.0f, 1.0f};
         ID3D11Buffer *buf = GetDebugManager()->MakeCBuffer(failColour, sizeof(failColour));
@@ -1336,7 +1352,7 @@ ResourceId D3D11Replay::RenderOverlay(ResourceId texid, FloatVector clearCol, De
       m_pDevice->ReplayLog(0, eventId, eReplay_OnlyDraw);
 
       // if buffer was depth only then check if current depth target supports stencil
-      if(overlay == DebugOverlay::Depth)
+      if(useDepthWriteStencilPass)
       {
         DXGI_FORMAT dsCurFmt = dsViewDesc.Format;
         DXGI_FORMAT dsNewFmt = dsCurFmt;
@@ -1534,10 +1550,9 @@ ResourceId D3D11Replay::RenderOverlay(ResourceId texid, FloatVector clearCol, De
       d = dsDesc;
       d.DepthWriteMask = D3D11_DEPTH_WRITE_MASK_ALL;
       d.StencilWriteMask = D3D11_DEFAULT_STENCIL_WRITE_MASK;
-      if(overlay == DebugOverlay::Depth)
+      if(useDepthWriteStencilPass)
       {
         // Write stencil 0x1 for depth passing pixels
-        d.DepthFunc = cur.DepthFunc;
         d.StencilEnable = TRUE;
         d.FrontFace.StencilFunc = D3D11_COMPARISON_ALWAYS;
         d.FrontFace.StencilFailOp = D3D11_STENCIL_OP_KEEP;
@@ -1546,6 +1561,10 @@ ResourceId D3D11Replay::RenderOverlay(ResourceId texid, FloatVector clearCol, De
         d.BackFace = d.FrontFace;
         stencilRef = 1;
       }
+      if(overlay == DebugOverlay::Depth)
+      {
+        d.DepthFunc = cur.DepthFunc;
+      }
       else if(overlay == DebugOverlay::Stencil)
       {
         d.FrontFace = cur.FrontFace;
@@ -1562,7 +1581,7 @@ ResourceId D3D11Replay::RenderOverlay(ResourceId texid, FloatVector clearCol, De
 
       m_pImmediateContext->OMSetDepthStencilState(os, stencilRef);
 
-      if(overlay == DebugOverlay::Depth)
+      if(useDepthWriteStencilPass)
       {
         m_pImmediateContext->ClearDepthStencilView(dsView, D3D11_CLEAR_STENCIL, 0.0f, 0x0);
         m_pImmediateContext->OMSetBlendState(m_Overlay.DepthBlendRTMaskZero, NULL, 0xffffffff);
@@ -1579,7 +1598,7 @@ ResourceId D3D11Replay::RenderOverlay(ResourceId texid, FloatVector clearCol, De
 
       m_pDevice->ReplayLog(0, eventId, eReplay_OnlyDraw);
 
-      if(overlay == DebugOverlay::Depth)
+      if(useDepthWriteStencilPass)
       {
         // Resolve stencil = 0x1 pixels to green
         m_pImmediateContext->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);

renderdoc/driver/d3d12/d3d12_overlay.cpp

@@ -2120,6 +2120,21 @@ ResourceId D3D12Replay::RenderOverlay(ResourceId texid, FloatVector clearCol, De
       ID3D12Resource *renderDepthStencil = NULL;
       bool useDepthWriteStencilPass = (overlay == DebugOverlay::Depth) && renderDepth;
 
+      if(useDepthWriteStencilPass)
+      {
+        useDepthWriteStencilPass = false;
+        WrappedID3D12PipelineState::ShaderEntry *wrappedPS = pipe->PS();
+        if(wrappedPS)
+        {
+          ShaderReflection &reflection = pipe->PS()->GetDetails();
+          for(SigParameter &output : reflection.outputSignature)
+          {
+            if(output.systemValue == ShaderBuiltin::DepthOutput)
+              useDepthWriteStencilPass = true;
+          }
+        }
+      }
+
       HRESULT hr;
       DXGI_FORMAT dsFmt = dsViewDesc.Format;
       // the depth overlay uses stencil buffer as a mask for the passing pixels

renderdoc/driver/gl/gl_overlay.cpp

@@ -1011,9 +1011,24 @@ ResourceId GLReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, Debug
     GLenum copyQueryEnum = texQueryEnum;
 
     GLuint depthCopy = 0, stencilCopy = 0;
-    bool useBlitFramebuffer = true;
     bool useDepthStencilMask = (overlay == DebugOverlay::Depth) && (curDepth != 0);
 
+    if(useDepthStencilMask)
+    {
+      useDepthStencilMask = false;
+      PerStageReflections stages;
+      m_pDriver->FillReflectionArray(rs.Program, stages);
+      const ShaderReflection *reflection = stages.refls[(uint32_t)ShaderStage::Fragment];
+      if(reflection)
+      {
+        for(const SigParameter &output : reflection->outputSignature)
+        {
+          if(output.systemValue == ShaderBuiltin::DepthOutput)
+            useDepthStencilMask = true;
+        }
+      }
+    }
+
     // create matching depth for existing FBO
     if(curDepth != 0)
     {
@@ -1051,8 +1066,8 @@ ResourceId GLReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, Debug
         else if(depth == 0 && stencil == 8)
           fmt = eGL_STENCIL_INDEX8;
       }
-      // For depth overlay : need a stencil buffer
-      if(overlay == DebugOverlay::Depth)
+      // Need a stencil buffer for the depth overlay stencil mask method
+      if(useDepthStencilMask)
       {
         GLenum oldFmt = fmt;
         if((oldFmt == eGL_DEPTH_COMPONENT16) || (oldFmt == eGL_DEPTH_COMPONENT24) ||
@@ -1066,11 +1081,11 @@ ResourceId GLReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, Debug
           if(DebugData.overlayTexSlices > 1)
           {
             useDepthStencilMask = false;
+            fmt = oldFmt;
             RDCWARN("Depth overlay using fallback method instead of stencil mask");
           }
           else
           {
-            useBlitFramebuffer = false;
             curStencil = curDepth;
           }
         }
@@ -1271,6 +1286,7 @@ ResourceId GLReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, Debug
                                  stencilCopy, sub.mip);
     }
 
+    bool useBlitFramebuffer = !useDepthStencilMask;
     if(useBlitFramebuffer)
     {
       // get latest depth/stencil from read FBO (existing FBO) into draw FBO (overlay FBO)

renderdoc/driver/vulkan/vk_overlay.cpp

@@ -1644,6 +1644,24 @@ ResourceId VulkanReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, D
         if(overlay == DebugOverlay::Depth)
           useDepthWriteStencilPass = true;
 
+        if(useDepthWriteStencilPass)
+        {
+          useDepthWriteStencilPass = false;
+          const VulkanCreationInfo::Pipeline::Shader &ps = pipeInfo.shaders[4];
+          if(ps.module != ResourceId())
+          {
+            ShaderReflection *reflection = ps.refl;
+            if(reflection)
+            {
+              for(SigParameter &output : reflection->outputSignature)
+              {
+                if(output.systemValue == ShaderBuiltin::DepthOutput)
+                  useDepthWriteStencilPass = true;
+              }
+            }
+          }
+        }
+
         VkAttachmentDescription attDescs[] = {
             {0, overlayFormat, VK_SAMPLE_COUNT_1_BIT, VK_ATTACHMENT_LOAD_OP_LOAD,
              VK_ATTACHMENT_STORE_OP_STORE, VK_ATTACHMENT_LOAD_OP_DONT_CARE,

util/test/demos/d3d11/d3d11_overlay_test.cpp

@@ -69,6 +69,34 @@ PixOut main(v2f IN)
   return OUT;
 }
 
+)EOSHADER";
+
+  std::string discardPixel = R"EOSHADER(
+
+struct v2f
+{
+	float4 pos : SV_POSITION;
+	float4 col : COLOR0;
+	float2 uv : TEXCOORD0;
+};
+
+struct PixOut
+{
+	float4 colour : SV_Target0;
+};
+
+PixOut main(v2f IN)
+{
+  PixOut OUT;
+	OUT.colour  = IN.col;
+  if ((IN.pos.x > 327.0) && (IN.pos.x < 339.0) &&
+      (IN.pos.y > 38.0) && (IN.pos.y < 48.0))
+	{
+    discard;
+	}
+  return OUT;
+}
+
 )EOSHADER";
 
   int main()
@@ -86,6 +114,7 @@ PixOut main(v2f IN)
     ID3D11PixelShaderPtr ps = CreatePS(psblob);
     ID3D11PixelShaderPtr whiteps = CreatePS(Compile(whitePixel, "main", "ps_4_0"));
     ID3D11PixelShaderPtr depthwriteps = CreatePS(Compile(depthWritePixel, "main", "ps_4_0"));
+    ID3D11PixelShaderPtr discardps = CreatePS(Compile(discardPixel, "main", "ps_4_0"));
 
     const DefaultA2V VBData[] = {
         // this triangle occludes in depth
@@ -164,6 +193,11 @@ PixOut main(v2f IN)
         {Vec3f(+1.0f, -1.0f, 0.99f), Vec4f(0.2f, 0.2f, 0.2f, 1.0f), Vec2f(0.0f, 0.0f)},
         {Vec3f(-1.0f, +1.0f, 0.99f), Vec4f(0.2f, 0.2f, 0.2f, 1.0f), Vec2f(0.0f, 0.0f)},
         {Vec3f(+1.0f, +1.0f, 0.99f), Vec4f(0.2f, 0.2f, 0.2f, 1.0f), Vec2f(0.0f, 0.0f)},
+
+        // discard rectangle
+        {Vec3f(0.6f, +0.7f, 0.5f), Vec4f(0.0f, 0.0f, 0.0f, 1.0f), Vec2f(0.0f, 0.0f)},
+        {Vec3f(0.7f, +0.9f, 0.5f), Vec4f(0.0f, 0.0f, 0.0f, 1.0f), Vec2f(0.0f, 1.0f)},
+        {Vec3f(0.8f, +0.7f, 0.5f), Vec4f(0.0f, 0.0f, 0.0f, 1.0f), Vec2f(1.0f, 0.0f)},
     };
 
     ID3D11BufferPtr vb = MakeBuffer().Vertex().Data(VBData);
@@ -299,6 +333,11 @@ PixOut main(v2f IN)
           SetDepthState(depth);
           ctx->PSSetShader(depthwriteps, NULL, 0);
           ctx->Draw(24, 9);
+
+          markerName = "Discard " + markerName;
+          setMarker(markerName);
+          ctx->PSSetShader(discardps, NULL, 0);
+          ctx->Draw(3, 42);
           ctx->PSSetShader(ps, NULL, 0);
 
           depth.StencilEnable = FALSE;

util/test/demos/d3d12/d3d12_overlay_test.cpp

@@ -110,6 +110,34 @@ PixOut main(v2f IN)
 	else
 	{
 		OUT.depth = IN.pos.z;
+  }
+  return OUT;
+}
+
+)EOSHADER";
+
+  std::string discardPixel = R"EOSHADER(
+
+struct v2f
+{
+	float4 col : COLOR0;
+	float2 uv : TEXCOORD0;
+	float4 pos : SV_POSITION;
+};
+
+struct PixOut
+{
+	float4 colour : SV_Target0;
+};
+
+PixOut main(v2f IN)
+{
+  PixOut OUT;
+	OUT.colour  = IN.col;
+  if ((IN.pos.x > 327.0) && (IN.pos.x < 339.0) &&
+      (IN.pos.y > 38.0) && (IN.pos.y < 48.0))
+	{
+    discard;
 	}
   return OUT;
 }
@@ -126,6 +154,7 @@ PixOut main(v2f IN)
     ID3DBlobPtr psblob[3] = {};
     ID3DBlobPtr whitepsblob[3] = {};
     ID3DBlobPtr depthwritepsblob[3] = {};
+    ID3DBlobPtr discardpsblob[3] = {};
 
     {
       int i = 0;
@@ -138,6 +167,7 @@ PixOut main(v2f IN)
         psblob[i] = Compile(vertexEndPosPixel, "main", "ps" + profile);
         whitepsblob[i] = Compile(whitePixel, "main", "ps" + profile);
         depthwritepsblob[i] = Compile(depthWritePixel, "main", "ps" + profile);
+        discardpsblob[i] = Compile(discardPixel, "main", "ps" + profile);
         i++;
       }
     }
@@ -205,6 +235,11 @@ PixOut main(v2f IN)
         {Vec3f(-1.3f, -1.3f, 0.95f), Vec4f(0.1f, 0.1f, 0.5f, 1.0f), Vec2f(0.0f, 0.0f)},
         {Vec3f(0.0f, 1.3f, 0.95f), Vec4f(0.1f, 0.1f, 0.5f, 1.0f), Vec2f(0.0f, 1.0f)},
         {Vec3f(1.3f, -1.3f, 0.95f), Vec4f(0.1f, 0.1f, 0.5f, 1.0f), Vec2f(1.0f, 0.0f)},
+
+        // discard rectangle
+        {Vec3f(0.6f, +0.7f, 0.5f), Vec4f(0.0f, 0.0f, 0.0f, 1.0f), Vec2f(0.0f, 0.0f)},
+        {Vec3f(0.7f, +0.9f, 0.5f), Vec4f(0.0f, 0.0f, 0.0f, 1.0f), Vec2f(0.0f, 1.0f)},
+        {Vec3f(0.8f, +0.7f, 0.5f), Vec4f(0.0f, 0.0f, 0.0f, 1.0f), Vec2f(1.0f, 0.0f)},
     };
 
     ID3D12ResourcePtr vb = MakeBuffer().Data(VBData);
@@ -228,6 +263,7 @@ PixOut main(v2f IN)
     ID3D12PipelineStatePtr backgroundPipe[3][countFmts][2];
     ID3D12PipelineStatePtr pipe[3][countFmts][2];
     ID3D12PipelineStatePtr depthWritePixelShaderPipe[3][countFmts][2];
+    ID3D12PipelineStatePtr discardPixelShaderPipe[3][countFmts][2];
     ID3D12PipelineStatePtr whitepipe[3];
     ID3D12PipelineStatePtr sampleMaskPipe[3][countFmts];
 
@@ -307,6 +343,12 @@ PixOut main(v2f IN)
         creator.GraphicsDesc.SampleDesc = yesMSAA;
         depthWritePixelShaderPipe[i][f][1] = creator;
 
+        creator.PS(discardpsblob[i]);
+        creator.GraphicsDesc.SampleDesc = noMSAA;
+        discardPixelShaderPipe[i][f][0] = creator;
+        creator.GraphicsDesc.SampleDesc = yesMSAA;
+        discardPixelShaderPipe[i][f][1] = creator;
+
         creator.PS(psblob[i]);
         creator.GraphicsDesc.SampleDesc = yesMSAA;
         sampleMaskPipe[i][f] = creator;
@@ -446,6 +488,11 @@ PixOut main(v2f IN)
 
             cmd->SetPipelineState(depthWritePixelShaderPipe[pass][f][is_msaa ? 1 : 0]);
             cmd->DrawInstanced(24, 1, 9, 0);
+
+            markerName = "Discard " + markerName;
+            setMarker(cmd, markerName);
+            cmd->SetPipelineState(discardPixelShaderPipe[pass][f][is_msaa ? 1 : 0]);
+            cmd->DrawInstanced(3, 1, 36, 0);
             cmd->SetPipelineState(pipe[pass][f][is_msaa ? 1 : 0]);
 
             if(!is_msaa)