Fix D3D12 pixel history for events that do not have a pixel shader

renderdoc/driver/d3d12/d3d12_pixelhistory.cpp

@@ -511,6 +511,15 @@ struct D3D12PixelHistoryCallback : public D3D12ActionCallback
     }
   }
 
+  bool IsPSOUsingDXIL(D3D12_EXPANDED_PIPELINE_STATE_STREAM_DESC &pipeDesc)
+  {
+    // Check early stages first, for both VS and MS, but use PS as a fallback. If any shader stage
+    // uses DXIL, they all need to use DXIL.
+    return DXBC::DXBCContainer::CheckForDXIL(pipeDesc.VS.pShaderBytecode, pipeDesc.VS.BytecodeLength) ||
+           DXBC::DXBCContainer::CheckForDXIL(pipeDesc.MS.pShaderBytecode, pipeDesc.MS.BytecodeLength) ||
+           DXBC::DXBCContainer::CheckForDXIL(pipeDesc.PS.pShaderBytecode, pipeDesc.PS.BytecodeLength);
+  }
+
   // ModifyPSOForStencilIncrement modifies the provided pipeDesc, by disabling depth test
   // and write, stencil is set to always pass and increment, scissor is set to scissor around
   // the target pixel, and all color modifications are disabled.
@@ -830,8 +839,7 @@ struct D3D12OcclusionCallback : public D3D12PixelHistoryCallback
 
     ModifyPSOForStencilIncrement(eid, pipeDesc, true);
 
-    bool dxil =
-        DXBC::DXBCContainer::CheckForDXIL(pipeDesc.PS.pShaderBytecode, pipeDesc.PS.BytecodeLength);
+    bool dxil = IsPSOUsingDXIL(pipeDesc);
 
     ID3DBlob *psBlob =
         m_pDevice->GetShaderCache()->MakeFixedColShader(D3D12ShaderCache::HIGHLIGHT, dxil);
@@ -1146,7 +1154,7 @@ struct D3D12ColorAndStencilCallback : public D3D12PixelHistoryCallback
     D3D12_EXPANDED_PIPELINE_STATE_STREAM_DESC desc;
     origPSO->Fill(desc);
 
-    bool dxil = DXBC::DXBCContainer::CheckForDXIL(desc.PS.pShaderBytecode, desc.PS.BytecodeLength);
+    bool dxil = IsPSOUsingDXIL(desc);
     ID3DBlob *psBlob =
         m_pDevice->GetShaderCache()->MakeFixedColShader(D3D12ShaderCache::HIGHLIGHT, dxil);
     if(psBlob == NULL)
@@ -1683,8 +1691,7 @@ struct D3D12TestsFailedCallback : public D3D12PixelHistoryCallback
 
     if(pipeCreateFlags & PipelineCreationFlags_FixedColorShader)
     {
-      bool dxil =
-          DXBC::DXBCContainer::CheckForDXIL(pipeDesc.PS.pShaderBytecode, pipeDesc.PS.BytecodeLength);
+      bool dxil = IsPSOUsingDXIL(pipeDesc);
 
       ID3DBlob *FixedColorPS = m_ShaderCache->GetFixedColorShader(dxil);
       pipeDesc.PS.pShaderBytecode = FixedColorPS->GetBufferPointer();
@@ -2131,8 +2138,7 @@ struct D3D12PixelHistoryPerFragmentCallback : D3D12PixelHistoryCallback
     }
     else
     {
-      bool dxil =
-          DXBC::DXBCContainer::CheckForDXIL(pipeDesc.PS.pShaderBytecode, pipeDesc.PS.BytecodeLength);
+      bool dxil = IsPSOUsingDXIL(pipeDesc);
 
       // TODO: This shader outputs to SV_Target0, will we need to modify the
       // shader (or patch it) if the target image isn't RT0?

renderdoc/driver/shaders/dxbc/dxbc_container.cpp

@@ -470,6 +470,9 @@ D3D_PRIMITIVE_TOPOLOGY DXBCContainer::GetOutputTopology(const void *ByteCode, si
 
   const byte *data = (const byte *)ByteCode;    // just for convenience
 
+  if(ByteCode == NULL || ByteCodeLength == 0)
+    return D3D_PRIMITIVE_TOPOLOGY_UNDEFINED;
+
   if(header->fourcc != FOURCC_DXBC)
     return D3D_PRIMITIVE_TOPOLOGY_UNDEFINED;
 
@@ -776,7 +779,7 @@ const byte *DXBCContainer::FindChunk(const bytebuf &ByteCode, uint32_t fourcc, s
 
 void DXBCContainer::GetHash(uint32_t hash[4], const void *ByteCode, size_t BytecodeLength)
 {
-  if(BytecodeLength < sizeof(FileHeader))
+  if(BytecodeLength < sizeof(FileHeader) || ByteCode == NULL)
   {
     memset(hash, 0, sizeof(uint32_t) * 4);
     return;
@@ -816,7 +819,7 @@ void DXBCContainer::GetHash(uint32_t hash[4], const void *ByteCode, size_t Bytec
 
 bool DXBCContainer::IsHashedContainer(const void *ByteCode, size_t BytecodeLength)
 {
-  if(BytecodeLength < sizeof(FileHeader))
+  if(BytecodeLength < sizeof(FileHeader) || ByteCode == NULL)
     return false;
 
   const FileHeader *header = (const FileHeader *)ByteCode;
@@ -836,7 +839,7 @@ bool DXBCContainer::IsHashedContainer(const void *ByteCode, size_t BytecodeLengt
 
 bool DXBCContainer::HashContainer(void *ByteCode, size_t BytecodeLength)
 {
-  if(BytecodeLength < sizeof(FileHeader))
+  if(BytecodeLength < sizeof(FileHeader) || ByteCode == NULL)
     return false;
 
   FileHeader *header = (FileHeader *)ByteCode;
@@ -943,6 +946,9 @@ bool DXBCContainer::UsesExtensionUAV(uint32_t slot, uint32_t space, const void *
 
   const byte *data = (const byte *)ByteCode;    // just for convenience
 
+  if(ByteCode == NULL || BytecodeLength == 0)
+    return false;
+
   if(header->fourcc != FOURCC_DXBC)
     return false;
 
@@ -978,6 +984,9 @@ bool DXBCContainer::CheckForDebugInfo(const void *ByteCode, size_t ByteCodeLengt
 
   char *data = (char *)ByteCode;    // just for convenience
 
+  if(ByteCode == NULL || ByteCodeLength == 0)
+    return false;
+
   if(header->fourcc != FOURCC_DXBC)
     return false;
 
@@ -1003,6 +1012,9 @@ bool DXBCContainer::CheckForDXIL(const void *ByteCode, size_t ByteCodeLength)
 
   char *data = (char *)ByteCode;    // just for convenience
 
+  if(ByteCode == NULL || ByteCodeLength == 0)
+    return false;
+
   if(header->fourcc != FOURCC_DXBC)
     return false;
 
@@ -1029,6 +1041,9 @@ rdcstr DXBCContainer::GetDebugBinaryPath(const void *ByteCode, size_t ByteCodeLe
 
   char *data = (char *)ByteCode;    // just for convenience
 
+  if(ByteCode == NULL || ByteCodeLength == 0)
+    return debugPath;
+
   if(header->fourcc != FOURCC_DXBC)
     return debugPath;