[DXIL] Avoid infinite loop / stack overflow on invalid instruction (#…

…7048)

The simplify instruction pass has a function called `findScalarElement`.
This function will possibly recurse infinitely into the first operand of
the instruction that it is passed. If this instruction, for example, is
`%2 = insertelement <2 x i32> %2, i32 %sub, i32 0`
Then the pass will recurse into the first operand of the insertelement
instruction, which is the same insertelement instruction, and so on.
This PR prevents infinite recursion and stack overflow by bailing out of
the function early if there is a detection of the same value in the
first operand.
Fixes #7034

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

lib/Analysis/VectorUtils2.cpp

@@ -42,6 +42,10 @@ llvm::Value *llvm::findScalarElement(llvm::Value *V, unsigned EltNo) {
     if (EltNo == IIElt)
       return III->getOperand(1);
 
+    // Guard against infinite loop on malformed, unreachable IR.
+    if (III == III->getOperand(0))
+      return nullptr;
+
     // Otherwise, the insertelement doesn't modify the value, recurse on its
     // vector input.
     return findScalarElement(III->getOperand(0), EltNo);

tools/clang/test/CodeGenHLSL/minimal2.ll

@@ -0,0 +1,49 @@
+; RUN: opt -simplify-inst -S < %s | FileCheck %s
+
+%"class.Texture3D<unsigned int>" = type { i32, %"class.Texture3D<unsigned int>::mips_type" }
+%"class.Texture3D<unsigned int>::mips_type" = type { i32 }
+%dx.types.Handle = type { i8* }
+%dx.types.ResourceProperties = type { i32, i32 }
+
+@"\01?t1@@3V?$Texture3D@I@@A" = external global %"class.Texture3D<unsigned int>", align 4
+
+; Function Attrs: nounwind
+define void @main(<2 x i32> %dtid, i32 %laneID) #0 {
+"\01?f1@@s1@@U1@@Z.exit":
+  br label %if.end.13
+
+while.cond.6.preheader:                           ; No predecessors!
+  ; CHECK: %0 = extractelement <2 x i32> %1, i32 1
+  %0 = extractelement <2 x i32> %2, i32 1
+  %cmp7.14 = icmp ne i32 %0, 0
+  br i1 %cmp7.14, label %while.body, label %if.end.13
+
+while.body:                                       ; preds = %while.cond.6.preheader
+  %1 = extractelement <2 x i32> %2, i32 0
+  ; CHECK: %sub = sub i32 %sub, 1
+  ; CHECK: %1 = insertelement <2 x i32> %1, i32 %sub, i32 0
+  %sub = sub i32 %1, 1
+  %2 = insertelement <2 x i32> %2, i32 %sub, i32 0
+  br label %if.end.13
+
+if.end.13:                                        ; preds = %while.body, %while.cond.6.preheader, %"\01?f1@@s1@@U1@@Z.exit"
+  ret void
+}
+
+; Function Attrs: nounwind
+declare void @llvm.lifetime.start(i64, i8* nocapture) #0
+
+; Function Attrs: nounwind
+declare void @llvm.lifetime.end(i64, i8* nocapture) #0
+
+; Function Attrs: nounwind readnone
+declare i1 @"dx.hl.op.rn.i1 (i32, i1)"(i32, i1) #1
+
+; Function Attrs: nounwind readonly
+declare i32 @"dx.hl.op.ro.i32 (i32, %dx.types.Handle, <4 x i32>)"(i32, %dx.types.Handle, <4 x i32>) #2
+
+; Function Attrs: nounwind readnone
+declare %dx.types.Handle @"dx.hl.createhandle..%dx.types.Handle (i32, %\22class.Texture3D<unsigned int>\22)"(i32, %"class.Texture3D<unsigned int>") #1
+
+; Function Attrs: nounwind readnone
+declare %dx.types.Handle @"dx.hl.annotatehandle..%dx.types.Handle (i32, %dx.types.Handle, %dx.types.ResourceProperties, %\22class.Texture3D<unsigned int>\22)"(i32, %dx.types.Handle, %dx.types.ResourceProperties, %"class.Texture3D<unsigned int>") #1