diff --git a/tests/testsuite.src/used_binaries.at b/tests/testsuite.src/used_binaries.at
index 1d1cf61ed..13de5781b 100644
--- a/tests/testsuite.src/used_binaries.at
+++ b/tests/testsuite.src/used_binaries.at
@@ -372,10 +372,16 @@ AT_DATA([prog.cob], [
            EXIT PROGRAM.
 ])
 
-AT_CHECK([TMPDIR="" TMP="notthere" TEMP="" $COMPILE prog.cob], [0], [],
-[libcob: warning: Temporary directory TMP is invalid, adjust TMPDIR!
+AT_CHECK([TMPDIR="" TMP="notthere" TEMP="" $COMPILE prog.cob 2> compiler.output], [0], [],
+[ignore],[
+    AT_CHECK([grep libcob: compiler.output], [0], [libcob: warning: Temporary directory TMP is invalid, adjust TMPDIR!])
+  ],
+  [
+    AT_CHECK([cat compiler.output], [0], [libcob: warning: Temporary directory TMP is invalid, adjust TMPDIR!
 ])
-AT_CHECK([$COBCRUN_DIRECT ./prog], [0], [OK], [])
+    AT_CHECK([$COBCRUN_DIRECT ./prog], [0], [OK], [])
+  ])
+
 AT_CHECK([TMPDIR="" TMP="" TEMP="./prog.cob" $COMPILE prog.cob], [0], [],
 [libcob: warning: Temporary directory TEMP is invalid, adjust TMPDIR!
 ])