simulation: signal failure on timeout (seL4#340)

Return exit code 0 on expected success string, and exit code 1 on
timeout for all simulation scripts. Reduce standard timeout to 20min.

Previously, failure to produce the success string would lead to a
timeout which did not signal a test failure and would just continue on
to the next test.

Signed-off-by: Gerwin Klein <[email protected]>

camkes-vm/build.py

@@ -8,7 +8,7 @@
 Expects seL4-platforms/ to be co-located or otherwise in the PYTHONPATH.
 """
 
-from builds import Build, run_build_script, run_builds, load_builds, release_mq_locks, SKIP
+from builds import Build, run_build_script, run_builds, load_builds, release_mq_locks, SKIP, sim_script
 from pprint import pprint
 
 import os
@@ -44,10 +44,7 @@ def run_build(manifest_dir: str, build: Build):
     ]
 
     if plat.has_simulation and plat.name != 'PC99':
-        script.append(
-            ["bash", "-c",
-             f"expect -c 'spawn ./simulate; set timeout 3000; expect \"{build.success}\"'"]
-        )
+        script.append(sim_script(build.success))
 
     return run_build_script(manifest_dir, build, script)
 

rump-hello/build.py

@@ -8,7 +8,7 @@
 Expects seL4-platforms/ to be co-located or otherwise in the PYTHONPATH.
 """
 
-from builds import Build, run_build_script, run_builds, load_builds
+from builds import Build, run_build_script, run_builds, load_builds, sim_script
 from builds import release_mq_locks, SKIP
 from pprint import pprint
 
@@ -32,10 +32,7 @@ def run_build(manifest_dir: str, build: Build):
     ]
 
     if build.req == 'sim':
-        script.append(
-            ["bash", "-c",
-             f"expect -c 'spawn ./simulate; set timeout 3000; expect \"{build.success}\"'"]
-        )
+        script.append(sim_script(build.success))
     else:
         script.append(["tar", "czf", f"../{build.name}-images.tar.gz", "images/"])
 

seL4-platforms/builds.py

@@ -27,7 +27,8 @@
 
 # exported names:
 __all__ = [
-    "Build", "load_builds", "run_builds", "run_build_script", "junit_results", "sanitise_junit"
+    "Build", "load_builds", "run_builds", "run_build_script", "junit_results", "sanitise_junit",
+    "sim_script"
 ]
 
 # where to expect jUnit results by default
@@ -552,6 +553,14 @@ def summarise_junit(file_path: str) -> Tuple[int, List[str]]:
                   "-q", junit_results, parsed_junit_results]
 
 
+def sim_script(success: str, timeout=1200):
+    return [
+        "expect", "-c",
+        'spawn ./simulate; set timeout %d; expect { "%s" {exit 0} timeout {exit 1} }' %
+        (timeout, success)
+    ]
+
+
 def run_build_script(manifest_dir: str,
                      run: Union[Run, Build],
                      script,

sel4test-sim/build.py

@@ -18,13 +18,13 @@
 def run_simulation(manifest_dir: str, build: Build):
     """Run one simulation build and test."""
 
-    expect = f"\"{build.success}\""
+    expect = '{ "%s" {exit 0} timeout {exit 1} }' % build.success
 
     script = [
         ["../init-build.sh"] + build.settings_args(),
         ["ninja"],
         ["bash", "-c",
-         f"expect -c 'spawn ./simulate; set timeout 3000; expect {expect}' | tee {junit_results}"]
+         f"expect -c 'spawn ./simulate; set timeout 1200; expect {expect}' | tee {junit_results}"]
     ]
 
     return run_build_script(manifest_dir, build, script, junit=True)

webserver/build.py

@@ -8,7 +8,7 @@
 Expects seL4-platforms/ to be co-located or otherwise in the PYTHONPATH.
 """
 
-from builds import Build, run_build_script, run_builds, load_builds, release_mq_locks, SKIP
+from builds import Build, run_build_script, run_builds, load_builds, release_mq_locks, SKIP, sim_script
 from pprint import pprint
 
 import os
@@ -36,10 +36,7 @@ def run_build(manifest_dir: str, build: Build):
     ]
 
     if plat.has_simulation and plat.name != 'PC99':
-        script.append(
-            ["bash", "-c",
-             f"expect -c 'spawn ./simulate; set timeout 3000; expect \"{build.success}\"'"]
-        )
+        script.append(sim_script(build.success))
 
     return run_build_script(manifest_dir, build, script)