From 49c59bf2b22ff88fecf7ba8da61404bbef6d620a Mon Sep 17 00:00:00 2001
From: Gerwin Klein <gerwin.klein@proofcraft.systems>
Date: Sat, 2 Mar 2024 00:46:16 +0100
Subject: [PATCH] simulation: signal failure on timeout (#340)

Return exit code 0 on expected success string, and exit code 1 on
timeout for all simulation scripts. Reduce standard timeout to 20min.

Previously, failure to produce the success string would lead to a
timeout which did not signal a test failure and would just continue on
to the next test.

Signed-off-by: Gerwin Klein <gerwin.klein@proofcraft.systems>
---
 camkes-vm/build.py       |  7 ++-----
 rump-hello/build.py      |  7 ++-----
 seL4-platforms/builds.py | 11 ++++++++++-
 sel4test-sim/build.py    |  4 ++--
 webserver/build.py       |  7 ++-----
 5 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/camkes-vm/build.py b/camkes-vm/build.py
index a5b8625d..de8ebbe8 100644
--- a/camkes-vm/build.py
+++ b/camkes-vm/build.py
@@ -8,7 +8,7 @@
 Expects seL4-platforms/ to be co-located or otherwise in the PYTHONPATH.
 """
 
-from builds import Build, run_build_script, run_builds, load_builds, release_mq_locks, SKIP
+from builds import Build, run_build_script, run_builds, load_builds, release_mq_locks, SKIP, sim_script
 from pprint import pprint
 
 import os
@@ -44,10 +44,7 @@ def run_build(manifest_dir: str, build: Build):
     ]
 
     if plat.has_simulation and plat.name != 'PC99':
-        script.append(
-            ["bash", "-c",
-             f"expect -c 'spawn ./simulate; set timeout 3000; expect \"{build.success}\"'"]
-        )
+        script.append(sim_script(build.success))
 
     return run_build_script(manifest_dir, build, script)
 
diff --git a/rump-hello/build.py b/rump-hello/build.py
index 8da57225..c4c78b21 100644
--- a/rump-hello/build.py
+++ b/rump-hello/build.py
@@ -8,7 +8,7 @@
 Expects seL4-platforms/ to be co-located or otherwise in the PYTHONPATH.
 """
 
-from builds import Build, run_build_script, run_builds, load_builds
+from builds import Build, run_build_script, run_builds, load_builds, sim_script
 from builds import release_mq_locks, SKIP
 from pprint import pprint
 
@@ -32,10 +32,7 @@ def run_build(manifest_dir: str, build: Build):
     ]
 
     if build.req == 'sim':
-        script.append(
-            ["bash", "-c",
-             f"expect -c 'spawn ./simulate; set timeout 3000; expect \"{build.success}\"'"]
-        )
+        script.append(sim_script(build.success))
     else:
         script.append(["tar", "czf", f"../{build.name}-images.tar.gz", "images/"])
 
diff --git a/seL4-platforms/builds.py b/seL4-platforms/builds.py
index d7dab602..ca0df1a3 100644
--- a/seL4-platforms/builds.py
+++ b/seL4-platforms/builds.py
@@ -27,7 +27,8 @@
 
 # exported names:
 __all__ = [
-    "Build", "load_builds", "run_builds", "run_build_script", "junit_results", "sanitise_junit"
+    "Build", "load_builds", "run_builds", "run_build_script", "junit_results", "sanitise_junit",
+    "sim_script"
 ]
 
 # where to expect jUnit results by default
@@ -552,6 +553,14 @@ def summarise_junit(file_path: str) -> Tuple[int, List[str]]:
                   "-q", junit_results, parsed_junit_results]
 
 
+def sim_script(success: str, timeout=1200):
+    return [
+        "expect", "-c",
+        'spawn ./simulate; set timeout %d; expect { "%s" {exit 0} timeout {exit 1} }' %
+        (timeout, success)
+    ]
+
+
 def run_build_script(manifest_dir: str,
                      run: Union[Run, Build],
                      script,
diff --git a/sel4test-sim/build.py b/sel4test-sim/build.py
index fadfbb98..9930f2da 100644
--- a/sel4test-sim/build.py
+++ b/sel4test-sim/build.py
@@ -18,13 +18,13 @@
 def run_simulation(manifest_dir: str, build: Build):
     """Run one simulation build and test."""
 
-    expect = f"\"{build.success}\""
+    expect = '{ "%s" {exit 0} timeout {exit 1} }' % build.success
 
     script = [
         ["../init-build.sh"] + build.settings_args(),
         ["ninja"],
         ["bash", "-c",
-         f"expect -c 'spawn ./simulate; set timeout 3000; expect {expect}' | tee {junit_results}"]
+         f"expect -c 'spawn ./simulate; set timeout 1200; expect {expect}' | tee {junit_results}"]
     ]
 
     return run_build_script(manifest_dir, build, script, junit=True)
diff --git a/webserver/build.py b/webserver/build.py
index 19e83487..946b7906 100644
--- a/webserver/build.py
+++ b/webserver/build.py
@@ -8,7 +8,7 @@
 Expects seL4-platforms/ to be co-located or otherwise in the PYTHONPATH.
 """
 
-from builds import Build, run_build_script, run_builds, load_builds, release_mq_locks, SKIP
+from builds import Build, run_build_script, run_builds, load_builds, release_mq_locks, SKIP, sim_script
 from pprint import pprint
 
 import os
@@ -36,10 +36,7 @@ def run_build(manifest_dir: str, build: Build):
     ]
 
     if plat.has_simulation and plat.name != 'PC99':
-        script.append(
-            ["bash", "-c",
-             f"expect -c 'spawn ./simulate; set timeout 3000; expect \"{build.success}\"'"]
-        )
+        script.append(sim_script(build.success))
 
     return run_build_script(manifest_dir, build, script)