Merge pull request #23464 from def-/pr-checks-services

platform-checks: Also save services.log before calling c.rm
MaterializeInc · Nov 28, 2023 · 099649a · 099649a
2 parents 1e06067 + 5ba6cdb
commit 099649a
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 11 deletions.
diff --git a/ci/plugins/mzcompose/hooks/post-command b/ci/plugins/mzcompose/hooks/post-command
@@ -91,11 +91,11 @@ find cores -name 'core.*' | while read -r core; do
     exe=$(echo "$core" | sed -e "s/core\.\(.*\)\.[0-9]*/\1/" -e "s/.*\!//")
     bin/ci-builder run stable gdb --batch -ex "bt full" -ex "thread apply all bt" -ex "quit" cores/"$exe" "$core" > "$core".txt || true
     if grep -q "Program terminated with signal SIGABRT, Aborted." "$core".txt; then
-      echo "SIGABRT found in \"$core.txt\", ignoring core file"
+        echo "SIGABRT found in \"$core.txt\", ignoring core file"
     else
-      zstd --rm "$core"
-      buildkite-agent artifact upload "$core".txt
-      buildkite-agent artifact upload "$core".zst
+        zstd --rm "$core"
+        buildkite-agent artifact upload "$core".txt
+        buildkite-agent artifact upload "$core".zst
     fi
 done
 # can be huge, clean up
@@ -107,3 +107,8 @@ artifacts=(run.log services.log journalctl-merge.log netstat-ant.log netstat-pan
 artifacts_str=$(IFS=";"; echo "${artifacts[*]}")
 buildkite-agent artifact upload "$artifacts_str"
 bin/ci-builder run stable bin/ci-logged-errors-detect "${artifacts[@]}"
+
+if [ ! -s services.log ] && [ "$BUILDKITE_STEP_KEY" != "persist-maelstrom" ]; then
+    echo "services.log is empty"
+    exit 1
+fi
diff --git a/misc/python/materialize/mzcompose/composition.py b/misc/python/materialize/mzcompose/composition.py
@@ -778,6 +778,12 @@ def sanity_restart_mz(self) -> None:
                 "Sanity Restart skipped because Mz not in services or `sanity_restart` label not set"
             )
 
+    def capture_logs(self) -> None:
+        # Capture logs into services.log since they will be lost otherwise
+        # after dowing a composition.
+        with open(MZ_ROOT / "services.log", "a") as f:
+            self.invoke("logs", "--no-color", capture=f)
+
     def down(
         self,
         destroy_volumes: bool = True,
@@ -796,12 +802,7 @@ def down(
         """
         if sanity_restart_mz:
             self.sanity_restart_mz()
-
-        # Capture logs into services.log since they will be lost otherwise
-        # after dowing a composition.
-        with open(MZ_ROOT / "services.log", "a") as f:
-            self.invoke("logs", "--no-color", capture=f)
-
+        self.capture_logs()
         self.invoke(
             "down",
             *(["--volumes"] if destroy_volumes else []),
@@ -863,6 +864,7 @@ def rm(
                 service. Note that this does not destroy any named volumes
                 attached to the service.
         """
+        self.capture_logs()
         self.invoke(
             "rm",
             "--force",

diff --git a/test/cluster-isolation/mzcompose.py b/test/cluster-isolation/mzcompose.py
@@ -78,7 +78,7 @@ class Disruption:
 
 > CREATE TABLE panic_table (f1 TEXT);
 
-> INSERT INTO panic_table VALUES ('panic!');
+> INSERT INTO panic_table VALUES ('forced panic');
 
 ! INSERT INTO panic_table SELECT mz_internal.mz_panic(f1) FROM panic_table;
 contains: statement timeout