From 5ba6cdba81aa776aa718ff3f13f461a91c8b52a7 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 27 Nov 2023 17:17:31 +0000 Subject: [PATCH] platform-checks: Also save services.log before calling c.rm Currently services.log is empty in platform-checks in successful runs, as noticed by Nikhil in https://materializeinc.slack.com/archives/C01CFKM1QRF/p1701104667677569?thread_ts=1701087470.941589&cid=C01CFKM1QRF --- ci/plugins/mzcompose/hooks/post-command | 13 +++++++++---- misc/python/materialize/mzcompose/composition.py | 14 ++++++++------ test/cluster-isolation/mzcompose.py | 2 +- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/ci/plugins/mzcompose/hooks/post-command b/ci/plugins/mzcompose/hooks/post-command index 7ac1ed82edecd..95eacef495e30 100755 --- a/ci/plugins/mzcompose/hooks/post-command +++ b/ci/plugins/mzcompose/hooks/post-command @@ -91,11 +91,11 @@ find cores -name 'core.*' | while read -r core; do exe=$(echo "$core" | sed -e "s/core\.\(.*\)\.[0-9]*/\1/" -e "s/.*\!//") bin/ci-builder run stable gdb --batch -ex "bt full" -ex "thread apply all bt" -ex "quit" cores/"$exe" "$core" > "$core".txt || true if grep -q "Program terminated with signal SIGABRT, Aborted." "$core".txt; then - echo "SIGABRT found in \"$core.txt\", ignoring core file" + echo "SIGABRT found in \"$core.txt\", ignoring core file" else - zstd --rm "$core" - buildkite-agent artifact upload "$core".txt - buildkite-agent artifact upload "$core".zst + zstd --rm "$core" + buildkite-agent artifact upload "$core".txt + buildkite-agent artifact upload "$core".zst fi done # can be huge, clean up @@ -107,3 +107,8 @@ artifacts=(run.log services.log journalctl-merge.log netstat-ant.log netstat-pan artifacts_str=$(IFS=";"; echo "${artifacts[*]}") buildkite-agent artifact upload "$artifacts_str" bin/ci-builder run stable bin/ci-logged-errors-detect "${artifacts[@]}" + +if [ ! -s services.log ] && [ "$BUILDKITE_STEP_KEY" != "persist-maelstrom" ]; then + echo "services.log is empty" + exit 1 +fi diff --git a/misc/python/materialize/mzcompose/composition.py b/misc/python/materialize/mzcompose/composition.py index e47e9a1bccb11..9a7b312c265b7 100644 --- a/misc/python/materialize/mzcompose/composition.py +++ b/misc/python/materialize/mzcompose/composition.py @@ -778,6 +778,12 @@ def sanity_restart_mz(self) -> None: "Sanity Restart skipped because Mz not in services or `sanity_restart` label not set" ) + def capture_logs(self) -> None: + # Capture logs into services.log since they will be lost otherwise + # after dowing a composition. + with open(MZ_ROOT / "services.log", "a") as f: + self.invoke("logs", "--no-color", capture=f) + def down( self, destroy_volumes: bool = True, @@ -796,12 +802,7 @@ def down( """ if sanity_restart_mz: self.sanity_restart_mz() - - # Capture logs into services.log since they will be lost otherwise - # after dowing a composition. - with open(MZ_ROOT / "services.log", "a") as f: - self.invoke("logs", "--no-color", capture=f) - + self.capture_logs() self.invoke( "down", *(["--volumes"] if destroy_volumes else []), @@ -863,6 +864,7 @@ def rm( service. Note that this does not destroy any named volumes attached to the service. """ + self.capture_logs() self.invoke( "rm", "--force", diff --git a/test/cluster-isolation/mzcompose.py b/test/cluster-isolation/mzcompose.py index 7cec507d46974..83d3ddad6c302 100644 --- a/test/cluster-isolation/mzcompose.py +++ b/test/cluster-isolation/mzcompose.py @@ -78,7 +78,7 @@ class Disruption: > CREATE TABLE panic_table (f1 TEXT); -> INSERT INTO panic_table VALUES ('panic!'); +> INSERT INTO panic_table VALUES ('forced panic'); ! INSERT INTO panic_table SELECT mz_internal.mz_panic(f1) FROM panic_table; contains: statement timeout