From 890f1f13179e74d4b728599da66466acf0338762 Mon Sep 17 00:00:00 2001 From: htuch Date: Fri, 30 Nov 2018 16:25:13 -0500 Subject: [PATCH] ci: collect failed test logs as artifacts. (#5164) This allows us to avoid relying on bazel output behavior (possibly being eaten in #5104). Turned on for a bunch of builds, but not all. E.g. coverage is skipped as it's a bit more complicated. Risk Level: Low Testing: Tested in PR with CircleCI. Signed-off-by: Harvey Tuch --- .circleci/config.yml | 10 ++++++++++ ci/build_setup.sh | 4 ++++ ci/do_ci.sh | 30 +++++++++++++++++++++++------- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 401052dd3c12..907b42d3a4ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,6 +18,8 @@ jobs: - setup_remote_docker - run: ci/docker_push.sh - run: ci/docker_tag.sh + - store_artifacts: + path: /build/envoy/generated/failed-testlogs asan: executor: ubuntu-build steps: @@ -27,12 +29,16 @@ jobs: - run: curl https://api.github.com/repos/envoyproxy/envoy/pulls/$CIRCLE_PR_NUMBER/commits - checkout - run: ci/do_circle_ci.sh bazel.asan + - store_artifacts: + path: /build/envoy/generated/failed-testlogs tsan: executor: ubuntu-build steps: - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken - checkout - run: ci/do_circle_ci.sh bazel.tsan + - store_artifacts: + path: /build/envoy/generated/failed-testlogs api: executor: ubuntu-build steps: @@ -43,6 +49,8 @@ jobs: fingerprints: - "fb:f3:fe:be:1c:b2:ec:b6:25:f9:7b:a6:87:54:02:8c" - run: ci/api_mirror.sh + - store_artifacts: + path: /build/envoy/generated/failed-testlogs filter_example_mirror: executor: ubuntu-build steps: @@ -72,6 +80,8 @@ jobs: - run: curl -v https://go.googlesource.com - run: curl -6 -v https://go.googlesource.com || true - run: ./ci/do_circle_ci_ipv6_tests.sh + - store_artifacts: + path: /tmp/envoy-docker/envoy/generated/failed-testlogs coverage: executor: ubuntu-build diff --git a/ci/build_setup.sh b/ci/build_setup.sh index b487c25a5efc..3e69b913449e 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -105,6 +105,10 @@ mkdir -p "${ENVOY_DELIVERY_DIR}" export ENVOY_COVERAGE_DIR="${ENVOY_BUILD_DIR}"/generated/coverage mkdir -p "${ENVOY_COVERAGE_DIR}" +# This is where we dump failed test logs for CI collection. +export ENVOY_FAILED_TEST_LOGS="${ENVOY_BUILD_DIR}"/generated/failed-testlogs +mkdir -p "${ENVOY_FAILED_TEST_LOGS}" + # This is where we build for bazel.release* and bazel.dev. export ENVOY_CI_DIR="${ENVOY_SRCDIR}"/ci diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 2c30f8170fdd..f97d8ae10184 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -14,6 +14,22 @@ fi echo "building using ${NUM_CPUS} CPUs" +function bazel_with_collection() { + declare -r BAZEL_OUTPUT="${ENVOY_SRCDIR}"/bazel.output.txt + bazel $* | tee "${BAZEL_OUTPUT}" + declare BAZEL_STATUS="${PIPESTATUS[0]}" + if [ "${BAZEL_STATUS}" != "0" ] + then + declare -r FAILED_TEST_LOGS="$(grep " /build.*test.log" "${BAZEL_OUTPUT}" | sed -e 's/ \/build.*\/testlogs\/\(.*\)/\1/')" + cd bazel-testlogs + for f in ${FAILED_TEST_LOGS} + do + cp --parents -f $f "${ENVOY_FAILED_TEST_LOGS}" + done + exit "${BAZEL_STATUS}" + fi +} + function bazel_release_binary_build() { echo "Building..." cd "${ENVOY_CI_DIR}" @@ -61,14 +77,14 @@ if [[ "$1" == "bazel.release" ]]; then echo "Testing $* ..." # Run only specified tests. Argument can be a single test # (e.g. '//test/common/common:assert_test') or a test group (e.g. '//test/common/...') - bazel test ${BAZEL_TEST_OPTIONS} -c opt $* + bazel_with_collection test ${BAZEL_TEST_OPTIONS} -c opt $* else echo "Testing..." # We have various test binaries in the test directory such as tools, benchmarks, etc. We # run a build pass to make sure they compile. bazel build ${BAZEL_BUILD_OPTIONS} -c opt //include/... //source/... //test/... # Now run all of the tests which should already be compiled. - bazel test ${BAZEL_TEST_OPTIONS} -c opt //test/... + bazel_with_collection test ${BAZEL_TEST_OPTIONS} -c opt //test/... fi exit 0 elif [[ "$1" == "bazel.release.server_only" ]]; then @@ -93,7 +109,7 @@ elif [[ "$1" == "bazel.asan" ]]; then echo "bazel ASAN/UBSAN debug build with tests..." cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" echo "Building and testing..." - bazel test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan @envoy//test/... \ + bazel_with_collection test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan @envoy//test/... \ //:echo2_integration_test //:envoy_binary_test # Also validate that integration test traffic capture (useful when debugging etc.) # works. This requires that we set CAPTURE_ENV. We do this under bazel.asan to @@ -101,7 +117,7 @@ elif [[ "$1" == "bazel.asan" ]]; then CAPTURE_TMP=/tmp/capture/ rm -rf "${CAPTURE_TMP}" mkdir -p "${CAPTURE_TMP}" - bazel test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan \ + bazel_with_collection test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan \ @envoy//test/integration:ssl_integration_test \ --test_env=CAPTURE_PATH="${CAPTURE_TMP}/capture" # Verify that some pb_text files have been created. We can't check for pcap, @@ -114,7 +130,7 @@ elif [[ "$1" == "bazel.tsan" ]]; then echo "bazel TSAN debug build with tests..." cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" echo "Building and testing..." - bazel test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-tsan @envoy//test/... \ + bazel_with_collection test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-tsan @envoy//test/... \ //:echo2_integration_test //:envoy_binary_test exit 0 elif [[ "$1" == "bazel.dev" ]]; then @@ -149,7 +165,7 @@ elif [[ "$1" == "bazel.ipv6_tests" ]]; then setup_clang_toolchain echo "Testing..." cd "${ENVOY_CI_DIR}" - bazel test ${BAZEL_TEST_OPTIONS} --test_env=ENVOY_IP_TEST_VERSIONS=v6only -c fastbuild \ + bazel_with_collection test ${BAZEL_TEST_OPTIONS} --test_env=ENVOY_IP_TEST_VERSIONS=v6only -c fastbuild \ //test/integration/... //test/common/network/... exit 0 elif [[ "$1" == "bazel.api" ]]; then @@ -158,7 +174,7 @@ elif [[ "$1" == "bazel.api" ]]; then echo "Building API..." bazel build ${BAZEL_BUILD_OPTIONS} -c fastbuild @envoy_api//envoy/... echo "Testing API..." - bazel test ${BAZEL_TEST_OPTIONS} -c fastbuild @envoy_api//test/... @envoy_api//tools/... \ + bazel_with_collection test ${BAZEL_TEST_OPTIONS} -c fastbuild @envoy_api//test/... @envoy_api//tools/... \ @envoy_api//tools:capture2pcap_test exit 0 elif [[ "$1" == "bazel.coverage" ]]; then