From 7a0330bb6805933df161508e563d6cf6872f37bc Mon Sep 17 00:00:00 2001 From: Edd Barrett Date: Thu, 9 Jan 2025 09:18:15 +0000 Subject: [PATCH] Undocker benchmark runs. As discussed with @ltratt, since we don't really know the performance implications of docker, and since we plan to use a dedicated machine (with fewer security concerns), run the benchmarks on bare metal. --- .benchmark_inner.sh | 22 --------------------- .buildbot.sh | 2 +- Dockerfile-benchmarking | 20 ------------------- benchmark.sh | 43 +++++++++++++++++++++++++++-------------- common.sh | 20 ++++++++++++------- 5 files changed, 42 insertions(+), 65 deletions(-) delete mode 100644 .benchmark_inner.sh delete mode 100644 Dockerfile-benchmarking diff --git a/.benchmark_inner.sh b/.benchmark_inner.sh deleted file mode 100644 index fbd69ae..0000000 --- a/.benchmark_inner.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh -# -# This script is run inside docker to collect benchmark results. - -set -eu - -# Build everything -. ./common.sh -setup - -# Collect some extra info and put in a TOML file alongside the results file. -EXTRA_TOML=extra.toml -TOML_BIN=~/.local/bin/toml -touch ${EXTRA_TOML} -${TOML_BIN} add_section --toml-path ${EXTRA_TOML} versions -${TOML_BIN} set --toml-path ${EXTRA_TOML} versions.yk-benchmarks "$(git rev-parse HEAD)" -${TOML_BIN} set --toml-path ${EXTRA_TOML} versions.yk "$(cd yk && git rev-parse HEAD)" -${TOML_BIN} set --toml-path ${EXTRA_TOML} versions.ykllvm "$(cd yk/ykllvm && git rev-parse HEAD)" -${TOML_BIN} set --toml-path ${EXTRA_TOML} versions.yklua "$(cd yklua && git rev-parse HEAD)" - -# Run benchmarks. -~/.local/bin/rebench --no-denoise -c rebench.conf diff --git a/.buildbot.sh b/.buildbot.sh index bbdde40..44e50d6 100644 --- a/.buildbot.sh +++ b/.buildbot.sh @@ -15,7 +15,7 @@ setup sed -e 's/invocations: [0-9]\+/invocations: 2/g' \ -e 's/iterations: [0-9]\+/iterations: 1/g' \ rebench.conf > rebench_ci.conf -~/.local/bin/rebench --no-denoise -c rebench_ci.conf +venv/bin/rebench --no-denoise -c rebench_ci.conf # Do some minimal checks on the reporter. cd reporter diff --git a/Dockerfile-benchmarking b/Dockerfile-benchmarking deleted file mode 100644 index 539aed8..0000000 --- a/Dockerfile-benchmarking +++ /dev/null @@ -1,20 +0,0 @@ -FROM debian:latest -WORKDIR /bm -RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ - --mount=target=/var/cache/apt,type=cache,sharing=locked \ - rm -f /etc/apt/apt.conf.d/docker-clean && \ - apt update && apt install -y pipx git ninja-build libtinfo-dev \ - libzip-dev mold gdb clang-15 curl cmake && \ - update-alternatives --install /usr/bin/cc cc /usr/bin/clang-15 999 && \ - update-alternatives --set cc /usr/bin/clang-15 && \ - update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-15 999 && \ - update-alternatives --set c++ /usr/bin/clang++-15 && \ - update-alternatives --install /usr/bin/ld ld /usr/bin/mold 999 && \ - update-alternatives --set ld /usr/bin/mold && \ - ln -sf /usr/bin/clang-15 /usr/bin/clang && \ - ln -sf /usr/bin/clang++-15 /usr/bin/clang++ -ARG BM_UID -RUN useradd -m -u ${BM_UID} ci && chown ${BM_UID}:${BM_UID} . -RUN chown ${BM_UID}:${BM_UID} . -COPY --chown=${BM_UID}:${BM_UID} . . -CMD ["sh", "-x", ".benchmark_inner.sh"] diff --git a/benchmark.sh b/benchmark.sh index 3551285..d3bd574 100644 --- a/benchmark.sh +++ b/benchmark.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Run a benchmarking session inside a docker container and file away the +# Run a benchmarking session inside a temporary directory and file away the # results file. set -eu @@ -47,21 +47,34 @@ RES_DIR=$1; shift RES_SUBDIR="${RES_DIR}/$(${FMT_EPOCH} ${TIMESTAMP} +%Y%m)" mkdir -p ${RES_SUBDIR} -# Run benchmarks inside docker. +# Set up a fresh time-stamped directory. YMDHMS=$(${FMT_EPOCH} ${TIMESTAMP} +%Y%m%d_%H%M%S) -IMAGE_TAG="bm-${YMDHMS}" -CONT_NAME=$(pwgen -s 16 1) -BM_UID=$(id -u) -docker build --build-arg BM_UID=${BM_UID} -t ${IMAGE_TAG} --file Dockerfile-benchmarking . -docker run --cap-add CAP_PERFMON -u ${BM_UID} --name ${CONT_NAME} ${IMAGE_TAG} +RUN_DIR="${PWD}/runs/${YMDHMS}" +mkdir -p ${RUN_DIR} +cleanup() { + rm -rf ${RUN_DIR} +} +trap 'cleanup' EXIT + +. ./common.sh +cd ${RUN_DIR} +setup +ln -s ../../rebench.conf . +ln -s ../../suites . -# Stash extra info file. -docker container cp ${CONT_NAME}:/bm/extra.toml \ - ${RES_SUBDIR}/${YMDHMS}-extra.toml +# Collect some extra info and put in a TOML file alongside the results file. +EXTRA_TOML=extra.toml +TOML_BIN=venv/bin/toml +touch ${EXTRA_TOML} +${TOML_BIN} add_section --toml-path ${EXTRA_TOML} versions +${TOML_BIN} set --toml-path ${EXTRA_TOML} versions.yk-benchmarks "$(git rev-parse HEAD)" +${TOML_BIN} set --toml-path ${EXTRA_TOML} versions.yk "$(cd yk && git rev-parse HEAD)" +${TOML_BIN} set --toml-path ${EXTRA_TOML} versions.ykllvm "$(cd yk/ykllvm && git rev-parse HEAD)" +${TOML_BIN} set --toml-path ${EXTRA_TOML} versions.yklua "$(cd yklua && git rev-parse HEAD)" -# File away the results file in the output directory. -docker container cp ${CONT_NAME}:/bm/benchmark.data \ - ${RES_SUBDIR}/${YMDHMS}.data +# Run benchmarks. +venv/bin/rebench -q --no-denoise -c rebench.conf -# Remove the container. -docker container rm ${CONT_NAME} +# File away the results file (and extra info file) in the output directory. +cp ${EXTRA_TOML} ../../${RES_SUBDIR}/${YMDHMS}-extra.toml +cp benchmark.data ../../${RES_SUBDIR}/${YMDHMS}.data diff --git a/common.sh b/common.sh index c71996d..45d9c23 100644 --- a/common.sh +++ b/common.sh @@ -1,16 +1,21 @@ # Common shared functionality. # Build/install everything required for benchmarking. +# +# The python bits are installed inside a virtual env in a directory +# `$(PWD)/venv`. setup() { # Install Rust + export CARGO_HOME="$(pwd)/.cargo" + export RUSTUP_HOME="$(pwd)/.rustup" export RUSTUP_INIT_SKIP_PATH_CHECK="yes" curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup.sh sh rustup.sh --default-host x86_64-unknown-linux-gnu \ - --default-toolchain nightly \ - --no-modify-path \ - --profile default \ - -y - export PATH=~/.cargo/bin/:${PATH} + --default-toolchain nightly \ + --no-modify-path \ + --profile default \ + -y + export PATH=${PWD}/.cargo/bin/:${PATH} # Set up yk. git clone --recurse-submodules https://github.com/ykjit/yk @@ -35,6 +40,7 @@ setup() { make -j $(nproc) cd .. - pipx install rebench - pipx install toml-cli + python3 -m venv venv + ./venv/bin/pip install rebench + ./venv/bin/pip install toml-cli }