From 70bf755f9e62e4bb373e5082876a37b2a89abbc9 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 31 Oct 2022 10:54:53 -0400
Subject: [PATCH 01/14] script: reorganize build scripts

In preparation for running builds and tests in containers, make some
organizational changes to the run-make-*.sh scripts.

Original version of the patch can be found at https://github.com/ceph/ceph/pull/46071

Original-version-by: Ernesto Puerta <epuertat@redhat.com>
Co-authored-by: Ernesto Puerta <epuertat@redhat.com>
Co-authored-by: John Mulligan <jmulligan@redhat.com>
Signed-off-by: Ernesto Puerta <epuertat@redhat.com>
Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 run-make-check.sh      | 46 +++++------------------------
 src/script/run-make.sh | 66 ++++++++++++++++++++++++++++++++----------
 2 files changed, 57 insertions(+), 55 deletions(-)

diff --git a/run-make-check.sh b/run-make-check.sh
index 6274303873985..f923f955cd0c0 100755
--- a/run-make-check.sh
+++ b/run-make-check.sh
@@ -22,10 +22,6 @@ source src/script/run-make.sh
 
 set -e
 
-function in_jenkins() {
-    test -n "$JENKINS_HOME"
-}
-
 function run() {
     # to prevent OSD EMFILE death on tests, make sure ulimit >= 1024
     $DRY_RUN ulimit -n $(ulimit -Hn)
@@ -40,7 +36,9 @@ function run() {
     if [ $(nproc) -gt $m ]; then
         m=$(nproc)
     fi
-    $DRY_RUN sudo /sbin/sysctl -q -w fs.aio-max-nr=$((65536 * $(nproc)))
+    if [ "$(/sbin/sysctl -n fs.aio-max-nr )" -lt "$((65536 * $(nproc)))" ]; then
+        $DRY_RUN sudo /sbin/sysctl -q -w fs.aio-max-nr=$((65536 * $(nproc)))
+    fi
 
     CHECK_MAKEOPTS=${CHECK_MAKEOPTS:-$DEFAULT_MAKEOPTS}
     if in_jenkins; then
@@ -71,43 +69,13 @@ function main() {
     fi
     # uses run-make.sh to install-deps
     FOR_MAKE_CHECK=1 prepare
-    local cxx_compiler=g++
-    local c_compiler=gcc
-    for i in $(seq 14 -1 10); do
-        if type -t clang-$i > /dev/null; then
-            cxx_compiler="clang++-$i"
-            c_compiler="clang-$i"
-            break
-        fi
-    done
-    # Init defaults after deps are installed.
-    local cmake_opts
-    cmake_opts+=" -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_C_COMPILER=$c_compiler"
-    cmake_opts+=" -DCMAKE_CXX_FLAGS_DEBUG=-Werror"
-    cmake_opts+=" -DENABLE_GIT_VERSION=OFF"
-    cmake_opts+=" -DWITH_GTEST_PARALLEL=ON"
-    cmake_opts+=" -DWITH_FIO=ON"
-    cmake_opts+=" -DWITH_CEPHFS_SHELL=ON"
-    cmake_opts+=" -DWITH_GRAFANA=ON"
-    cmake_opts+=" -DWITH_SPDK=ON"
-    cmake_opts+=" -DWITH_RBD_MIRROR=ON"
-    if [ $WITH_SEASTAR ]; then
-        cmake_opts+=" -DWITH_SEASTAR=ON"
-    fi
-    if [ $WITH_ZBD ]; then
-        cmake_opts+=" -DWITH_ZBD=ON"
-    fi
-    if [ $WITH_RBD_RWL ]; then
-        cmake_opts+=" -DWITH_RBD_RWL=ON"
-    fi
-    cmake_opts+=" -DWITH_RBD_SSD_CACHE=ON"
-    in_jenkins && echo "CI_DEBUG: Our cmake_opts are: $cmake_opts
-                        CI_DEBUG: Running ./configure"
-    configure "$cmake_opts" "$@"
+    configure "$@"
     in_jenkins && echo "CI_DEBUG: Running 'build tests'"
     build tests
     echo "make check: successful build on $(git rev-parse HEAD)"
     FOR_MAKE_CHECK=1 run
 }
 
-main "$@"
+if [ "$0" = "$BASH_SOURCE" ]; then
+    main "$@"
+fi
diff --git a/src/script/run-make.sh b/src/script/run-make.sh
index 683272666a9d9..c746cf752b7e8 100755
--- a/src/script/run-make.sh
+++ b/src/script/run-make.sh
@@ -67,7 +67,11 @@ function do_install() {
     pkgs=$@
     shift
     ret=0
-    $DRY_RUN sudo $install_cmd $pkgs || ret=$?
+    SUDO=""
+    if [ "$EUID" -ne 0 ]; then
+        SUDO="sudo"
+    fi
+    $DRY_RUN $SUDO $install_cmd $pkgs || ret=$?
     if test $ret -eq 0 ; then
         return
     fi
@@ -75,9 +79,9 @@ function do_install() {
     if [[ $install_cmd == *"apt-get"* ]]; then
         if test $ret -eq 100 ; then
             # dpkg was interrupted
-            $DRY_RUN sudo dpkg --configure -a
-            in_jenkins && echo "CI_DEBUG: Running 'sudo $install_cmd $pkgs'"
-            $DRY_RUN sudo $install_cmd $pkgs
+            $DRY_RUN $SUDO dpkg --configure -a
+            in_jenkins && echo "CI_DEBUG: Running '$SUDO $install_cmd $pkgs'"
+            $DRY_RUN $SUDO $install_cmd $pkgs
         else
             return $ret
         fi
@@ -88,11 +92,7 @@ function prepare() {
     local which_pkg="which"
     source /etc/os-release
     if test -f /etc/redhat-release ; then
-        if ! type bc > /dev/null 2>&1 ; then
-            echo "Please install bc and re-run." 
-            exit 1
-        fi
-        if test "$(echo "$VERSION_ID >= 22" | bc)" -ne 0; then
+        if [ "$VERSION_ID" -ge "22" ]; then
             install_cmd="dnf -y install"
         else
             install_cmd="yum install -y"
@@ -104,14 +104,14 @@ function prepare() {
         which_pkg="debianutils"
     fi
 
-    if ! type sudo > /dev/null 2>&1 ; then
+    if [ "$EUID" -ne 0 ] && ! type sudo > /dev/null 2>&1 ; then
         echo "Please install sudo and re-run. This script assumes it is running"
-        echo "as a normal user with the ability to run commands as root via sudo." 
+        echo "as a normal user with the ability to run commands as root via sudo."
         exit 1
     fi
     if [ -n "$install_cmd" ]; then
         in_jenkins && echo "CI_DEBUG: Running '$install_cmd ccache $which_pkg clang'"
-        do_install "$install_cmd" ccache $which_pkg clang
+        do_install "$install_cmd" ccache git $which_pkg clang
     else
         echo "WARNING: Don't know how to install packages" >&2
         echo "This probably means distribution $ID is not supported by run-make-check.sh" >&2
@@ -127,7 +127,9 @@ function prepare() {
         $DRY_RUN source ./install-deps.sh || return 1
         trap clean_up_after_myself EXIT
     fi
+}
 
+function configure() {
     cat <<EOM
 Note that the binaries produced by this script do not contain correct time
 and git version information, which may make them unsuitable for debugging
@@ -148,12 +150,44 @@ EOM
         ccache -p | grep max_size
     fi
     $DRY_RUN ccache -sz # Reset the ccache statistics and show the current configuration
-}
 
-function configure() {
-    local cmake_build_opts=$(detect_ceph_dev_pkgs)
+    local cxx_compiler=g++
+    local c_compiler=gcc
+    for i in $(seq 14 -1 10); do
+        if type -t clang-$i > /dev/null; then
+            cxx_compiler="clang++-$i"
+            c_compiler="clang-$i"
+            break
+        fi
+    done
+    local cmake_opts
+    cmake_opts+=" -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_C_COMPILER=$c_compiler"
+    cmake_opts+=" -DCMAKE_CXX_FLAGS_DEBUG=-Werror"
+    cmake_opts+=" -DENABLE_GIT_VERSION=OFF"
+    cmake_opts+=" -DWITH_GTEST_PARALLEL=ON"
+    cmake_opts+=" -DWITH_FIO=ON"
+    cmake_opts+=" -DWITH_CEPHFS_SHELL=ON"
+    cmake_opts+=" -DWITH_GRAFANA=ON"
+    cmake_opts+=" -DWITH_SPDK=ON"
+    cmake_opts+=" -DWITH_RBD_MIRROR=ON"
+    if [ $WITH_SEASTAR ]; then
+        cmake_opts+=" -DWITH_SEASTAR=ON"
+    fi
+    if [ $WITH_ZBD ]; then
+        cmake_opts+=" -DWITH_ZBD=ON"
+    fi
+    if [ $WITH_RBD_RWL ]; then
+        cmake_opts+=" -DWITH_RBD_RWL=ON"
+    fi
+    cmake_opts+=" -DWITH_RBD_SSD_CACHE=ON"
+
+    cmake_opts+=$(detect_ceph_dev_pkgs)
+
+    in_jenkins && echo "CI_DEBUG: Our cmake_opts are: $cmake_opts
+                        CI_DEBUG: Running ./configure"
     in_jenkins && echo "CI_DEBUG: Running do_cmake.sh"
-    $DRY_RUN ./do_cmake.sh $cmake_build_opts $@ || return 1
+
+    $DRY_RUN ./do_cmake.sh $cmake_opts $@ || return 1
 }
 
 function build() {

From 974effaa95ddbc9816a37e382a96e691f4e3cbdd Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 6 Oct 2022 13:43:41 -0400
Subject: [PATCH 02/14] script: have run-make.sh honor BUILD_DIR like
 do_cmake.sh does

The BUILD_DIR environment variable is honored by do_cmake.sh in order to
create multiple build output directories. Before this change run-make.sh
did not support BUILD_DIR the same way as do_cmake.sh. This change makes
it possible to use BUILD_DIR with run-make.sh.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/script/run-make.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/script/run-make.sh b/src/script/run-make.sh
index c746cf752b7e8..d2ccf1fbf4d54 100755
--- a/src/script/run-make.sh
+++ b/src/script/run-make.sh
@@ -195,7 +195,11 @@ function build() {
     if test -n "$targets"; then
         targets="--target $targets"
     fi
-    $DRY_RUN cd build
+    local bdir=build
+    if [ "$BUILD_DIR" ]; then
+        bdir="$BUILD_DIR"
+    fi
+    $DRY_RUN cd "${bdir}"
     BUILD_MAKEOPTS=${BUILD_MAKEOPTS:-$DEFAULT_MAKEOPTS}
     test "$BUILD_MAKEOPTS" && echo "make will run with option(s) $BUILD_MAKEOPTS"
     # older cmake does not support --parallel or -j, so pass it to underlying generator

From b315700bbfd64a22b62e27631e055d1d2392e447 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Thu, 29 Sep 2022 10:34:12 -0400
Subject: [PATCH 03/14] install-deps.sh: move functions above all "main" script
 body

Previously, the main part (top level body) of the script started and
then some function definitions occurred and then the main part of the
script resumed after that. I, and others, find this confusing so this
change moves the function definitions to occur before the main body of
the install-deps.sh script.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 install-deps.sh | 122 ++++++++++++++++++++++++------------------------
 1 file changed, 61 insertions(+), 61 deletions(-)

diff --git a/install-deps.sh b/install-deps.sh
index 3762b3de03565..64486ba51b74e 100755
--- a/install-deps.sh
+++ b/install-deps.sh
@@ -253,6 +253,67 @@ EOF
     fi
 }
 
+function populate_wheelhouse() {
+    in_jenkins && echo "CI_DEBUG: Running populate_wheelhouse() in install-deps.sh"
+    local install=$1
+    shift
+
+    # although pip comes with virtualenv, having a recent version
+    # of pip matters when it comes to using wheel packages
+    PIP_OPTS="--timeout 300 --exists-action i"
+    pip $PIP_OPTS $install \
+      'setuptools >= 0.8' 'pip >= 21.0' 'wheel >= 0.24' 'tox >= 2.9.1' || return 1
+    if test $# != 0 ; then
+        # '--use-feature=fast-deps --use-deprecated=legacy-resolver' added per
+        # https://github.com/pypa/pip/issues/9818 These should be able to be
+        # removed at some point in the future.
+        pip --use-feature=fast-deps --use-deprecated=legacy-resolver $PIP_OPTS $install $@ || return 1
+    fi
+}
+
+function activate_virtualenv() {
+    in_jenkins && echo "CI_DEBUG: Running activate_virtualenv() in install-deps.sh"
+    local top_srcdir=$1
+    local env_dir=$top_srcdir/install-deps-python3
+
+    if ! test -d $env_dir ; then
+        python3 -m venv ${env_dir}
+        . $env_dir/bin/activate
+        if ! populate_wheelhouse install ; then
+            rm -rf $env_dir
+            return 1
+        fi
+    fi
+    . $env_dir/bin/activate
+}
+
+function preload_wheels_for_tox() {
+    in_jenkins && echo "CI_DEBUG: Running preload_wheels_for_tox() in install-deps.sh"
+    local ini=$1
+    shift
+    pushd . > /dev/null
+    cd $(dirname $ini)
+    local require_files=$(ls *requirements*.txt 2>/dev/null) || true
+    local constraint_files=$(ls *constraints*.txt 2>/dev/null) || true
+    local require=$(echo -n "$require_files" | sed -e 's/^/-r /')
+    local constraint=$(echo -n "$constraint_files" | sed -e 's/^/-c /')
+    local md5=wheelhouse/md5
+    if test "$require"; then
+        if ! test -f $md5 || ! md5sum -c $md5 > /dev/null; then
+            rm -rf wheelhouse
+        fi
+    fi
+    if test "$require" && ! test -d wheelhouse ; then
+        type python3 > /dev/null 2>&1 || continue
+        activate_virtualenv $top_srcdir || exit 1
+        python3 -m pip install --upgrade pip
+        populate_wheelhouse "wheel -w $wip_wheelhouse" $require $constraint || exit 1
+        mv $wip_wheelhouse wheelhouse
+        md5sum $require_files $constraint_files > $md5
+    fi
+    popd > /dev/null
+}
+
 for_make_check=false
 if tty -s; then
     # interactive
@@ -465,67 +526,6 @@ EOF
     esac
 fi
 
-function populate_wheelhouse() {
-    in_jenkins && echo "CI_DEBUG: Running populate_wheelhouse() in install-deps.sh"
-    local install=$1
-    shift
-
-    # although pip comes with virtualenv, having a recent version
-    # of pip matters when it comes to using wheel packages
-    PIP_OPTS="--timeout 300 --exists-action i"
-    pip $PIP_OPTS $install \
-      'setuptools >= 0.8' 'pip >= 21.0' 'wheel >= 0.24' 'tox >= 2.9.1' || return 1
-    if test $# != 0 ; then
-        # '--use-feature=fast-deps --use-deprecated=legacy-resolver' added per
-        # https://github.com/pypa/pip/issues/9818 These should be able to be
-        # removed at some point in the future.
-        pip --use-feature=fast-deps --use-deprecated=legacy-resolver $PIP_OPTS $install $@ || return 1
-    fi
-}
-
-function activate_virtualenv() {
-    in_jenkins && echo "CI_DEBUG: Running activate_virtualenv() in install-deps.sh"
-    local top_srcdir=$1
-    local env_dir=$top_srcdir/install-deps-python3
-
-    if ! test -d $env_dir ; then
-        python3 -m venv ${env_dir}
-        . $env_dir/bin/activate
-        if ! populate_wheelhouse install ; then
-            rm -rf $env_dir
-            return 1
-        fi
-    fi
-    . $env_dir/bin/activate
-}
-
-function preload_wheels_for_tox() {
-    in_jenkins && echo "CI_DEBUG: Running preload_wheels_for_tox() in install-deps.sh"
-    local ini=$1
-    shift
-    pushd . > /dev/null
-    cd $(dirname $ini)
-    local require_files=$(ls *requirements*.txt 2>/dev/null) || true
-    local constraint_files=$(ls *constraints*.txt 2>/dev/null) || true
-    local require=$(echo -n "$require_files" | sed -e 's/^/-r /')
-    local constraint=$(echo -n "$constraint_files" | sed -e 's/^/-c /')
-    local md5=wheelhouse/md5
-    if test "$require"; then
-        if ! test -f $md5 || ! md5sum -c $md5 > /dev/null; then
-            rm -rf wheelhouse
-        fi
-    fi
-    if test "$require" && ! test -d wheelhouse ; then
-        type python3 > /dev/null 2>&1 || continue
-        activate_virtualenv $top_srcdir || exit 1
-        python3 -m pip install --upgrade pip
-        populate_wheelhouse "wheel -w $wip_wheelhouse" $require $constraint || exit 1
-        mv $wip_wheelhouse wheelhouse
-        md5sum $require_files $constraint_files > $md5
-    fi
-    popd > /dev/null
-}
-
 # use pip cache if possible but do not store it outside of the source
 # tree
 # see https://pip.pypa.io/en/stable/reference/pip_install.html#caching

From 8b529a0d416acd9b1fd102fd384ab0b64736e66c Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 3 Oct 2022 14:43:19 -0400
Subject: [PATCH 04/14] install-deps.sh: support INSTALL_EXTRA_PKGS

Instead of requiring other scripts to install packages independently,
teach install-deps.sh to install additional packages from the variable
INSTALL_EXTRA_PKGS. Now, other scripts should just set
INSTALL_EXTRA_PKGS and call install-deps.sh.

In particular, this fixes an issue installing packages in a clean (ex.
container) system that doesn't yet have repositories set up. Since this
task is performed by install-deps.sh already we avoid a chicken-and-egg
issue (or doing redundant work of setting up repos) in other scripts.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 install-deps.sh | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/install-deps.sh b/install-deps.sh
index 64486ba51b74e..703946a573218 100755
--- a/install-deps.sh
+++ b/install-deps.sh
@@ -325,6 +325,10 @@ else
 fi
 
 if [ x$(uname)x = xFreeBSDx ]; then
+    if [ "$INSTALL_EXTRA_PACKAGES" ]; then
+        echo "Installing extra packages not supported on FreeBSD" >&2
+        exit 1
+    fi
     $SUDO pkg install -yq \
         devel/babeltrace \
         devel/binutils \
@@ -390,6 +394,9 @@ else
     case "$ID" in
     debian|ubuntu|devuan|elementary|softiron)
         echo "Using apt-get to install dependencies"
+        if [ "$INSTALL_EXTRA_PACKAGES" ]; then
+            $SUDO apt-get install -y $INSTALL_EXTRA_PACKAGES
+        fi
         $SUDO apt-get install -y devscripts equivs
         $SUDO apt-get install -y dpkg-dev
         ensure_python3_sphinx_on_ubuntu
@@ -493,6 +500,9 @@ EOF
                 fi
                 ;;
         esac
+        if [ "$INSTALL_EXTRA_PACKAGES" ]; then
+            $SUDO dnf install -y $INSTALL_EXTRA_PACKAGES
+        fi
         munge_ceph_spec_in $with_seastar $with_zbd $for_make_check $DIR/ceph.spec
         # for python3_pkgversion macro defined by python-srpm-macros, which is required by python3-devel
         $SUDO dnf install -y python3-devel
@@ -516,6 +526,9 @@ EOF
         echo "Using zypper to install dependencies"
         zypp_install="zypper --gpg-auto-import-keys --non-interactive install --no-recommends"
         $SUDO $zypp_install systemd-rpm-macros rpm-build || exit 1
+        if [ "$INSTALL_EXTRA_PACKAGES" ]; then
+            $SUDO $zypp_install $INSTALL_EXTRA_PACKAGES
+        fi
         munge_ceph_spec_in $with_seastar false $for_make_check $DIR/ceph.spec
         $SUDO $zypp_install $(rpmspec -q --buildrequires $DIR/ceph.spec) || exit 1
         ;;

From e8b370a9c7dd465345d5dc62513d635339a150b3 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 3 Oct 2022 15:08:30 -0400
Subject: [PATCH 05/14] install-deps.sh: copy ubuntu/apt retry logic from
 run-make.sh

Copy the logic from run-make.sh into install-deps.sh so that we can later
remove it from run-make.sh. It helps prevent breakage when apt-get is
interrupted.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 install-deps.sh | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/install-deps.sh b/install-deps.sh
index 703946a573218..22be8ed1b422e 100755
--- a/install-deps.sh
+++ b/install-deps.sh
@@ -395,7 +395,16 @@ else
     debian|ubuntu|devuan|elementary|softiron)
         echo "Using apt-get to install dependencies"
         if [ "$INSTALL_EXTRA_PACKAGES" ]; then
-            $SUDO apt-get install -y $INSTALL_EXTRA_PACKAGES
+            if ! $SUDO apt-get install -y $INSTALL_EXTRA_PACKAGES ; then
+                # try again. ported over from run-make.sh (orignally e278295)
+                # In the case that apt-get is interrupted, like when a jenkins
+                # job is cancelled, the package manager will be in an inconsistent
+                # state. Run the command again after `dpkg --configure -a` to
+                # bring package manager back into a clean state.
+                $SUDO dpkg --configure -a
+                in_jenkins && echo "CI_DEBUG: trying to install $INSTALL_EXTRA_PACKAGES again"
+                $SUDO apt-get install -y $INSTALL_EXTRA_PACKAGES
+            fi
         fi
         $SUDO apt-get install -y devscripts equivs
         $SUDO apt-get install -y dpkg-dev

From 5f92f545c67efa38e1c977b1c87281abad5ec748 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Wed, 5 Oct 2022 10:19:32 -0400
Subject: [PATCH 06/14] script: use install-deps.sh to install extra packages
 wanted by run-make.sh

The run-make.sh script's prepare method pulls in additional dependencies
that are needed by the CI build and tests. To avoid issues such as these
packages not being available until after install-deps.sh being run in a
container environment, we allow install-deps.sh and its new
INSTALL_EXTRA_PKGS input variable to handle all of the dependency
installation.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/script/run-make.sh | 64 ++++--------------------------------------
 1 file changed, 6 insertions(+), 58 deletions(-)

diff --git a/src/script/run-make.sh b/src/script/run-make.sh
index d2ccf1fbf4d54..4b9a17b790695 100755
--- a/src/script/run-make.sh
+++ b/src/script/run-make.sh
@@ -58,75 +58,23 @@ function detect_ceph_dev_pkgs() {
     echo "$cmake_opts"
 }
 
-function do_install() {
-    local install_cmd
-    local pkgs
-    local ret
-    install_cmd=$1
-    shift
-    pkgs=$@
-    shift
-    ret=0
-    SUDO=""
-    if [ "$EUID" -ne 0 ]; then
-        SUDO="sudo"
-    fi
-    $DRY_RUN $SUDO $install_cmd $pkgs || ret=$?
-    if test $ret -eq 0 ; then
-        return
-    fi
-    # try harder if apt-get, and it was interrutped
-    if [[ $install_cmd == *"apt-get"* ]]; then
-        if test $ret -eq 100 ; then
-            # dpkg was interrupted
-            $DRY_RUN $SUDO dpkg --configure -a
-            in_jenkins && echo "CI_DEBUG: Running '$SUDO $install_cmd $pkgs'"
-            $DRY_RUN $SUDO $install_cmd $pkgs
-        else
-            return $ret
-        fi
-    fi
-}
 function prepare() {
-    local install_cmd
     local which_pkg="which"
-    source /etc/os-release
-    if test -f /etc/redhat-release ; then
-        if [ "$VERSION_ID" -ge "22" ]; then
-            install_cmd="dnf -y install"
-        else
-            install_cmd="yum install -y"
-        fi
-    elif type zypper > /dev/null 2>&1 ; then
-        install_cmd="zypper --gpg-auto-import-keys --non-interactive install --no-recommends"
-    elif type apt-get > /dev/null 2>&1 ; then
-        install_cmd="apt-get install -y"
+    if command -v apt-get > /dev/null 2>&1 ; then
         which_pkg="debianutils"
     fi
 
-    if [ "$EUID" -ne 0 ] && ! type sudo > /dev/null 2>&1 ; then
-        echo "Please install sudo and re-run. This script assumes it is running"
-        echo "as a normal user with the ability to run commands as root via sudo."
-        exit 1
-    fi
-    if [ -n "$install_cmd" ]; then
-        in_jenkins && echo "CI_DEBUG: Running '$install_cmd ccache $which_pkg clang'"
-        do_install "$install_cmd" ccache git $which_pkg clang
-    else
-        echo "WARNING: Don't know how to install packages" >&2
-        echo "This probably means distribution $ID is not supported by run-make-check.sh" >&2
+    if test -f ./install-deps.sh ; then
+        in_jenkins && echo "CI_DEBUG: Running install-deps.sh"
+        INSTALL_EXTRA_PACKAGES="ccache git $which_pkg clang"
+        $DRY_RUN source ./install-deps.sh || return 1
+        trap clean_up_after_myself EXIT
     fi
 
     if ! type ccache > /dev/null 2>&1 ; then
         echo "ERROR: ccache could not be installed"
         exit 1
     fi
-
-    if test -f ./install-deps.sh ; then
-        in_jenkins && echo "CI_DEBUG: Running install-deps.sh"
-        $DRY_RUN source ./install-deps.sh || return 1
-        trap clean_up_after_myself EXIT
-    fi
 }
 
 function configure() {

From 29d9a827e7396dcdc51b124f3e40e3144fa9ccb9 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 31 Oct 2022 13:50:56 -0400
Subject: [PATCH 07/14] script: add lib-build.sh for common high level funcs
 and no main

The intention of this file is collect some of the most basic or common
shell functions used across the various build scripts.
I would also like to ensure that functions added here are validated
using `shellcheck`. Currently, there's no automation for this, just
the honor system, but eventually we can start automating validating
this and other scripts with shellcheck.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/script/lib-build.sh | 27 +++++++++++++++++++++++++++
 src/script/run-make.sh  | 11 +++++++----
 2 files changed, 34 insertions(+), 4 deletions(-)
 create mode 100644 src/script/lib-build.sh

diff --git a/src/script/lib-build.sh b/src/script/lib-build.sh
new file mode 100644
index 0000000000000..21da09eb62ddb
--- /dev/null
+++ b/src/script/lib-build.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+#
+# lib-build.sh - A library of build and test bash shell functions.
+#
+# There should be few, or none, globals in this file beyond function
+# definitions.
+#
+# This script should be `shellcheck`ed. Please run shellcheck when
+# making changes to this script and use ignore comments
+# (ref: https://www.shellcheck.net/wiki/Ignore ) to explicitly mark
+# where a line is intentionally ignoring a typical rule.
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2.1 of the License, or (at your option) any later version.
+#
+
+# The following global only exists to help detect if lib-build has already been
+# sourced. This is only needed because the scripts that are being migrated are
+# often sourcing (as opposed to exec'ing one another).
+# shellcheck disable=SC2034
+_SOURCED_LIB_BUILD=1
+
+function in_jenkins() {
+    [ -n "$JENKINS_HOME" ]
+}
diff --git a/src/script/run-make.sh b/src/script/run-make.sh
index 4b9a17b790695..77bf57d6d9620 100755
--- a/src/script/run-make.sh
+++ b/src/script/run-make.sh
@@ -2,15 +2,18 @@
 
 set -e
 
+if ! [ "${_SOURCED_LIB_BUILD}" = 1 ]; then
+    SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    CEPH_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+    . "${CEPH_ROOT}/src/script/lib-build.sh" || exit 2
+fi
+
+
 trap clean_up_after_myself EXIT
 
 ORIGINAL_CCACHE_CONF="$HOME/.ccache/ccache.conf"
 SAVED_CCACHE_CONF="$HOME/.run-make-check-saved-ccache-conf"
 
-function in_jenkins() {
-    test -n "$JENKINS_HOME"
-}
-
 function save_ccache_conf() {
     test -f $ORIGINAL_CCACHE_CONF && cp $ORIGINAL_CCACHE_CONF $SAVED_CCACHE_CONF || true
 }

From 9b7db758c3f2a381e58790b60aee8908b9a6f1fa Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 31 Oct 2022 13:52:50 -0400
Subject: [PATCH 08/14] install-deps.sh: source lib-build.sh

Currently, only in_jenkins is used. More will come later.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 install-deps.sh | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/install-deps.sh b/install-deps.sh
index 22be8ed1b422e..678bd023cd5d1 100755
--- a/install-deps.sh
+++ b/install-deps.sh
@@ -12,6 +12,14 @@
 #  version 2.1 of the License, or (at your option) any later version.
 #
 set -e
+
+if ! [ "${_SOURCED_LIB_BUILD}" = 1 ]; then
+    SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    CEPH_ROOT="${SCRIPT_DIR}"
+    . "${CEPH_ROOT}/src/script/lib-build.sh" || exit 2
+fi
+
+
 DIR=/tmp/install-deps.$$
 trap "rm -fr $DIR" EXIT
 mkdir -p $DIR
@@ -24,9 +32,6 @@ export LC_ALL=C.UTF-8
 
 ARCH=$(uname -m)
 
-function in_jenkins() {
-    test -n "$JENKINS_HOME"
-}
 
 function munge_ceph_spec_in {
     local with_seastar=$1

From 93b94811cb10bb936c5abf87d708ae96d7939a8e Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 31 Oct 2022 15:06:25 -0400
Subject: [PATCH 09/14] script: add a common ci_debug function to print ci
 debug lines

Reduces some of the boilerplate around emitting the "CI_DEBUG:"
prefixed debug lines for the CI. Additionally, enables using
the FORCE_CI_DEBUG var to enable ci debug lines even when not
in a jenkins environment.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/script/lib-build.sh |  6 ++++++
 src/script/run-make.sh  | 10 +++++-----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/script/lib-build.sh b/src/script/lib-build.sh
index 21da09eb62ddb..a6ff447c99ee4 100644
--- a/src/script/lib-build.sh
+++ b/src/script/lib-build.sh
@@ -25,3 +25,9 @@ _SOURCED_LIB_BUILD=1
 function in_jenkins() {
     [ -n "$JENKINS_HOME" ]
 }
+
+function ci_debug() {
+    if in_jenkins || [ "${FORCE_CI_DEBUG}" ]; then
+        echo "CI_DEBUG: $*"
+    fi
+}
diff --git a/src/script/run-make.sh b/src/script/run-make.sh
index 77bf57d6d9620..3ab00861519d9 100755
--- a/src/script/run-make.sh
+++ b/src/script/run-make.sh
@@ -68,7 +68,7 @@ function prepare() {
     fi
 
     if test -f ./install-deps.sh ; then
-        in_jenkins && echo "CI_DEBUG: Running install-deps.sh"
+        ci_debug "Running install-deps.sh"
         INSTALL_EXTRA_PACKAGES="ccache git $which_pkg clang"
         $DRY_RUN source ./install-deps.sh || return 1
         trap clean_up_after_myself EXIT
@@ -134,9 +134,9 @@ EOM
 
     cmake_opts+=$(detect_ceph_dev_pkgs)
 
-    in_jenkins && echo "CI_DEBUG: Our cmake_opts are: $cmake_opts
-                        CI_DEBUG: Running ./configure"
-    in_jenkins && echo "CI_DEBUG: Running do_cmake.sh"
+    ci_debug "Our cmake_opts are: $cmake_opts"
+    ci_debug "Running ./configure"
+    ci_debug "Running do_cmake.sh"
 
     $DRY_RUN ./do_cmake.sh $cmake_opts $@ || return 1
 }
@@ -154,7 +154,7 @@ function build() {
     BUILD_MAKEOPTS=${BUILD_MAKEOPTS:-$DEFAULT_MAKEOPTS}
     test "$BUILD_MAKEOPTS" && echo "make will run with option(s) $BUILD_MAKEOPTS"
     # older cmake does not support --parallel or -j, so pass it to underlying generator
-    in_jenkins && echo "CI_DEBUG: Running cmake"
+    ci_debug "Running cmake"
     $DRY_RUN cmake --build . $targets -- $BUILD_MAKEOPTS || return 1
     $DRY_RUN ccache -s # print the ccache statistics to evaluate the efficiency
 }

From 1361f2e850616ee67eb1424945ac6e3960356266 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Mon, 31 Oct 2022 15:18:25 -0400
Subject: [PATCH 10/14] install-deps.sh: use ci_debug function

Replace a number of boilerplate CI_DEBUG lines with the recently added
ci_debug function.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 install-deps.sh | 38 ++++++++++++++++++--------------------
 1 file changed, 18 insertions(+), 20 deletions(-)

diff --git a/install-deps.sh b/install-deps.sh
index 678bd023cd5d1..4f5d880555691 100755
--- a/install-deps.sh
+++ b/install-deps.sh
@@ -68,7 +68,7 @@ function munge_debian_control {
 }
 
 function ensure_decent_gcc_on_ubuntu {
-    in_jenkins && echo "CI_DEBUG: Start ensure_decent_gcc_on_ubuntu() in install-deps.sh"
+    ci_debug "Start ensure_decent_gcc_on_ubuntu() in install-deps.sh"
     # point gcc to the one offered by g++-7 if the used one is not
     # new enough
     local old=$(gcc -dumpfullversion -dumpversion)
@@ -105,7 +105,7 @@ ENDOFKEY
 }
 
 function ensure_python3_sphinx_on_ubuntu {
-    in_jenkins && echo "CI_DEBUG: Running ensure_python3_sphinx_on_ubuntu() in install-deps.sh"
+    ci_debug "Running ensure_python3_sphinx_on_ubuntu() in install-deps.sh"
     local sphinx_command=/usr/bin/sphinx-build
     # python-sphinx points $sphinx_command to
     # ../share/sphinx/scripts/python2/sphinx-build when it's installed
@@ -116,7 +116,7 @@ function ensure_python3_sphinx_on_ubuntu {
 }
 
 function install_pkg_on_ubuntu {
-    in_jenkins && echo "CI_DEBUG: Running install_pkg_on_ubuntu() in install-deps.sh"
+    ci_debug "Running install_pkg_on_ubuntu() in install-deps.sh"
     local project=$1
     shift
     local sha1=$1
@@ -133,7 +133,7 @@ function install_pkg_on_ubuntu {
         for pkg in $pkgs; do
             if ! apt -qq list $pkg 2>/dev/null | grep -q installed; then
                 missing_pkgs+=" $pkg"
-                in_jenkins && echo "CI_DEBUG: missing_pkgs=$missing_pkgs"
+                ci_debug "missing_pkgs=$missing_pkgs"
             fi
         done
     fi
@@ -146,7 +146,7 @@ function install_pkg_on_ubuntu {
 }
 
 function install_boost_on_ubuntu {
-    in_jenkins && echo "CI_DEBUG: Running install_boost_on_ubuntu() in install-deps.sh"
+    ci_debug "Running install_boost_on_ubuntu() in install-deps.sh"
     local ver=1.79
     local installed_ver=$(apt -qq list --installed ceph-libboost*-dev 2>/dev/null |
                               grep -e 'libboost[0-9].[0-9]\+-dev' |
@@ -187,7 +187,7 @@ function install_boost_on_ubuntu {
 }
 
 function install_libzbd_on_ubuntu {
-    in_jenkins && echo "CI_DEBUG: Running install_libzbd_on_ubuntu() in install-deps.sh"
+    ci_debug "Running install_libzbd_on_ubuntu() in install-deps.sh"
     local codename=$1
     local project=libzbd
     local sha1=1fadde94b08fab574b17637c2bebd2b1e7f9127b
@@ -259,7 +259,7 @@ EOF
 }
 
 function populate_wheelhouse() {
-    in_jenkins && echo "CI_DEBUG: Running populate_wheelhouse() in install-deps.sh"
+    ci_debug "Running populate_wheelhouse() in install-deps.sh"
     local install=$1
     shift
 
@@ -277,7 +277,7 @@ function populate_wheelhouse() {
 }
 
 function activate_virtualenv() {
-    in_jenkins && echo "CI_DEBUG: Running activate_virtualenv() in install-deps.sh"
+    ci_debug "Running activate_virtualenv() in install-deps.sh"
     local top_srcdir=$1
     local env_dir=$top_srcdir/install-deps-python3
 
@@ -293,7 +293,7 @@ function activate_virtualenv() {
 }
 
 function preload_wheels_for_tox() {
-    in_jenkins && echo "CI_DEBUG: Running preload_wheels_for_tox() in install-deps.sh"
+    ci_debug "Running preload_wheels_for_tox() in install-deps.sh"
     local ini=$1
     shift
     pushd . > /dev/null
@@ -407,7 +407,7 @@ else
                 # state. Run the command again after `dpkg --configure -a` to
                 # bring package manager back into a clean state.
                 $SUDO dpkg --configure -a
-                in_jenkins && echo "CI_DEBUG: trying to install $INSTALL_EXTRA_PACKAGES again"
+                ci_debug "trying to install $INSTALL_EXTRA_PACKAGES again"
                 $SUDO apt-get install -y $INSTALL_EXTRA_PACKAGES
             fi
         fi
@@ -439,7 +439,7 @@ else
         fi
         touch $DIR/status
 
-        in_jenkins && echo "CI_DEBUG: Running munge_debian_control() in install-deps.sh"
+        ci_debug "Running munge_debian_control() in install-deps.sh"
         backports=""
         control=$(munge_debian_control "$VERSION" "debian/control")
         case "$VERSION" in
@@ -462,19 +462,17 @@ else
             build_profiles+=",pkg.ceph.pmdk"
         fi
 
-        in_jenkins && cat <<EOF
-CI_DEBUG: for_make_check=$for_make_check
-CI_DEBUG: with_seastar=$with_seastar
-CI_DEBUG: with_jaeger=$with_jaeger
-CI_DEBUG: build_profiles=$build_profiles
-CI_DEBUG: Now running 'mk-build-deps' and installing ceph-build-deps package
-EOF
+        ci_debug "for_make_check=$for_make_check"
+        ci_debug "with_seastar=$with_seastar"
+        ci_debug "with_jaeger=$with_jaeger"
+        ci_debug "build_profiles=$build_profiles"
+        ci_debug "Now running 'mk-build-deps' and installing ceph-build-deps package"
 
         $SUDO env DEBIAN_FRONTEND=noninteractive mk-build-deps \
               --build-profiles "${build_profiles#,}" \
               --install --remove \
               --tool="apt-get -y --no-install-recommends $backports" $control || exit 1
-        in_jenkins && echo "CI_DEBUG: Removing ceph-build-deps"
+        ci_debug "Removing ceph-build-deps"
         $SUDO env DEBIAN_FRONTEND=noninteractive apt-get -y remove ceph-build-deps
         if [ "$control" != "debian/control" ] ; then rm $control; fi
 
@@ -572,4 +570,4 @@ if $for_make_check; then
     type git > /dev/null || (echo "Dashboard uses git to pull dependencies." ; false)
 fi
 
-in_jenkins && echo "CI_DEBUG: End install-deps.sh" || true
+ci_debug "End install-deps.sh" || true

From a76a0cfdc63b8f7daefefeb1af542db56310a2da Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 1 Nov 2022 09:57:16 -0400
Subject: [PATCH 11/14] script: move get_processors to lib-build.sh

This function can be more useful because the NPROC value can
be provided regardless of how many cores nproc actually detects
and may be handy in a restricted environment like a container.

The new version quotes some values and uses $((...)) as per shellcheck
warning that "expr is antiquated".

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/script/lib-build.sh | 17 +++++++++++++++++
 src/script/run-make.sh  | 13 -------------
 2 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/src/script/lib-build.sh b/src/script/lib-build.sh
index a6ff447c99ee4..17c2fe72148c7 100644
--- a/src/script/lib-build.sh
+++ b/src/script/lib-build.sh
@@ -31,3 +31,20 @@ function ci_debug() {
         echo "CI_DEBUG: $*"
     fi
 }
+
+# get_processors returns 1/2 the value of the value returned by
+# the nproc program OR the value of the environment variable NPROC
+# allowing the user to tune the number of cores visible to the
+# build scripts.
+function get_processors() {
+    # get_processors() depends on coreutils nproc.
+    if [ -n "$NPROC" ]; then
+        echo "$NPROC"
+    else
+        if [ "$(nproc)" -ge 2 ]; then
+            echo "$(($(nproc) / 2))"
+        else
+            echo 1
+        fi
+    fi
+}
diff --git a/src/script/run-make.sh b/src/script/run-make.sh
index 3ab00861519d9..4331e581aa584 100755
--- a/src/script/run-make.sh
+++ b/src/script/run-make.sh
@@ -27,19 +27,6 @@ function clean_up_after_myself() {
     restore_ccache_conf
 }
 
-function get_processors() {
-    # get_processors() depends on coreutils nproc.
-    if test -n "$NPROC" ; then
-        echo $NPROC
-    else
-        if test $(nproc) -ge 2 ; then
-            expr $(nproc) / 2
-        else
-            echo 1
-        fi
-    fi
-}
-
 function detect_ceph_dev_pkgs() {
     local cmake_opts="-DWITH_FMT_VERSION=9.0.0"
     local boost_root=/opt/ceph

From 7edc4309f4781362f86fd83fd2d53eba42e5f73f Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 1 Nov 2022 10:03:31 -0400
Subject: [PATCH 12/14] run-make-check: use get_processors in run-make-check
 script

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 run-make-check.sh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/run-make-check.sh b/run-make-check.sh
index f923f955cd0c0..0ebe2b5de6bdf 100755
--- a/run-make-check.sh
+++ b/run-make-check.sh
@@ -33,11 +33,13 @@ function run() {
     # increase the aio-max-nr, which is by default 65536. we could reach this
     # limit while running seastar tests and bluestore tests.
     local m=16
-    if [ $(nproc) -gt $m ]; then
-        m=$(nproc)
+    local procs="$(($(get_processors) * 2))"
+    if [ "${procs}" -gt $m ]; then
+        m="${procs}"
     fi
-    if [ "$(/sbin/sysctl -n fs.aio-max-nr )" -lt "$((65536 * $(nproc)))" ]; then
-        $DRY_RUN sudo /sbin/sysctl -q -w fs.aio-max-nr=$((65536 * $(nproc)))
+    local aiomax="$((65536 * procs))"
+    if [ "$(/sbin/sysctl -n fs.aio-max-nr )" -lt "${aiomax}" ]; then
+        $DRY_RUN sudo /sbin/sysctl -q -w fs.aio-max-nr="${aiomax}"
     fi
 
     CHECK_MAKEOPTS=${CHECK_MAKEOPTS:-$DEFAULT_MAKEOPTS}

From 561bf4ea9b4dd8f6710a959c5bd38eb4085a9d52 Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 1 Nov 2022 14:51:57 -0400
Subject: [PATCH 13/14] script: add discover_compiler function to lib-build.sh

The discover_compiler function is an abstraction over the current
compiler detection code in run-make.sh. It is intended to be flexible
enough to work on {centos,rhel} systems, but currently is just an
updated version of the logic from run-make.sh. The intent is that this
function will grow and become useful for other scripts used for
building (possibly do_cmake.sh for example).

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/script/lib-build.sh | 29 +++++++++++++++++++++++++++++
 src/script/run-make.sh  | 14 +++++---------
 2 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/src/script/lib-build.sh b/src/script/lib-build.sh
index 17c2fe72148c7..c6ce94b633ad7 100644
--- a/src/script/lib-build.sh
+++ b/src/script/lib-build.sh
@@ -48,3 +48,32 @@ function get_processors() {
         fi
     fi
 }
+
+# discover_compiler takes one argument, purpose, which may be used
+# to adjust the results for a specific need. It sets three environment
+# variables `discovered_c_compiler`, `discovered_cxx_compiler` and
+# `discovered_compiler_env`. The `discovered_compiler_env` variable
+# may be blank. If not, it will contain a file that needs to be sourced
+# prior to using the compiler.
+function discover_compiler() {
+    # nb: currently purpose is not used for detection
+    local purpose="$1"
+    ci_debug "Finding compiler for ${purpose}"
+
+    local compiler_env=""
+    local cxx_compiler=g++
+    local c_compiler=gcc
+    # ubuntu/debian ci builds prefer clang
+    for i in {14..10}; do
+        if type -t "clang-$i" > /dev/null; then
+            cxx_compiler="clang++-$i"
+            c_compiler="clang-$i"
+            break
+        fi
+    done
+
+    export discovered_c_compiler="${c_compiler}"
+    export discovered_cxx_compiler="${cxx_compiler}"
+    export discovered_compiler_env="${compiler_env}"
+    return 0
+}
diff --git a/src/script/run-make.sh b/src/script/run-make.sh
index 4331e581aa584..8e04e6fc8a46f 100755
--- a/src/script/run-make.sh
+++ b/src/script/run-make.sh
@@ -89,15 +89,11 @@ EOM
     fi
     $DRY_RUN ccache -sz # Reset the ccache statistics and show the current configuration
 
-    local cxx_compiler=g++
-    local c_compiler=gcc
-    for i in $(seq 14 -1 10); do
-        if type -t clang-$i > /dev/null; then
-            cxx_compiler="clang++-$i"
-            c_compiler="clang-$i"
-            break
-        fi
-    done
+    if ! discover_compiler ci-build ; then
+        ci_debug "Failed to discover a compiler"
+    fi
+    local cxx_compiler="${discovered_cxx_compiler}"
+    local c_compiler="${discovered_c_compiler}"
     local cmake_opts
     cmake_opts+=" -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_C_COMPILER=$c_compiler"
     cmake_opts+=" -DCMAKE_CXX_FLAGS_DEBUG=-Werror"

From 435d4c75386b0b58b33e3e2024983c608b46249c Mon Sep 17 00:00:00 2001
From: John Mulligan <jmulligan@redhat.com>
Date: Tue, 1 Nov 2022 14:58:16 -0400
Subject: [PATCH 14/14] script: add gcc-toolset-11 support to discover_compiler

In order to configure, build, and run tests in a CentOS 8 (or similar)
container we need a functioning gcc-toolset compiler. This relies on
the environment script being sourced as well. cmake does not appear
to be able to discover this compiler own its own.

Signed-off-by: John Mulligan <jmulligan@redhat.com>
---
 src/script/lib-build.sh | 9 +++++++++
 src/script/run-make.sh  | 4 ++++
 2 files changed, 13 insertions(+)

diff --git a/src/script/lib-build.sh b/src/script/lib-build.sh
index c6ce94b633ad7..6c4455344379d 100644
--- a/src/script/lib-build.sh
+++ b/src/script/lib-build.sh
@@ -71,6 +71,15 @@ function discover_compiler() {
             break
         fi
     done
+    # but if this is {centos,rhel} we need gcc-toolset
+    if [ -f "/opt/rh/gcc-toolset-11/enable" ]; then
+        ci_debug "Detected SCL gcc-toolset-11 environment file"
+        compiler_env="/opt/rh/gcc-toolset-11/enable"
+        # shellcheck disable=SC1090
+        cxx_compiler="$(. ${compiler_env} && command -v g++)"
+        # shellcheck disable=SC1090
+        c_compiler="$(. ${compiler_env} && command -v gcc)"
+    fi
 
     export discovered_c_compiler="${c_compiler}"
     export discovered_cxx_compiler="${cxx_compiler}"
diff --git a/src/script/run-make.sh b/src/script/run-make.sh
index 8e04e6fc8a46f..431f2001b388f 100755
--- a/src/script/run-make.sh
+++ b/src/script/run-make.sh
@@ -92,6 +92,10 @@ EOM
     if ! discover_compiler ci-build ; then
         ci_debug "Failed to discover a compiler"
     fi
+    if [ "${discovered_compiler_env}" ]; then
+        ci_debug "Enabling compiler environment file: ${discovered_compiler_env}"
+        . "${discovered_compiler_env}"
+    fi
     local cxx_compiler="${discovered_cxx_compiler}"
     local c_compiler="${discovered_c_compiler}"
     local cmake_opts