Merge pull request ceph#48697 from phlogistonjohn/jjm-cleaner-build-sh

build scripts: various cleanups and minor changes preparing for containerized build and test Reviewed-by: Ernesto Puerta <[email protected]> Reviewed-by: Kefu Chai <[email protected]>
nizamial09 · Mar 6, 2023 · ce08eda · ce08eda
2 parents 03773df + 435d4c7
commit ce08eda
Show file tree

Hide file tree

Showing 4 changed files with 262 additions and 203 deletions.
diff --git a/install-deps.sh b/install-deps.sh
@@ -12,6 +12,14 @@
 #  version 2.1 of the License, or (at your option) any later version.
 #
 set -e
+
+if ! [ "${_SOURCED_LIB_BUILD}" = 1 ]; then
+    SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    CEPH_ROOT="${SCRIPT_DIR}"
+    . "${CEPH_ROOT}/src/script/lib-build.sh" || exit 2
+fi
+
+
 DIR=/tmp/install-deps.$$
 trap "rm -fr $DIR" EXIT
 mkdir -p $DIR
@@ -24,9 +32,6 @@ export LC_ALL=C.UTF-8
 
 ARCH=$(uname -m)
 
-function in_jenkins() {
-    test -n "$JENKINS_HOME"
-}
 
 function munge_ceph_spec_in {
     local with_seastar=$1
@@ -63,7 +68,7 @@ function munge_debian_control {
 }
 
 function ensure_decent_gcc_on_ubuntu {
-    in_jenkins && echo "CI_DEBUG: Start ensure_decent_gcc_on_ubuntu() in install-deps.sh"
+    ci_debug "Start ensure_decent_gcc_on_ubuntu() in install-deps.sh"
     # point gcc to the one offered by g++-7 if the used one is not
     # new enough
     local old=$(gcc -dumpfullversion -dumpversion)
@@ -100,7 +105,7 @@ ENDOFKEY
 }
 
 function ensure_python3_sphinx_on_ubuntu {
-    in_jenkins && echo "CI_DEBUG: Running ensure_python3_sphinx_on_ubuntu() in install-deps.sh"
+    ci_debug "Running ensure_python3_sphinx_on_ubuntu() in install-deps.sh"
     local sphinx_command=/usr/bin/sphinx-build
     # python-sphinx points $sphinx_command to
     # ../share/sphinx/scripts/python2/sphinx-build when it's installed
@@ -111,7 +116,7 @@ function ensure_python3_sphinx_on_ubuntu {
 }
 
 function install_pkg_on_ubuntu {
-    in_jenkins && echo "CI_DEBUG: Running install_pkg_on_ubuntu() in install-deps.sh"
+    ci_debug "Running install_pkg_on_ubuntu() in install-deps.sh"
     local project=$1
     shift
     local sha1=$1
@@ -128,7 +133,7 @@ function install_pkg_on_ubuntu {
         for pkg in $pkgs; do
             if ! apt -qq list $pkg 2>/dev/null | grep -q installed; then
                 missing_pkgs+=" $pkg"
-                in_jenkins && echo "CI_DEBUG: missing_pkgs=$missing_pkgs"
+                ci_debug "missing_pkgs=$missing_pkgs"
             fi
         done
     fi
@@ -142,7 +147,7 @@ function install_pkg_on_ubuntu {
 }
 
 function install_boost_on_ubuntu {
-    in_jenkins && echo "CI_DEBUG: Running install_boost_on_ubuntu() in install-deps.sh"
+    ci_debug "Running install_boost_on_ubuntu() in install-deps.sh"
     local ver=1.79
     local installed_ver=$(apt -qq list --installed ceph-libboost*-dev 2>/dev/null |
                               grep -e 'libboost[0-9].[0-9]\+-dev' |
@@ -183,7 +188,7 @@ function install_boost_on_ubuntu {
 }
 
 function install_libzbd_on_ubuntu {
-    in_jenkins && echo "CI_DEBUG: Running install_libzbd_on_ubuntu() in install-deps.sh"
+    ci_debug "Running install_libzbd_on_ubuntu() in install-deps.sh"
     local codename=$1
     local project=libzbd
     local sha1=1fadde94b08fab574b17637c2bebd2b1e7f9127b
@@ -254,6 +259,67 @@ EOF
     fi
 }
 
+function populate_wheelhouse() {
+    ci_debug "Running populate_wheelhouse() in install-deps.sh"
+    local install=$1
+    shift
+
+    # although pip comes with virtualenv, having a recent version
+    # of pip matters when it comes to using wheel packages
+    PIP_OPTS="--timeout 300 --exists-action i"
+    pip $PIP_OPTS $install \
+      'setuptools >= 0.8' 'pip >= 21.0' 'wheel >= 0.24' 'tox >= 2.9.1' || return 1
+    if test $# != 0 ; then
+        # '--use-feature=fast-deps --use-deprecated=legacy-resolver' added per
+        # https://github.com/pypa/pip/issues/9818 These should be able to be
+        # removed at some point in the future.
+        pip --use-feature=fast-deps --use-deprecated=legacy-resolver $PIP_OPTS $install $@ || return 1
+    fi
+}
+
+function activate_virtualenv() {
+    ci_debug "Running activate_virtualenv() in install-deps.sh"
+    local top_srcdir=$1
+    local env_dir=$top_srcdir/install-deps-python3
+
+    if ! test -d $env_dir ; then
+        python3 -m venv ${env_dir}
+        . $env_dir/bin/activate
+        if ! populate_wheelhouse install ; then
+            rm -rf $env_dir
+            return 1
+        fi
+    fi
+    . $env_dir/bin/activate
+}
+
+function preload_wheels_for_tox() {
+    ci_debug "Running preload_wheels_for_tox() in install-deps.sh"
+    local ini=$1
+    shift
+    pushd . > /dev/null
+    cd $(dirname $ini)
+    local require_files=$(ls *requirements*.txt 2>/dev/null) || true
+    local constraint_files=$(ls *constraints*.txt 2>/dev/null) || true
+    local require=$(echo -n "$require_files" | sed -e 's/^/-r /')
+    local constraint=$(echo -n "$constraint_files" | sed -e 's/^/-c /')
+    local md5=wheelhouse/md5
+    if test "$require"; then
+        if ! test -f $md5 || ! md5sum -c $md5 > /dev/null; then
+            rm -rf wheelhouse
+        fi
+    fi
+    if test "$require" && ! test -d wheelhouse ; then
+        type python3 > /dev/null 2>&1 || continue
+        activate_virtualenv $top_srcdir || exit 1
+        python3 -m pip install --upgrade pip
+        populate_wheelhouse "wheel -w $wip_wheelhouse" $require $constraint || exit 1
+        mv $wip_wheelhouse wheelhouse
+        md5sum $require_files $constraint_files > $md5
+    fi
+    popd > /dev/null
+}
+
 for_make_check=false
 if tty -s; then
     # interactive
@@ -265,6 +331,10 @@ else
 fi
 
 if [ x$(uname)x = xFreeBSDx ]; then
+    if [ "$INSTALL_EXTRA_PACKAGES" ]; then
+        echo "Installing extra packages not supported on FreeBSD" >&2
+        exit 1
+    fi
     $SUDO pkg install -yq \
         devel/babeltrace \
         devel/binutils \
@@ -330,6 +400,18 @@ else
     case "$ID" in
     debian|ubuntu|devuan|elementary|softiron)
         echo "Using apt-get to install dependencies"
+        if [ "$INSTALL_EXTRA_PACKAGES" ]; then
+            if ! $SUDO apt-get install -y $INSTALL_EXTRA_PACKAGES ; then
+                # try again. ported over from run-make.sh (orignally e278295)
+                # In the case that apt-get is interrupted, like when a jenkins
+                # job is cancelled, the package manager will be in an inconsistent
+                # state. Run the command again after `dpkg --configure -a` to
+                # bring package manager back into a clean state.
+                $SUDO dpkg --configure -a
+                ci_debug "trying to install $INSTALL_EXTRA_PACKAGES again"
+                $SUDO apt-get install -y $INSTALL_EXTRA_PACKAGES
+            fi
+        fi
         $SUDO apt-get install -y devscripts equivs
         $SUDO apt-get install -y dpkg-dev
         ensure_python3_sphinx_on_ubuntu
@@ -358,7 +440,7 @@ else
         fi
         touch $DIR/status
 
-        in_jenkins && echo "CI_DEBUG: Running munge_debian_control() in install-deps.sh"
+        ci_debug "Running munge_debian_control() in install-deps.sh"
         backports=""
         control=$(munge_debian_control "$VERSION" "debian/control")
         case "$VERSION" in
@@ -381,19 +463,17 @@ else
             build_profiles+=",pkg.ceph.pmdk"
         fi
 
-        in_jenkins && cat <<EOF
-CI_DEBUG: for_make_check=$for_make_check
-CI_DEBUG: with_seastar=$with_seastar
-CI_DEBUG: with_jaeger=$with_jaeger
-CI_DEBUG: build_profiles=$build_profiles
-CI_DEBUG: Now running 'mk-build-deps' and installing ceph-build-deps package
-EOF
+        ci_debug "for_make_check=$for_make_check"
+        ci_debug "with_seastar=$with_seastar"
+        ci_debug "with_jaeger=$with_jaeger"
+        ci_debug "build_profiles=$build_profiles"
+        ci_debug "Now running 'mk-build-deps' and installing ceph-build-deps package"
 
         $SUDO env DEBIAN_FRONTEND=noninteractive mk-build-deps \
               --build-profiles "${build_profiles#,}" \
               --install --remove \
               --tool="apt-get -y --no-install-recommends $backports" $control || exit 1
-        in_jenkins && echo "CI_DEBUG: Removing ceph-build-deps"
+        ci_debug "Removing ceph-build-deps"
         $SUDO env DEBIAN_FRONTEND=noninteractive apt-get -y remove ceph-build-deps
         if [ "$control" != "debian/control" ] ; then rm $control; fi
 
@@ -433,6 +513,9 @@ EOF
                 fi
                 ;;
         esac
+        if [ "$INSTALL_EXTRA_PACKAGES" ]; then
+            $SUDO dnf install -y $INSTALL_EXTRA_PACKAGES
+        fi
         munge_ceph_spec_in $with_seastar $with_zbd $for_make_check $DIR/ceph.spec
         # for python3_pkgversion macro defined by python-srpm-macros, which is required by python3-devel
         $SUDO dnf install -y python3-devel
@@ -456,6 +539,9 @@ EOF
         echo "Using zypper to install dependencies"
         zypp_install="zypper --gpg-auto-import-keys --non-interactive install --no-recommends"
         $SUDO $zypp_install systemd-rpm-macros rpm-build || exit 1
+        if [ "$INSTALL_EXTRA_PACKAGES" ]; then
+            $SUDO $zypp_install $INSTALL_EXTRA_PACKAGES
+        fi
         munge_ceph_spec_in $with_seastar false $for_make_check $DIR/ceph.spec
         $SUDO $zypp_install $(rpmspec -q --buildrequires $DIR/ceph.spec) || exit 1
         ;;
@@ -466,67 +552,6 @@ EOF
     esac
 fi
 
-function populate_wheelhouse() {
-    in_jenkins && echo "CI_DEBUG: Running populate_wheelhouse() in install-deps.sh"
-    local install=$1
-    shift
-
-    # although pip comes with virtualenv, having a recent version
-    # of pip matters when it comes to using wheel packages
-    PIP_OPTS="--timeout 300 --exists-action i"
-    pip $PIP_OPTS $install \
-      'setuptools >= 0.8' 'pip >= 21.0' 'wheel >= 0.24' 'tox >= 2.9.1' || return 1
-    if test $# != 0 ; then
-        # '--use-feature=fast-deps --use-deprecated=legacy-resolver' added per
-        # https://github.com/pypa/pip/issues/9818 These should be able to be
-        # removed at some point in the future.
-        pip --use-feature=fast-deps --use-deprecated=legacy-resolver $PIP_OPTS $install $@ || return 1
-    fi
-}
-
-function activate_virtualenv() {
-    in_jenkins && echo "CI_DEBUG: Running activate_virtualenv() in install-deps.sh"
-    local top_srcdir=$1
-    local env_dir=$top_srcdir/install-deps-python3
-
-    if ! test -d $env_dir ; then
-        python3 -m venv ${env_dir}
-        . $env_dir/bin/activate
-        if ! populate_wheelhouse install ; then
-            rm -rf $env_dir
-            return 1
-        fi
-    fi
-    . $env_dir/bin/activate
-}
-
-function preload_wheels_for_tox() {
-    in_jenkins && echo "CI_DEBUG: Running preload_wheels_for_tox() in install-deps.sh"
-    local ini=$1
-    shift
-    pushd . > /dev/null
-    cd $(dirname $ini)
-    local require_files=$(ls *requirements*.txt 2>/dev/null) || true
-    local constraint_files=$(ls *constraints*.txt 2>/dev/null) || true
-    local require=$(echo -n "$require_files" | sed -e 's/^/-r /')
-    local constraint=$(echo -n "$constraint_files" | sed -e 's/^/-c /')
-    local md5=wheelhouse/md5
-    if test "$require"; then
-        if ! test -f $md5 || ! md5sum -c $md5 > /dev/null; then
-            rm -rf wheelhouse
-        fi
-    fi
-    if test "$require" && ! test -d wheelhouse ; then
-        type python3 > /dev/null 2>&1 || continue
-        activate_virtualenv $top_srcdir || exit 1
-        python3 -m pip install --upgrade pip
-        populate_wheelhouse "wheel -w $wip_wheelhouse" $require $constraint || exit 1
-        mv $wip_wheelhouse wheelhouse
-        md5sum $require_files $constraint_files > $md5
-    fi
-    popd > /dev/null
-}
-
 # use pip cache if possible but do not store it outside of the source
 # tree
 # see https://pip.pypa.io/en/stable/reference/pip_install.html#caching
@@ -546,4 +571,4 @@ if $for_make_check; then
     type git > /dev/null || (echo "Dashboard uses git to pull dependencies." ; false)
 fi
 
-in_jenkins && echo "CI_DEBUG: End install-deps.sh" || true
+ci_debug "End install-deps.sh" || true
diff --git a/run-make-check.sh b/run-make-check.sh
@@ -22,10 +22,6 @@ source src/script/run-make.sh
 
 set -e
 
-function in_jenkins() {
-    test -n "$JENKINS_HOME"
-}
-
 function run() {
     # to prevent OSD EMFILE death on tests, make sure ulimit >= 1024
     $DRY_RUN ulimit -n $(ulimit -Hn)
@@ -37,10 +33,14 @@ function run() {
     # increase the aio-max-nr, which is by default 65536. we could reach this
     # limit while running seastar tests and bluestore tests.
     local m=16
-    if [ $(nproc) -gt $m ]; then
-        m=$(nproc)
+    local procs="$(($(get_processors) * 2))"
+    if [ "${procs}" -gt $m ]; then
+        m="${procs}"
+    fi
+    local aiomax="$((65536 * procs))"
+    if [ "$(/sbin/sysctl -n fs.aio-max-nr )" -lt "${aiomax}" ]; then
+        $DRY_RUN sudo /sbin/sysctl -q -w fs.aio-max-nr="${aiomax}"
     fi
-    $DRY_RUN sudo /sbin/sysctl -q -w fs.aio-max-nr=$((65536 * $(nproc)))
 
     CHECK_MAKEOPTS=${CHECK_MAKEOPTS:-$DEFAULT_MAKEOPTS}
     if in_jenkins; then
@@ -71,43 +71,13 @@ function main() {
     fi
     # uses run-make.sh to install-deps
     FOR_MAKE_CHECK=1 prepare
-    local cxx_compiler=g++
-    local c_compiler=gcc
-    for i in $(seq 14 -1 10); do
-        if type -t clang-$i > /dev/null; then
-            cxx_compiler="clang++-$i"
-            c_compiler="clang-$i"
-            break
-        fi
-    done
-    # Init defaults after deps are installed.
-    local cmake_opts
-    cmake_opts+=" -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_C_COMPILER=$c_compiler"
-    cmake_opts+=" -DCMAKE_CXX_FLAGS_DEBUG=-Werror"
-    cmake_opts+=" -DENABLE_GIT_VERSION=OFF"
-    cmake_opts+=" -DWITH_GTEST_PARALLEL=ON"
-    cmake_opts+=" -DWITH_FIO=ON"
-    cmake_opts+=" -DWITH_CEPHFS_SHELL=ON"
-    cmake_opts+=" -DWITH_GRAFANA=ON"
-    cmake_opts+=" -DWITH_SPDK=ON"
-    cmake_opts+=" -DWITH_RBD_MIRROR=ON"
-    if [ $WITH_SEASTAR ]; then
-        cmake_opts+=" -DWITH_SEASTAR=ON"
-    fi
-    if [ $WITH_ZBD ]; then
-        cmake_opts+=" -DWITH_ZBD=ON"
-    fi
-    if [ $WITH_RBD_RWL ]; then
-        cmake_opts+=" -DWITH_RBD_RWL=ON"
-    fi
-    cmake_opts+=" -DWITH_RBD_SSD_CACHE=ON"
-    in_jenkins && echo "CI_DEBUG: Our cmake_opts are: $cmake_opts
-                        CI_DEBUG: Running ./configure"
-    configure "$cmake_opts" "$@"
+    configure "$@"
     in_jenkins && echo "CI_DEBUG: Running 'build tests'"
     build tests
     echo "make check: successful build on $(git rev-parse HEAD)"
     FOR_MAKE_CHECK=1 run
 }
 
-main "$@"
+if [ "$0" = "$BASH_SOURCE" ]; then
+    main "$@"
+fi