diff --git a/buildenv/README.md b/buildenv/README.md new file mode 100644 index 000000000..f9bb0a81d --- /dev/null +++ b/buildenv/README.md @@ -0,0 +1,22 @@ + + +## Build scripts for unsupported environments + +The scripts contained in this directory can be used to build the compiler and necessary libraries to build DAPHNE in +unsupported environments (RHEL UBI, CentOS, ...) Besides the build scripts, there is a docker file to create a UBI image +to build for Redhat 8. +See also the [usage information in the documentation section](/doc/BuildEnvironment.md) diff --git a/buildenv/build-all.sh b/buildenv/build-all.sh new file mode 100755 index 000000000..a01bc6090 --- /dev/null +++ b/buildenv/build-all.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash + +# Stop immediately if any command fails. +set -e + +source env.sh + +# download cache: +export DLC=$ROOTDIR/DLC +if [ ! -d $DLC ]; then + echo Creating DLC dir + mkdir -p $DLC +fi + +mkdir -p $ROOTDIR/installed + +export GMP_VERSION=6.3.0 +export MPFR_VERSION=4.2.1 +export MPC_VERSION=1.3.1 +export ELF_VERSION=0.191 +export BZIP2_VERSION=1.0.8 + +if [ ! -f $DLC/gmp-${GMP_VERSION}.tar.xz ];then + wget https://gmplib.org/download/gmp/gmp-${GMP_VERSION}.tar.xz -P $DLC + tar xf $DLC/gmp-${GMP_VERSION}.tar.xz + cd gmp-${GMP_VERSION} + ./configure --enable-shared --enable-static --prefix=$ROOTDIR/installed + make -j$NUM_VCORES + make check + make install + cd - +fi + +if [ ! -f $DLC/mpfr-${MPFR_VERSION}.tar.xz ];then + wget https://www.mpfr.org/mpfr-current/mpfr-${MPFR_VERSION}.tar.xz -P $DLC + tar xf $DLC/mpfr-${MPFR_VERSION}.tar.xz + cd mpfr-${MPFR_VERSION} + ./configure --enable-shared --enable-static --prefix=$ROOTDIR/installed --with-gmp=$ROOTDIR/installed + make -j$NUM_VCORES + make install + cd - +fi + +if [ ! -f $DLC/mpc-${MPC_VERSION}.tar.gz ];then + wget https://ftp.gnu.org/gnu/mpc/mpc-${MPC_VERSION}.tar.gz -P $DLC + tar xf $DLC/mpc-${MPC_VERSION}.tar.gz + cd mpc-${MPC_VERSION} + ./configure --enable-shared --enable-static --prefix=$ROOTDIR/installed --with-gmp=$ROOTDIR/installed --with-mpfr=$ROOTDIR/installed + make -j$NUM_VCORES + make install + cd - +fi + + +#bzip needs this change in the Makefile (otherwise the python compile fails) +#CFLAGS=-Wall -Winline -O2 -g $(BIGFILES) +#CFLAGS= -O3 -fomit-frame-pointer -funroll-loops -fPIC + +# if [ ! -f $DLC/bzip2-${BZIP2_VERSION}.tar.gz ];then +# wget https://sourceware.org/pub/bzip2/bzip2-${BZIP2_VERSION}.tar.gz -P $DLC +# tar xf $DLC/bzip2-${BZIP2_VERSION}.tar.gz +# cd bzip2-${BZIP2_VERSION} +# make -j$NUM_VCORES +# make install PREFIX=$ROOTDIR/installed +# cd - +# fi + +#optional stuff for libelf +# export LZMA_VERSION +# export ZSTD_VERSION +# wget https://www.7-zip.org/a/lzma2301.7z -P $DLC +# wget https://github.com/facebook/zstd/releases/download/v1.5.6/zstd-1.5.6.tar.gz -P $DLC + +if [ ! -f $DLC/elfutils-${ELF_VERSION}.tar.bz2 ]; then + wget https://sourceware.org/elfutils/ftp/${ELF_VERSION}/elfutils-${ELF_VERSION}.tar.bz2 -P $DLC + tar xf $DLC/elfutils-${ELF_VERSION}.tar.bz2 + cd elfutils-${ELF_VERSION} + ./configure --disable-libdebuginfod --disable-debuginfod --prefix=$ROOTDIR/installed + make -j$NUM_VCORES + make install + cd - +fi + +# ----------------------------------------------------------------------------- +if true; then + ./build-gcc.sh +fi + +# ----------------------------------------------------------------------------- +if true; then + if [ ! -f $DLC/binutils-${BINUTILS_VERSION}.tar.xz ]; then + echo Downloading Binutils + wget https://ftp.gnu.org/gnu/binutils/binutils-${BINUTILS_VERSION}.tar.xz -P $DLC + tar xf $DLC/binutils-${BINUTILS_VERSION}.tar.xz + cd binutils-${BINUTILS_VERSION} + ./configure --prefix=$PWD/../installed + make -j$NUM_VCORES + make install + cd - + fi +fi + +# ----------------------------------------------------------------------------- + +if true; then + if [ ! -f $DLC/Python-${PYTHON_VERSION}.tar.xz ];then + wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz -P $DLC + tar xf $DLC/Python-${PYTHON_VERSION}.tar.xz + cd Python-${PYTHON_VERSION} + time ./configure --prefix=${ROOTDIR}/installed --with-pydebug --enable-optimizations #--with-lto + time make -j$(nproc) + time make install + cd $ROOTDIR + fi +fi + +# ----------------------------------------------------------------------------- +if true; then + if [ ! -f $DLC/cmake-${CMAKE_VERSION}.tar.gz ];then + wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz -P $DLC + tar xf $DLC/cmake-${CMAKE_VERSION}.tar.gz + cd cmake-${CMAKE_VERSION} + ./bootstrap --prefix=$ROOTDIR/installed --parallel=$NUM_VCORES --no-qt-gui + make -j$NUM_VCORES + make install/strip + cd $ROOTDIR + fi +fi + +# ----------------------------------------------------------------------------- +if true; then + if [ ! -f $DLC/re2c-${RE2C_VERSION}.tar.xz ];then + wget https://github.com/skvadrik/re2c/releases/download/${RE2C_VERSION}/re2c-${RE2C_VERSION}.tar.xz -P $DLC + tar xf $DLC/re2c-${RE2C_VERSION}.tar.xz + cd re2c-$RE2C_VERSION + cmake -S . -B build -DCMAKE_INSTALL_PREFIX=$ROOTDIR/installed + cmake --build build --target install/strip + cd $ROOTDIR + fi +fi + +# ----------------------------------------------------------------------------- +# git clone git://github.com/ninja-build/ninja.git +if true; then + if [ ! -f $DLC/v${NINJA_VERSION}.tar.gz ];then + wget https://github.com/ninja-build/ninja/archive/refs/tags/v${NINJA_VERSION}.tar.gz -P $DLC + tar xf $DLC/v${NINJA_VERSION}.tar.gz + cd ninja-$NINJA_VERSION + cmake -S . -B build -DCMAKE_INSTALL_PREFIX=$ROOTDIR/installed + cmake --build build --target install/strip + cd $ROOTDIR + fi +fi + +# ----------------------------------------------------------------------------- +if [ ! -d jdk-11.0.22+7 ];then + wget -qO- https://github.com/adoptium/temurin11-binaries/releases/download/jdk-${JDK_VERSION}%2B${JDK_BUILD}/OpenJDK11U-jdk_x64_linux_hotspot_${JDK_VERSION}_${JDK_BUILD}.tar.gz | tar xzf - +fi + +# ----------------------------------------------------------------------------- +if true; then + ./build-clang.sh +fi + +if [ ! -f $DLC/$CUDA12_PACKAGE ];then + wget https://developer.download.nvidia.com/compute/cuda/12.4.0/local_installers/${CUDA12_PACKAGE} -P $DLC +fi + +if [ ! -f $DLC/cudnn-linux-x86_64-9.0.0.312_cuda12-archive.tar.xz ];then + wget https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/$CUDNN9_PACKAGE -P $DLC +fi + +if [ ! -d "$ROOTDIR/installed/cuda-$CUDA12_VERSION" ];then + echo "Extracting CUDA $CUDA12_VERSION SDK and CUDNN9" + chmod u+x "$DLC/$CUDA12_PACKAGE" + "$DLC/$CUDA12_PACKAGE" --silent --toolkit --no-drm --no-man-page --no-opengl-libs --override --installpath="$ROOTDIR/installed/cuda-$CUDA12_VERSION" + tar xf "$DLC/$CUDNN9_PACKAGE" --directory="$ROOTDIR/installed/cuda-$CUDA12_VERSION/targets/x86_64-linux/" --strip-components=1 +fi + +set +e diff --git a/buildenv/build-clang.sh b/buildenv/build-clang.sh new file mode 100755 index 000000000..74bb009a4 --- /dev/null +++ b/buildenv/build-clang.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +source ./env.sh +ROOTDIR=$(pwd) + +if [ ! -f $DLC/llvmorg-${LLVM_VERSION}.tar.gz ]; then + wget https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-${LLVM_VERSION}.tar.gz -P $DLC +fi + +tar xf $DLC/llvmorg-${LLVM_VERSION}.tar.gz +cd llvm-project-llvmorg-${LLVM_VERSION} +mkdir --parents build-clang +cd build-clang +cmake -G Ninja ../llvm \ + -DLLVM_ENABLE_PROJECTS=clang \ + -DLLVM_BUILD_EXAMPLES=OFF \ + -DLLVM_TARGETS_TO_BUILD="X86" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_ENABLE_RTTI=ON \ + -DCMAKE_INSTALL_PREFIX=$ROOTDIR/installed +cmake --build . --target clang +cmake --build . --target install/strip +cd .. +mkdir --parents build-lld +cd build-lld +cmake -G Ninja ../llvm \ + -DLLVM_ENABLE_PROJECTS=lld \ + -DLLVM_BUILD_EXAMPLES=OFF \ + -DLLVM_TARGETS_TO_BUILD="X86" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_ENABLE_RTTI=ON \ + -DCMAKE_INSTALL_PREFIX=$ROOTDIR/installed +cmake --build . --target lld +cmake --build . --target install/strip + +cd $ROOTDIR diff --git a/buildenv/build-gcc.sh b/buildenv/build-gcc.sh new file mode 100755 index 000000000..7e89333c5 --- /dev/null +++ b/buildenv/build-gcc.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +source ./env.sh + +if [ ! -f $DLC/gcc-${GCC_VERSION}.tar.xz ]; then + wget ftp://ftp.lip6.fr/pub/gcc/releases/gcc-${GCC_VERSION}/gcc-${GCC_VERSION}.tar.xz -P $DLC +fi + +tar xf $DLC/gcc-${GCC_VERSION}.tar.xz +cd gcc-${GCC_VERSION} +./contrib/download_prerequisites +mkdir ../build-gcc +cd ../build-gcc +unset C_INCLUDE_PATH CPLUS_INCLUDE_PATH CFLAGS CXXFLAGS +../gcc-${GCC_VERSION}/configure --prefix=$ROOTDIR/installed --enable-languages=c,c++,fortran --disable-libquadmath \ + --disable-lto --disable-libquadmath-support --disable-werror --disable-bootstrap --enable-gold --disable-multilib --disable-libssp +make -j$NUM_VCORES +make install +cd $ROOTDIR/installed/bin +ln -sf gcc cc +ln -sf g++ c++ +cd $ROOTDIR + diff --git a/buildenv/build-ubi8.sh b/buildenv/build-ubi8.sh new file mode 100755 index 000000000..02da40a05 --- /dev/null +++ b/buildenv/build-ubi8.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker build --tag daphne-ubi8 -f ./daphne-ubi8.Dockerfile . \ No newline at end of file diff --git a/buildenv/daphne-ubi8.Dockerfile b/buildenv/daphne-ubi8.Dockerfile new file mode 100644 index 000000000..e51a88d18 --- /dev/null +++ b/buildenv/daphne-ubi8.Dockerfile @@ -0,0 +1,13 @@ +FROM redhat/ubi8 + +RUN dnf update -y --refresh + +# dev tools +RUN dnf install -y redhat-lsb-core less nano git wget unzip file rsync vim xz + +# build deps for DAPHNE deps +RUN dnf install -y gcc gcc-c++ pkg-config patch zstd libuuid-devel zlib-devel bzip2-devel + +# build deps from epel +RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm +RUN dnf install -y lbzip2 ccache openssl-devel diff --git a/buildenv/entrypoint.sh b/buildenv/entrypoint.sh new file mode 100755 index 000000000..878fa7300 --- /dev/null +++ b/buildenv/entrypoint.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +echo Creating user: $USER +groupadd -r sudo +groupadd -g $GID $GROUP +useradd -u $UID -g $GID -G sudo,users -m $USER +dnf install -y sudo +printf "${USER} ALL=(ALL:ALL) NOPASSWD:ALL" | sudo EDITOR="tee -a" visudo > /dev/null +su $USER diff --git a/buildenv/env.sh b/buildenv/env.sh new file mode 100755 index 000000000..42adaec9b --- /dev/null +++ b/buildenv/env.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +export ROOTDIR=$PWD +export NUM_VCORES=$(nproc) +JDK_VERSION=11.0.22 +JDK_BUILD=7 +export JAVA_HOME=$ROOTDIR/jdk-$JDK_VERSION+$JDK_BUILD +# export PATH=$JAVA_HOME/bin:${ROOTDIR}/installed/bin:$PATH +export DLC=$ROOTDIR/DLC + +export CMAKE_VERSION=3.29.0 +export GCC_VERSION=13.2.0 +export NINJA_VERSION=1.11.1 +export RE2C_VERSION=3.1 +export PYTHON_VERSION=3.12.2 +export LLVM_VERSION=18.1.2 +export BINUTILS_VERSION=2.42 +export CUDA12_VERSION=12.4 +export CUDA_VERSION=$CUDA12_VERSION +export CUDA12_PACKAGE=cuda_12.4.0_550.54.14_linux.run +export CUDNN9_PACKAGE=cudnn-linux-x86_64-9.0.0.312_cuda12-archive.tar.xz +export CUDA_PATH=$ROOTDIR/installed/cuda-$CUDA_VERSION + +export PATH=$CUDA_PATH/bin:$PWD/installed/bin:$JAVA_HOME/bin:$PATH +export LD_LIBRARY_PATH=$CUDA_PATH/lib64:$PWD/installed/lib:$PWD/installed/lib64:$LD_LIBRARY_PATH +export LD_RUN_PATH=$LD_LIBRARY_PATH + + +# rhel für gcc compile: +# install xz, libmpc-devel gmp-devel mpfr-devel + +# exit +# run docker: +#!/usr/bin/env bash +# docker run --rm -it -v $PWD:/workdir --entrypoint=/workdir/commands.sh daphne-rhel bash diff --git a/buildenv/run-ubi8.sh b/buildenv/run-ubi8.sh new file mode 100755 index 000000000..c5d794113 --- /dev/null +++ b/buildenv/run-ubi8.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +docker run -e USER=$(id -un) -e UID=$(id -u) -e GROUP=$(id -gn) -e GID=$(id -g) --rm -it --gpus all \ + -v $PWD:/workdir -w /workdir --entrypoint=/workdir/entrypoint.sh daphne-ubi8 bash diff --git a/doc/BuildEnvironment.md b/doc/BuildEnvironment.md new file mode 100644 index 000000000..42779384d --- /dev/null +++ b/doc/BuildEnvironment.md @@ -0,0 +1,35 @@ + + +# Building in unsupported environments + +DAPHNE can also be built in environments other than the one endorsed by the DAPHNE project team. To help with that, +there are scripts to build your own toolchain in the [``buildenv``](/buildenv) directory. +These scripts can be used to build the compiler and necessary libraries to build DAPHNE in unsupported environments +(RHEL UBI, CentOS, ...) Besides the build scripts, there is a docker file to create a UBI image to build for Redhat 8. + + +Usage: +* Create Docker image with build-ubi8.sh +* Run the Docker image with run-ubi8.sh +* Run build-all.sh +* Run source env.sh inside or outside the Docker image to set PATH and linker variables to use the created environment +(you need to cd into the directory containing env.sh as this uses $PWD) + + +Beware that this procedure needs ~50GB of free disk space. Also, the provided CUDA SDK expects a recent driver (550+) +version. That will most likely be an issue on large installations - exchange the relevant version and file names for +another CUDA version in env.sh to build another version. \ No newline at end of file