[DAPHNE-daphne-eu#830] Support compilation with Clang

This commit enables compiling DAPHNE with Clang (tested version 18.1.3 from Ubuntu 24 LTS). Below is the collected change log of the pre-squash commits:

[DAPHNE-daphne-eu#830] Fix duplicate symbol linker issue

Compiling with Clang/LLD complains about duplicate symbols of supportsUnaryOp and supportsBinaryOp.
The definition of supportsBinaryOp would need to be declared static if left i the BinaryOpCode.h header which is included in several places.
Since this definition is not allowed and there is only one use, it is moved to its own file.

[DAPHNE-daphne-eu#830] Replace lambda function using invalid C++ with local static template function

std::pair<bool, auto> was used as a convenience function in DaphneDSLVisitor.cpp. According to [1] this is invalid and only works with GCC. Replacing it with a templated local function makes Clang compile it too. Further improvements to this file will be considered in future commit.

[1] https://stackoverflow.com/questions/59578575/stdpairauto-auto-return-type

[DAPHNE-daphne-eu#830] Fix anonymous namespace issue when compiling with Clang

When compiling with clang, there is a issue at runtime (LLVM complaining about anonymous namespaces not allowed in various places). These are hereby replaced with a non-anonymous namespace. For the lack of creativity these are all called "file_local".

[DAPHNE-daphne-eu#830] Fix compilation error (with clang) in json dependency

The use of EOF in our json dependency makes compilation fail with Clang.
Patch is based on nlohmannjson version 3.11.3

[DAPHNE-daphne-eu#830] Silence warning when using LLD

With this new CMake configuration, we silence a lot of warnings when using the LLD linker in combination with clang:
``clang++: warning: argument unused during compilation: '-fuse-ld=lld' [-Wunused-command-line-argument]``

A CMake version bump to 3.29 is necessary for this config option. Ubuntu 24 users need to switch to the snap version of CMake. The Docker images already provide CMake 3.30

[DAPHNE-daphne-eu#830] Clang option for build.sh

This commit adds the convenience option to build with Clang to the build.sh script

Resolves daphne-eu#830, Closes daphne-eu#831

CMakeLists.txt

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-cmake_minimum_required(VERSION 3.25.2)
+cmake_minimum_required(VERSION 3.29)
 
 # Build release version by default (override with -DCMAKE_BUILD_TYPE=Debug in your initial cmake invocation)
 # This needs to be set *before* the project() command
@@ -153,9 +153,8 @@ endif()
 
 find_program(LLD_PROGRAM lld)
 if(LLD_PROGRAM)
-    message(STATUS "Using lld")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fuse-ld=lld")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fuse-ld=lld")
+    set(CMAKE_LINKER_TYPE LLD)
+    message(STATUS "Using lld - LINKER_TYPE: ${CMAKE_LINKER_TYPE}")
 endif()
 
 find_package(Arrow REQUIRED)

build.sh

@@ -62,6 +62,7 @@ function printHelp {
     echo "  --hdfs            Compile with support for HDFS"
     echo "  --io_uring        Compile with support for io_uring"
     echo "  --no-papi         Compile without support for PAPI"
+    echo "  --clang           Compile using Clang (instead of GCC)"
 }
 
 #******************************************************************************
@@ -456,6 +457,7 @@ BUILD_IO_URING="-DUSE_IO_URING=OFF"
 BUILD_PAPI="-DUSE_PAPI=ON"
 WITH_DEPS=1
 WITH_SUBMODULE_UPDATE=1
+WITH_CLANG=""
 
 while [[ $# -gt 0 ]]; do
     key=$1
@@ -528,6 +530,10 @@ while [[ $# -gt 0 ]]; do
     -ns | --no-submodule-update)
         WITH_SUBMODULE_UPDATE=0
         ;;
+    --clang)
+        echo "Compiling with Clang"
+        WITH_CLANG="-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++"
+        ;;
       *)
         unknown_options="${unknown_options} ${key}"
         ;;
@@ -766,13 +772,14 @@ if [ $WITH_DEPS -gt 0 ]; then
     #------------------------------------------------------------------------------
     nlohmannjsonDirName=nlohmannjson
     nlohmannjsonSingleHeaderName=json.hpp
-    dep_nlohmannjson=("nlohmannjson_v${nlohmannjsonVersion}" "v1")
+    dep_nlohmannjson=("nlohmannjson_v${nlohmannjsonVersion}" "v2")
 
     if ! is_dependency_installed "${dep_nlohmannjson[@]}"; then
         daphne_msg "Get nlohmannjson version ${nlohmannjsonVersion}"
         mkdir -p "${installPrefix}/include/${nlohmannjsonDirName}"
         wget "https://github.com/nlohmann/json/releases/download/v$nlohmannjsonVersion/$nlohmannjsonSingleHeaderName" \
             -qO "${installPrefix}/include/${nlohmannjsonDirName}/${nlohmannjsonSingleHeaderName}"
+        patch -Np0 -i "${patchDir}/0007-nlohmannjson-replace-EOF.patch" -d "${installPrefix}/include/${nlohmannjsonDirName}"
         dependency_install_success "${dep_nlohmannjson[@]}"
     else
         daphne_msg "No need to download nlohmannjson again."
@@ -1108,7 +1115,7 @@ fi
 daphne_msg "Build Daphne"
 
 cmake -S "$projectRoot" -B "$daphneBuildDir" -G Ninja -DANTLR_VERSION="$antlrVersion" \
-    -DCMAKE_PREFIX_PATH="$installPrefix" \
+    -DCMAKE_PREFIX_PATH="$installPrefix" ${WITH_CLANG} \
     $BUILD_CUDA $BUILD_FPGAOPENCL $BUILD_DEBUG $BUILD_MPI $BUILD_HDFS $BUILD_PAPI
 
 cmake --build "$daphneBuildDir" --target "$target"

software-package-versions.txt

@@ -24,7 +24,7 @@ cudaVersion=12.6.1
 eigenVersion=3.4.0
 grpcVersion=1.38.0
 hwlocVersion=2.9.3
-nlohmannjsonVersion=3.10.5
+nlohmannjsonVersion=3.11.3
 openBlasVersion=0.3.23
 openMPIVersion=4.1.5
 papiVersion=7.0.1

src/compiler/lowering/AggAllOpLowering.cpp

@@ -174,7 +174,7 @@ class SumAllOpLowering : public OpConversionPattern<daphne::AllAggSumOp> {
     }
 };
 
-namespace {
+namespace file_local {
 /**
  * @brief Lowers the daphne::AggAll operator to a set of affine loops and
  * performs the aggregation on a MemRef which is created from the input
@@ -198,9 +198,9 @@ struct AggAllLoweringPass : public mlir::PassWrapper<AggAllLoweringPass, mlir::O
     }
     void runOnOperation() final;
 };
-} // end anonymous namespace
+} // namespace file_local
 
-void AggAllLoweringPass::runOnOperation() {
+void file_local::AggAllLoweringPass::runOnOperation() {
     mlir::ConversionTarget target(getContext());
     mlir::RewritePatternSet patterns(&getContext());
     LowerToLLVMOptions llvmOptions(&getContext());
@@ -236,5 +236,5 @@ void AggAllLoweringPass::runOnOperation() {
 }
 
 std::unique_ptr<mlir::Pass> mlir::daphne::createAggAllOpLoweringPass() {
-    return std::make_unique<AggAllLoweringPass>();
+    return std::make_unique<file_local::AggAllLoweringPass>();
 }

src/compiler/lowering/DaphneOptPass.cpp

@@ -43,7 +43,7 @@ class IntegerModOpt : public mlir::OpConversionPattern<mlir::daphne::EwModOp> {
     }
 };
 
-namespace {
+namespace file_local {
 /**
  * @brief This pass transforms operations (currently limited to the EwModOp) in
  * the DaphneDialect to a different set of operations also from the
@@ -64,9 +64,9 @@ struct DenseMatrixOptPass : public mlir::PassWrapper<DenseMatrixOptPass, mlir::O
                "also from the DaphneDialect.";
     }
 };
-} // end anonymous namespace
+} // namespace file_local
 
-void DenseMatrixOptPass::runOnOperation() {
+void file_local::DenseMatrixOptPass::runOnOperation() {
     mlir::ConversionTarget target(getContext());
     mlir::RewritePatternSet patterns(&getContext());
     mlir::LowerToLLVMOptions llvmOptions(&getContext());
@@ -89,4 +89,6 @@ void DenseMatrixOptPass::runOnOperation() {
     }
 }
 
-std::unique_ptr<mlir::Pass> mlir::daphne::createDaphneOptPass() { return std::make_unique<DenseMatrixOptPass>(); }
+std::unique_ptr<mlir::Pass> mlir::daphne::createDaphneOptPass() {
+    return std::make_unique<file_local::DenseMatrixOptPass>();
+}

src/compiler/lowering/DistributeComputationsPass.cpp

@@ -27,7 +27,7 @@
 
 using namespace mlir;
 
-namespace {
+namespace file_local {
 struct Distribute : public OpInterfaceConversionPattern<daphne::Distributable> {
     using OpInterfaceConversionPattern::OpInterfaceConversionPattern;
 
@@ -71,13 +71,13 @@ struct DistributeComputationsPass : public PassWrapper<DistributeComputationsPas
     StringRef getArgument() const final { return "distribute-computation"; }
     StringRef getDescription() const final { return "TODO"; }
 };
-} // namespace
+} // namespace file_local
 
 bool onlyMatrixOperands(Operation *op) {
     return llvm::all_of(op->getOperandTypes(), [](Type t) { return llvm::isa<daphne::MatrixType>(t); });
 }
 
-void DistributeComputationsPass::runOnOperation() {
+void file_local::DistributeComputationsPass::runOnOperation() {
     auto module = getOperation();
 
     RewritePatternSet patterns(&getContext());
@@ -105,5 +105,5 @@ void DistributeComputationsPass::runOnOperation() {
 }
 
 std::unique_ptr<Pass> daphne::createDistributeComputationsPass() {
-    return std::make_unique<DistributeComputationsPass>();
+    return std::make_unique<file_local::DistributeComputationsPass>();
 }

src/compiler/lowering/EwOpsLowering.cpp

@@ -200,7 +200,7 @@ using DivOpLowering = BinaryOpLowering<mlir::daphne::EwDivOp, mlir::arith::DivSI
 using PowOpLowering = BinaryOpLowering<mlir::daphne::EwPowOp, mlir::math::PowFOp, mlir::math::PowFOp>;
 // clang-format on
 
-namespace {
+namespace file_local {
 /**
  * @brief This pass lowers element-wise operations to affine loop
  * structures and arithmetic operations.
@@ -223,7 +223,7 @@ struct EwOpLoweringPass : public mlir::PassWrapper<EwOpLoweringPass, mlir::Opera
                "structures and arithmetic operations.";
     }
 };
-} // end anonymous namespace
+} // namespace file_local
 
 void populateLowerEwOpConversionPatterns(mlir::LLVMTypeConverter &typeConverter, mlir::RewritePatternSet &patterns) {
     // clang-format off
@@ -238,7 +238,7 @@ void populateLowerEwOpConversionPatterns(mlir::LLVMTypeConverter &typeConverter,
     // clang-format on
 }
 
-void EwOpLoweringPass::runOnOperation() {
+void file_local::EwOpLoweringPass::runOnOperation() {
     mlir::ConversionTarget target(getContext());
     mlir::RewritePatternSet patterns(&getContext());
     mlir::LowerToLLVMOptions llvmOptions(&getContext());
@@ -286,4 +286,6 @@ void EwOpLoweringPass::runOnOperation() {
         signalPassFailure();
 }
 
-std::unique_ptr<mlir::Pass> mlir::daphne::createEwOpLoweringPass() { return std::make_unique<EwOpLoweringPass>(); }
+std::unique_ptr<mlir::Pass> mlir::daphne::createEwOpLoweringPass() {
+    return std::make_unique<file_local::EwOpLoweringPass>();
+}

src/compiler/lowering/LowerToLLVMPass.cpp

@@ -879,7 +879,7 @@ class GenericCallOpLowering : public OpConversionPattern<daphne::GenericCallOp>
     }
 };
 
-namespace {
+namespace file_local {
 struct DaphneLowerToLLVMPass : public PassWrapper<DaphneLowerToLLVMPass, OperationPass<ModuleOp>> {
     explicit DaphneLowerToLLVMPass(const DaphneUserConfig &cfg) : cfg(cfg) {}
     const DaphneUserConfig &cfg;
@@ -889,9 +889,9 @@ struct DaphneLowerToLLVMPass : public PassWrapper<DaphneLowerToLLVMPass, Operati
     }
     void runOnOperation() final;
 };
-} // end anonymous namespace
+} // namespace file_local
 
-void DaphneLowerToLLVMPass::runOnOperation() {
+void file_local::DaphneLowerToLLVMPass::runOnOperation() {
     auto module = getOperation();
 
     RewritePatternSet patterns(&getContext());
@@ -950,5 +950,5 @@ void DaphneLowerToLLVMPass::runOnOperation() {
 }
 
 std::unique_ptr<Pass> daphne::createLowerToLLVMPass(const DaphneUserConfig &cfg) {
-    return std::make_unique<DaphneLowerToLLVMPass>(cfg);
+    return std::make_unique<file_local::DaphneLowerToLLVMPass>(cfg);
 }

src/compiler/lowering/MapOpLowering.cpp

@@ -81,7 +81,7 @@ class InlineMapOpLowering : public mlir::OpConversionPattern<mlir::daphne::MapOp
     }
 };
 
-namespace {
+namespace file_local {
 /**
  * @brief The MapOpLoweringPass rewrites the daphne::MapOp operator
  * to a set of perfectly nested affine loops and inserts for each element a call
@@ -107,9 +107,9 @@ struct MapOpLoweringPass : public mlir::PassWrapper<MapOpLoweringPass, mlir::Ope
                "UDF.";
     }
 };
-} // end anonymous namespace
+} // namespace file_local
 
-void MapOpLoweringPass::runOnOperation() {
+void file_local::MapOpLoweringPass::runOnOperation() {
     mlir::ConversionTarget target(getContext());
     mlir::RewritePatternSet patterns(&getContext());
     mlir::LowerToLLVMOptions llvmOptions(&getContext());
@@ -127,4 +127,6 @@ void MapOpLoweringPass::runOnOperation() {
     }
 }
 
-std::unique_ptr<mlir::Pass> mlir::daphne::createMapOpLoweringPass() { return std::make_unique<MapOpLoweringPass>(); }
+std::unique_ptr<mlir::Pass> mlir::daphne::createMapOpLoweringPass() {
+    return std::make_unique<file_local::MapOpLoweringPass>();
+}

src/compiler/lowering/MatMulOpLowering.cpp

@@ -606,7 +606,7 @@ class MatMulLowering : public OpConversionPattern<daphne::MatMulOp> {
     }
 };
 
-namespace {
+namespace file_local {
 /**
  * @brief The MatMulLoweringPass rewrites the MatMulOp from the DaphneDialect
  * to a affine loop structure implementing a multi tiled loop structure.
@@ -663,9 +663,9 @@ struct MatMulLoweringPass : public impl::MatMulOpLoweringPassBase<MatMulLowering
         return sizes;
     }
 };
-} // end anonymous namespace
+} // namespace file_local
 
-void MatMulLoweringPass::runOnOperation() {
+void file_local::MatMulLoweringPass::runOnOperation() {
     auto module = getOperation();
     mlir::ConversionTarget target(getContext());
     mlir::RewritePatternSet patterns(&getContext());
@@ -725,13 +725,13 @@ std::unique_ptr<OperationPass<ModuleOp>> mlir::daphne::createMatMulOpLoweringPas
     bool matmul_tile, int matmul_vec_size_bits, std::vector<unsigned> matmul_fixed_tile_sizes,
     bool matmul_use_fixed_tile_sizes, int matmul_unroll_factor, int matmul_unroll_jam_factor,
     int matmul_num_vec_registers, bool matmul_invert_loops) {
-    return std::make_unique<MatMulLoweringPass>(
+    return std::make_unique<file_local::MatMulLoweringPass>(
         matmul_tile, matmul_vec_size_bits, matmul_fixed_tile_sizes, matmul_use_fixed_tile_sizes, matmul_unroll_factor,
         matmul_unroll_jam_factor, matmul_num_vec_registers, matmul_invert_loops);
 }
 
 // This is used by daphne-opt and automatically inserts the options provided on
 // the command line into the pass.
 std::unique_ptr<OperationPass<ModuleOp>> mlir::daphne::createMatMulOpLoweringPass() {
-    return std::make_unique<MatMulLoweringPass>();
+    return std::make_unique<file_local::MatMulLoweringPass>();
 }

src/compiler/lowering/ModOpLowering.cpp

@@ -128,7 +128,7 @@ class EwModOpLowering : public mlir::OpConversionPattern<mlir::daphne::EwModOp>
     }
 };
 
-namespace {
+namespace file_local {
 /**
  * @brief Performs an integer mod optimization on the EwModOp operator by
  * lowering to an affine loop structure and performing the mod op on values
@@ -153,9 +153,9 @@ struct ModOpLoweringPass : public mlir::PassWrapper<ModOpLoweringPass, mlir::Ope
                "and performing the mod op on values loaded from a MemRef.";
     }
 };
-} // end anonymous namespace
+} // namespace file_local
 
-void ModOpLoweringPass::runOnOperation() {
+void file_local::ModOpLoweringPass::runOnOperation() {
     mlir::ConversionTarget target(getContext());
     mlir::RewritePatternSet patterns(&getContext());
     mlir::LowerToLLVMOptions llvmOptions(&getContext());
@@ -184,4 +184,6 @@ void ModOpLoweringPass::runOnOperation() {
     }
 }
 
-std::unique_ptr<mlir::Pass> mlir::daphne::createModOpLoweringPass() { return std::make_unique<ModOpLoweringPass>(); }
+std::unique_ptr<mlir::Pass> mlir::daphne::createModOpLoweringPass() {
+    return std::make_unique<file_local::ModOpLoweringPass>();
+}

src/compiler/lowering/PhyOperatorSelectionPass.cpp

@@ -64,14 +64,14 @@ class MatMulOpLowering : public OpConversionPattern<daphne::MatMulOp> {
     }
 };
 
-namespace {
+namespace file_local {
 struct PhyOperatorSelectionPass : public PassWrapper<PhyOperatorSelectionPass, OperationPass<ModuleOp>> {
     explicit PhyOperatorSelectionPass() {}
     void runOnOperation() final;
 };
-} // end anonymous namespace
+} // namespace file_local
 
-void PhyOperatorSelectionPass::runOnOperation() {
+void file_local::PhyOperatorSelectionPass::runOnOperation() {
     auto module = getOperation();
 
     ConversionTarget target(getContext());
@@ -108,4 +108,6 @@ void PhyOperatorSelectionPass::runOnOperation() {
         signalPassFailure();
 }
 
-std::unique_ptr<Pass> daphne::createPhyOperatorSelectionPass() { return std::make_unique<PhyOperatorSelectionPass>(); }
+std::unique_ptr<Pass> daphne::createPhyOperatorSelectionPass() {
+    return std::make_unique<file_local::PhyOperatorSelectionPass>();
+}

src/compiler/lowering/RewriteSqlOpPass.cpp

@@ -33,7 +33,7 @@
 
 using namespace mlir;
 
-namespace {
+namespace file_local {
 
 std::unordered_map<std::string, mlir::Value> tables;
 struct SqlReplacement : public RewritePattern {
@@ -81,9 +81,9 @@ struct RewriteSqlOpPass : public PassWrapper<RewriteSqlOpPass, OperationPass<Mod
     StringRef getArgument() const final { return "rewrite-sqlop"; }
     StringRef getDescription() const final { return "TODO"; }
 };
-} // namespace
+} // namespace file_local
 
-void RewriteSqlOpPass::runOnOperation() {
+void file_local::RewriteSqlOpPass::runOnOperation() {
     auto module = getOperation();
 
     RewritePatternSet patterns(&getContext());
@@ -98,4 +98,4 @@ void RewriteSqlOpPass::runOnOperation() {
         signalPassFailure();
 }
 
-std::unique_ptr<Pass> daphne::createRewriteSqlOpPass() { return std::make_unique<RewriteSqlOpPass>(); }
+std::unique_ptr<Pass> daphne::createRewriteSqlOpPass() { return std::make_unique<file_local::RewriteSqlOpPass>(); }