[DAPHNE-daphne-eu#416] Initial Intel FPGA Support

This commit introduces support for the Intel PAC D5005 FPGA accelerator card. It adds the necessary bits in the compiler infrastructure, a runtime kernel for SGEMM and context handling. The precompiled bitstream will be provided in the separate repository github.com/daphne-eu/supplemental-binaries to avoid blowing up the size of the main repository.

CMakeLists.txt

@@ -145,6 +145,19 @@ if(USE_ARROW)
     message(STATUS "Arrow/Parquet enabled")
 endif()
 
+option(USE_FPGAOPENCL "Whether to activate compilation of FPGA OpenCL features" OFF)
+if(USE_FPGAOPENCL)
+	if(NOT DEFINED ENV{QUARTUSDIR})
+		message(SEND_ERROR "Intel(R) Quartus installation directory should be defined by QUARTUSDIR varaiable (e.g. /opt/intel/intelFPGA_pro/21.4/)")
+		execute_process(COMMAND sleep 10)
+	endif()
+	include_directories($ENV{QUARTUSDIR}/hld/examples_aoc/common/inc/ $ENV{QUARTUSDIR}/hld/host/include/)
+	message(STATUS "cmake: using FPGA")
+	add_definitions(-DUSE_FPGAOPENCL)
+endif()
+
+set(CMAKE_VERBOSE_MAKEFILE ON)
+
 # *****************************************************************************
 # Project-specific include directories
 # *****************************************************************************
@@ -170,6 +183,10 @@ add_subdirectory(src/runtime/distributed/worker)
 add_subdirectory(src/runtime/local/datastructures)
 add_subdirectory(src/runtime/local/io)
 add_subdirectory(src/runtime/local/kernels)
+if(USE_FPGAOPENCL)
+    add_subdirectory(src/runtime/local/kernels/FPGAOPENCL)
+endif() 
 add_subdirectory(src/util)
 
+
 add_subdirectory(test)

build.sh

@@ -381,6 +381,7 @@ par_acceptAll="0"
 unknown_options=""
 BUILD_CUDA="-DUSE_CUDA=OFF"
 BUILD_ARROW="-DUSE_ARROW=OFF"
+BUILD_FPGAOPENCL="-DUSE_FPGAOPENCL=OFF"
 BUILD_DEBUG="-DCMAKE_BUILD_TYPE=Release"
 
 while [[ $# -gt 0 ]]; do
@@ -414,6 +415,10 @@ while [[ $# -gt 0 ]]; do
             echo using ARROW
             BUILD_ARROW="-DUSE_ARROW=ON"
             ;;
+        --fpgaopencl)
+            echo using FPGAOPENCL
+            export BUILD_FPGAOPENCL="-DUSE_FPGAOPENCL=ON"
+            ;;
         --debug)
             echo building DEBUG version
             export BUILD_DEBUG="-DCMAKE_BUILD_TYPE=Debug"
@@ -701,14 +706,25 @@ else
     daphne_msg "No need to build MLIR/LLVM again."
 fi
 
+if [[ $BUILD_FPGAOPENCL = *"ON"* ]]; then
+  FPGAOPENCL_BISTREAM_DIR="$projectRoot/src/runtime/local/kernels/FPGAOPENCL/bitstreams"
+  FPGAOPENCL_BISTREAM_URL="https://github.com/daphne-eu/supplemental-binaries/raw/main/fpga_bitstreams/"
+  if [ ! -d $FPGAOPENCL_BISTREAM_DIR ]; then
+    echo fetching FPGA bitstreams
+    mkdir -p $FPGAOPENCL_BISTREAM_DIR
+    cd $FPGAOPENCL_BISTREAM_DIR
+    wget $FPGAOPENCL_BISTREAM_URL/sgemm.aocx
+    cd -
+  fi
+fi
 
 # *****************************************************************************
 # Build DAPHNE target.
 # *****************************************************************************
 
 daphne_msg "Build Daphne"
 
-cmake -S "$projectRoot" -B "$daphneBuildDir" -G Ninja $BUILD_CUDA $BUILD_ARROW $BUILD_DEBUG \
+cmake -S "$projectRoot" -B "$daphneBuildDir" -G Ninja $BUILD_CUDA $BUILD_ARROW $BUILD_FPGAOPENCL  $BUILD_DEBUG \
   -DCMAKE_PREFIX_PATH="$installPrefix" -DANTLR_VERSION="$antlrVersion"  \
   -DMLIR_DIR="$buildPrefix/$llvmName/lib/cmake/mlir/" \
   -DLLVM_DIR="$buildPrefix/$llvmName/lib/cmake/llvm/"

doc/FPGAconfiguration.md

@@ -0,0 +1,62 @@
+<!--
+Copyright 2021 The DAPHNE Consortium
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# FPGA configuration for usage in DAPHNE 
+
+
+### System requirments
+
+Daphne build script for FPGA kernels support requires additional QUARTUSDIR system variable definition.
+Example command is presented in fpga-build-env.sh or in the following command:
+
+export QUARTUSDIR=/opt/intel/intelFPGA_pro/21.4
+
+To build the Daphne with the FPGA support -fpgaopencl flag has to be used:
+
+ ./build.sh --fpgaopenc 
+
+
+To run developed or precompiled, included in Daphne repository FPGA OpenCL kernels an installedand configured  FPGA device is required.
+Our example kernels have been tested using Intel(R) PAC D5005 card (https://www.intel.com/content/www/us/en/products/sku/193921/intel-fpga-pac-d5005/specifications.html)
+
+DAPHNE contains some example linear algebra kernels developed using T2SP framework(https://github.com/IntelLabs/t2sp/blob/master/README.md). 
+Example precompiled FPGA kernels can be usedon  DAPHNE DSL description level. 
+To prepare the system for the precompiled FPGA kernels some FPGA and OpenCL system variables are required. 
+The easiest way to set up required varables is to use the init_opencl.sh script from installed Intel(R) Quartus sowtware or from the 
+Intel(R) OpenCL RTE or Intel(R) OpenCL SDK packages. 
+
+Example script usage:
+source /opt/intel/intelFPGA_pro/21.4/hld/init_opencl.sh
+
+For additional details please look into https://www.intel.com/content/www/us/en/docs/programmable/683550/18-1/standard-edition-getting-started-guide.html
+or https://www.intel.com/content/www/us/en/software/programmable/sdk-for-opencl/overview.html. 
+
+
+### Precompiled FPGA kernels usage 
+
+To use a precompiled FPGA kernel a FPGA image is required (*.aocx). FPGA device has to programmed with particular image which contains required kernel implementation.
+Example FPGA programming command using example FPGA image:
+
+ aocl program acl0 src/runtime/local/kernels/FPGAOPENCL/bitstreams/sgemm.aocx   
+
+
+Additionally the BITSTREAM variable has to be defind in the system.
+Please look into the following example:
+
+ export BITSTREAM=src/runtime/local/kernels/FPGAOPENCL/bitstreams/sgemm.aocx 
+
+When another FPGA image contains implementation for another required computational kernel then FPGA device has to be reprogrammed and BITSTREAM variable value has to be changed.
+

fpga-build-env.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+#initialize Intel FPGA OpenCL environment
+export QUARTUSDIR=/opt/intel/intelFPGA_pro/21.4
+source $QUARTUSDIR/hld/init_opencl.sh
+echo $INTELFPGAOCLSDKROOT
+export ALTERAOCLSDKROOT=$INTELFPGAOCLSDKROOT
+#set up BITSTREAM variable for required FPGA image (can be different different for varius implemented kernels)
+export BITSTREAM=src/runtime/local/kernels/FPGAOPENCL/bitstreams/sgemm.aocx  # SGEMM computational kernel
+

scripts/examples/fpga-gemm.daph

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2021 The DAPHNE Consortium
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+############## IMPORTANT #################
+# EXAMPLE command to run the script: 
+# 
+# env BITSTREAM=../../src/runtime/local/kernels/FPGAOPENCL/bistreams/sgemm.aocx  ./build/bin/daphne -fpgaopencl scripts/examples/fpga-gemm.daph
+# 
+# WHERE: 
+# -fpgaopencl is a flag required for FPGA usage
+# BITSTREAM variable point out required FPGA image with its location 
+#########################################
+
+# Creating input matrices
+m  = rand(448,1024, as.f32(2.0), as.f32(2.0), 1.0, -1);
+//m  = rand(896,8192, as.f32(2.0), as.f32(2.0), 1.0, -1);
+//m  = rand(448,16384, as.f32(2.0), as.f32(2.0), 1.0, -1);
+
+m2 = rand(1024,512, as.f32(1.0), as.f32(1.0), 1.0, -1);
+//m2 = rand(8192,1024, as.f32(1.0), as.f32(1.0), 1.0, -1);
+//m2 = rand(16384,1024, as.f32(1.0), as.f32(1.0), 1.0, -1);
+
+# test prints
+//print(m[0,]);
+//print(m2[0,]);
+
+Z = m @ m2;
+//print(Z[,0]);
+
+print("Bye!");

src/api/cli/CMakeLists.txt

@@ -32,6 +32,10 @@ if(USE_CUDA AND CMAKE_CUDA_COMPILER)
     list(APPEND LIB_DEPS CUDAKernels)
 endif()
 
+if(USE_FPGAOPENCL)
+    list(APPEND LIB_DEPS FPGAOPENCLKernels)
+endif()
+
 add_llvm_executable(daphne daphne.cpp DaphneUserConfig.h DEPENDS ${LIB_DEPS})
 
 llvm_update_compile_flags(daphne)

src/api/cli/DaphneUserConfig.h

@@ -40,6 +40,7 @@ struct DaphneUserConfig {
     bool pinWorkers = false;
     bool hyperthreadingEnabled = false;
     bool debugMultiThreading = false;
+    bool use_fpgaopencl = false;
 
     bool debug_llvm = false;
     bool explain_kernels = false;
@@ -65,6 +66,11 @@ struct DaphneUserConfig {
     // ToDo: This is an arbitrary default taken from sample code
 //    int cublas_workspace_size = 1024 * 1024 * 4;
 #endif
+#ifdef USE_FPGAOPENCL
+    std::vector<int> fpga_devices;
+#endif
+
+
     std::string libdir;
     std::vector<std::string> library_paths;
 };

src/api/cli/daphne.cpp

@@ -176,6 +176,10 @@ main(int argc, char** argv)
             "cuda", cat(daphneOptions),
             desc("Use CUDA")
     );
+    opt<bool> fpgaopencl(
+            "fpgaopencl", cat(daphneOptions),
+            desc("Use FPGAOPENCL")
+    );
     opt<string> libDir(
             "libdir", cat(daphneOptions),
             desc("The directory containing kernel libraries")
@@ -324,6 +328,11 @@ main(int argc, char** argv)
         }
     }
 
+    if(fpgaopencl) {
+        user_config.use_fpgaopencl = true;
+    }
+
+
     // add this after the cli args loop to work around args order
     if(!user_config.libdir.empty() && user_config.use_cuda)
             user_config.library_paths.push_back(user_config.libdir + "/libCUDAKernels.so");

src/compiler/execution/DaphneIrExecutor.cpp

@@ -135,6 +135,12 @@ bool DaphneIrExecutor::runPasses(mlir::ModuleOp module)
             pm.addNestedPass<mlir::FuncOp>(mlir::daphne::createMarkCUDAOpsPass(userConfig_));
 #endif
 
+#ifdef USE_FPGAOPENCL
+        if(userConfig_.use_fpgaopencl)
+            pm.addNestedPass<mlir::FuncOp>(mlir::daphne::createMarkFPGAOPENCLOpsPass(userConfig_));
+#endif
+
+
         if(userConfig_.use_obj_ref_mgnt)
             pm.addNestedPass<mlir::FuncOp>(mlir::daphne::createManageObjRefsPass());
         if(userConfig_.explain_obj_ref_mgnt)
@@ -182,6 +188,14 @@ std::unique_ptr<mlir::ExecutionEngine> DaphneIrExecutor::createExecutionEngine(m
             }
         }
 #endif
+
+#ifdef USE_FPGAOPENCL
+        if(userConfig_.use_fpgaopencl) {
+            if(userConfig_.libdir.empty()) {
+                sharedLibRefs.push_back("build/src/runtime/local/kernels/libFPGAOPENCLKernels.so");
+            }
+        }
+#endif
         registerLLVMDialectTranslation(context_);
         // module.dump();
         auto maybeEngine = mlir::ExecutionEngine::create(

src/compiler/lowering/CMakeLists.txt

@@ -17,6 +17,7 @@ add_mlir_dialect_library(MLIRDaphneTransforms
     DistributeComputationsPass.cpp
     DistributePipelinesPass.cpp
     MarkCUDAOpsPass.cpp
+    MarkFPGAOPENCLOpsPass.cpp
     InsertDaphneContextPass.cpp
     ManageObjRefsPass.cpp
     LowerToLLVMPass.cpp

src/compiler/lowering/InsertDaphneContextPass.cpp

@@ -63,6 +63,13 @@ void InsertDaphneContextPass::runOnFunction()
     if (user_config.use_distributed){
         builder.create<daphne::CreateDistributedContextOp>(loc);
     }
+#ifdef USE_FPGAOPENCL
+    if(user_config.use_fpgaopencl) {
+        builder.create<daphne::CreateFPGAContextOp>(loc);
+    }
+#endif
+
+
     // Insert a DestroyDaphneContextOp as the last operation in the block, but
     // before the block's terminator.
     builder.setInsertionPoint(b.getTerminator());

src/compiler/lowering/MarkFPGAOPENCLOpsPass.cpp

@@ -0,0 +1,57 @@
+/*
+ *  Copyright 2021 The DAPHNE Consortium
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+#ifdef USE_FPGAOPENCL
+#include "compiler/CompilerUtils.h"
+#include "ir/daphneir/Daphne.h"
+#include "ir/daphneir/Passes.h"
+#include <mlir/IR/BlockAndValueMapping.h>
+
+#include <iostream>
+
+using namespace mlir;
+
+struct MarkFPGAOPENCLOpsPass : public PassWrapper<MarkFPGAOPENCLOpsPass, FunctionPass> {
+
+    /**
+     * @brief User configuration influencing the rewrite pass
+     */
+    const DaphneUserConfig& cfg;
+
+    explicit MarkFPGAOPENCLOpsPass(const DaphneUserConfig& cfg) : cfg(cfg) {
+    }
+
+    void runOnFunction() final;
+
+    bool checkUseFPGAOPENCL(Operation* op) const {
+//        std::cout << "checkUseFPGAOPENCL: " << op->getName().getStringRef().str() << std::endl;
+        return op->hasTrait<mlir::OpTrait::FPGAOPENCLSupport>();
+    }
+};
+
+void MarkFPGAOPENCLOpsPass::runOnFunction() {
+    getFunction()->walk([&](Operation* op) {
+        OpBuilder builder(op);
+        if(checkUseFPGAOPENCL(op)) {
+            op->setAttr("fpgaopencl_device", builder.getI32IntegerAttr(0));
+        }
+        WalkResult::advance();
+    });
+}
+
+std::unique_ptr<Pass> daphne::createMarkFPGAOPENCLOpsPass(const DaphneUserConfig& cfg) {
+    return std::make_unique<MarkFPGAOPENCLOpsPass>(cfg);
+}
+#endif 

src/compiler/lowering/RewriteToCallKernelOpPass.cpp

@@ -153,6 +153,10 @@ namespace
 //                else
 //                    std::cout << "attr = null: " << op->getName().getStringRef().str() << std::endl;
             }
+	    else if(op->hasAttr("fpgaopencl_device")) {
+		 callee << "FPGAOPENCL";
+	    }
+
 
             callee << '_' << op->getName().stripDialect().data();
 

src/ir/daphneir/Daphne.h

@@ -47,6 +47,12 @@
 #include <utility>
 #include <vector>
 
+namespace mlir::OpTrait {
+    template<class ConcreteOp>
+    class FPGAOPENCLSupport : public TraitBase<ConcreteOp, FPGAOPENCLSupport> {
+    };
+}
+
 namespace mlir::daphne {
     enum class MatrixRepresentation {
         Dense = 0,
@@ -67,4 +73,4 @@ namespace mlir::daphne {
 #define GET_OP_CLASSES
 #include "ir/daphneir/DaphneOps.h.inc"
 
-#endif //SRC_IR_DAPHNEIR_DAPHNE_H
+#endif //SRC_IR_DAPHNEIR_DAPHNE_H