From f7caa2ba76b713aee4ee141538ee97906cf619d5 Mon Sep 17 00:00:00 2001 From: Francesco Oliva Date: Thu, 4 Jul 2024 01:59:11 +0200 Subject: [PATCH] included and use the InferenceEngine library from another GitHub project --- CMakeLists.txt | 58 ++--- cmake/AddCompileDefinitions.cmake | 33 +++ cmake/LibTensorFlow.cmake | 16 -- cmake/LibTorch.cmake | 20 -- cmake/LinkBackend.cmake | 12 +- cmake/ONNXRuntime.cmake | 30 --- cmake/OpenCVdnn.cmake | 8 - cmake/OpenVino.cmake | 11 - cmake/{TensorRT.cmake => QueryGpu.cmake} | 22 -- cmake/SelectBackend.cmake | 20 -- cmake/SetCompilerFlags.cmake | 2 +- components/inference-engines/CMakeLists.txt | 18 -- .../src/InferenceInterface.cpp | 25 -- .../src/InferenceInterface.hpp | 29 --- .../src/libtensorflow/TFDetectionAPI.cpp | 51 ---- .../src/libtensorflow/TFDetectionAPI.hpp | 43 ---- .../src/libtorch/LibtorchInfer.cpp | 108 --------- .../src/libtorch/LibtorchInfer.hpp | 17 -- .../src/onnx-runtime/ORTInfer.cpp | 221 ------------------ .../src/onnx-runtime/ORTInfer.hpp | 22 -- .../src/opencv-dnn/OCVDNNInfer.cpp | 57 ----- .../src/opencv-dnn/OCVDNNInfer.hpp | 16 -- .../src/openvino/OVInfer.cpp | 31 --- .../src/openvino/OVInfer.hpp | 21 -- .../inference-engines/src/tensorrt/Logger.hpp | 14 -- .../src/tensorrt/TRTInfer.cpp | 175 -------------- .../src/tensorrt/TRTInfer.hpp | 43 ---- inc/InferenceBackendSetup.hpp | 36 --- 28 files changed, 61 insertions(+), 1098 deletions(-) create mode 100644 cmake/AddCompileDefinitions.cmake delete mode 100644 cmake/LibTensorFlow.cmake delete mode 100644 cmake/LibTorch.cmake delete mode 100644 cmake/ONNXRuntime.cmake delete mode 100644 cmake/OpenCVdnn.cmake delete mode 100644 cmake/OpenVino.cmake rename cmake/{TensorRT.cmake => QueryGpu.cmake} (74%) delete mode 100644 cmake/SelectBackend.cmake delete mode 100644 components/inference-engines/CMakeLists.txt delete mode 100644 components/inference-engines/src/InferenceInterface.cpp delete mode 100644 components/inference-engines/src/InferenceInterface.hpp delete mode 100644 components/inference-engines/src/libtensorflow/TFDetectionAPI.cpp delete mode 100644 components/inference-engines/src/libtensorflow/TFDetectionAPI.hpp delete mode 100644 components/inference-engines/src/libtorch/LibtorchInfer.cpp delete mode 100644 components/inference-engines/src/libtorch/LibtorchInfer.hpp delete mode 100644 components/inference-engines/src/onnx-runtime/ORTInfer.cpp delete mode 100644 components/inference-engines/src/onnx-runtime/ORTInfer.hpp delete mode 100644 components/inference-engines/src/opencv-dnn/OCVDNNInfer.cpp delete mode 100644 components/inference-engines/src/opencv-dnn/OCVDNNInfer.hpp delete mode 100644 components/inference-engines/src/openvino/OVInfer.cpp delete mode 100644 components/inference-engines/src/openvino/OVInfer.hpp delete mode 100644 components/inference-engines/src/tensorrt/Logger.hpp delete mode 100644 components/inference-engines/src/tensorrt/TRTInfer.cpp delete mode 100644 components/inference-engines/src/tensorrt/TRTInfer.hpp delete mode 100644 inc/InferenceBackendSetup.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c07fe4..ab52511 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,55 +8,39 @@ find_package(spdlog REQUIRED) message(STATUS "Home path: $ENV{HOME}") - -# Define the default backend if not set from the command line -if(NOT DEFINED DEFAULT_BACKEND) - unset(DEFAULT_BACKEND CACHE) - set(DEFAULT_BACKEND "LIBTORCH" CACHE STRING "Default backend for inference" FORCE) -endif() - -# Define the supported backends -set(SUPPORTED_BACKENDS "ONNX_RUNTIME" "LIBTORCH" "LIBTENSORFLOW" "OPENCV_DNN" "TENSORRT" "OPENVINO") - -# Check if the specified backend is supported -list(FIND SUPPORTED_BACKENDS ${DEFAULT_BACKEND} SUPPORTED_BACKEND_INDEX) -if (SUPPORTED_BACKEND_INDEX EQUAL -1) - message(STATUS "Unsupported default backend: ${DEFAULT_BACKEND}") - set(DEFAULT_BACKEND "OPENCV_DNN" CACHE STRING "Default backend for inference" FORCE) -endif() - -message(STATUS "Default backend: ${DEFAULT_BACKEND}") - # Fetch the VideoCapture project from GitHub include(FetchContent) FetchContent_Declare( - VideoCaptureLib + VideoCapture GIT_REPOSITORY https://github.com/olibartfast/VideoCapture.git GIT_TAG master # or the specific tag/branch you want to use ) - -FetchContent_MakeAvailable(VideoCaptureLib) +FetchContent_MakeAvailable(VideoCapture) +message(STATUS "VideoCapture_SOURCE_DIR module path: ${VideoCapture_SOURCE_DIR}") -message(STATUS "VideoCaptureLib_SOURCE_DIR module path: ${VideoCaptureLib_SOURCE_DIR}") +# Fetch the InferenceEngines project from GitHub +include(FetchContent) +FetchContent_Declare( + InferenceEngines + GIT_REPOSITORY https://github.com/olibartfast/inference-engines.git + GIT_TAG master # or the specific tag/branch you want to use +) +FetchContent_MakeAvailable(InferenceEngines) +message(STATUS "InferenceEngines_SOURCE_DIR module path: ${InferenceEngines_SOURCE_DIR}") set(DETECTORS_ROOT ${CMAKE_CURRENT_LIST_DIR}/components/detectors) -set(INFER_ROOT ${CMAKE_CURRENT_LIST_DIR}/components/inference-engines) list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake) message(STATUS "Cmake module path: ${CMAKE_MODULE_PATH}") # Add subdirectories add_subdirectory(${DETECTORS_ROOT}) -add_subdirectory(${INFER_ROOT}) + +include(AddCompileDefinitions) # Main executable set(SOURCES main.cpp) - - - -include(SelectBackend) - add_executable(${PROJECT_NAME} ${SOURCES}) # Include directories and link libraries @@ -64,9 +48,10 @@ target_include_directories(${PROJECT_NAME} PRIVATE inc ${OpenCV_INCLUDE_DIRS} ${spdlog_INCLUDE_DIRS} - ${VideoCaptureLib_SOURCE_DIR}/include - ${VideoCaptureLib_SOURCE_DIR}/src - components/inference-engines/src + ${VideoCapture_SOURCE_DIR}/include + ${VideoCapture_SOURCE_DIR}/src + ${InferenceEngines_SOURCE_DIR}/include + ${InferenceEngines_SOURCE_DIR}/src components/detectors/src ) @@ -74,10 +59,9 @@ target_link_libraries(${PROJECT_NAME} PRIVATE spdlog::spdlog_header_only ${OpenCV_LIBS} DetectorsLib - InferenceEnginesLib - VideoCaptureLib + InferenceEngines + VideoCapture ) - include(LinkBackend) -include(SetCompilerFlags) +include(SetCompilerFlags) \ No newline at end of file diff --git a/cmake/AddCompileDefinitions.cmake b/cmake/AddCompileDefinitions.cmake new file mode 100644 index 0000000..6a0af7d --- /dev/null +++ b/cmake/AddCompileDefinitions.cmake @@ -0,0 +1,33 @@ +if(DEFAULT_BACKEND STREQUAL "OPENCV_DNN") + add_compile_definitions(USE_OPENCV_DNN) +elseif (DEFAULT_BACKEND STREQUAL "ONNX_RUNTIME") + set(ORT_VERSION "1.15.1" CACHE STRING "Onnx runtime version") # modify accordingly + set(ONNX_RUNTIME_DIR $ENV{HOME}/onnxruntime-linux-x64-gpu-${ORT_VERSION} CACHE PATH "Path to onnxruntime") + message(STATUS "Onnx runtime version: ${ORT_VERSION}") + message(STATUS "Onnx runtime directory: ${ONNX_RUNTIME_DIR}") + find_package(CUDA) + if (CUDA_FOUND) + message(STATUS "Found CUDA") + set(CUDA_TOOLKIT_ROOT_DIR /usr/local/cuda) + else () + message(WARNING "CUDA not found. GPU support will be disabled.") + endif() + add_compile_definitions(USE_ONNX_RUNTIME) +elseif (DEFAULT_BACKEND STREQUAL "LIBTORCH") + set(Torch_DIR $ENV{HOME}/libtorch/share/cmake/Torch/ CACHE PATH "Path to libtorch") + find_package(Torch REQUIRED) + add_compile_definitions(USE_LIBTORCH) +elseif (DEFAULT_BACKEND STREQUAL "TENSORRT") + set(TRT_VERSION "8.6.1.6" CACHE STRING "Tensorrt version") # modify accordingly + set(TENSORRT_DIR $ENV{HOME}/TensorRT-${TRT_VERSION}/) + message(STATUS "TENSORRT_DIR: ${TENSORRT_DIR}") + find_package(CUDA REQUIRED) + include(QueryGpu) + add_compile_definitions(USE_TENSORRT) +elseif (DEFAULT_BACKEND STREQUAL "LIBTENSORFLOW") + find_package(TensorFlow REQUIRED) + add_compile_definitions(USE_LIBTENSORFLOW) +elseif (DEFAULT_BACKEND STREQUAL "OPENVINO") + find_package(OpenVINO REQUIRED) + add_compile_definitions(USE_OPENVINO) +endif() \ No newline at end of file diff --git a/cmake/LibTensorFlow.cmake b/cmake/LibTensorFlow.cmake deleted file mode 100644 index f211ac3..0000000 --- a/cmake/LibTensorFlow.cmake +++ /dev/null @@ -1,16 +0,0 @@ -# TensorFlow Configuration - -# Find TensorFlow -find_package(TensorFlow REQUIRED) - - -set(TensorFlow_SOURCES -${INFER_ROOT}/src/libtensorflow/TFDetectionAPI.cpp -) - -list(APPEND SOURCES ${TensorFlow_SOURCES}) - - -# Add compile definition to indicate TensorFlow usage -add_compile_definitions(USE_LIBTENSORFLOW) - diff --git a/cmake/LibTorch.cmake b/cmake/LibTorch.cmake deleted file mode 100644 index 87772fe..0000000 --- a/cmake/LibTorch.cmake +++ /dev/null @@ -1,20 +0,0 @@ -# LibTorch Configuration - -# Set LibTorch directory (modify accordingly) -set(Torch_DIR $ENV{HOME}/libtorch/share/cmake/Torch/ CACHE PATH "Path to libtorch") - -# Find LibTorch -find_package(Torch REQUIRED) - - -set(LIBTORCH_SOURCES - ${INFER_ROOT}/src/libtorch/LibtorchInfer.cpp - # Add more LibTorch source files here if needed -) - -# Append ONNX Runtime sources to the main sources -list(APPEND SOURCES ${LIBTORCH_SOURCES}) - - -# Add compile definition to indicate LibTorch usage -add_compile_definitions(USE_LIBTORCH) diff --git a/cmake/LinkBackend.cmake b/cmake/LinkBackend.cmake index 6d5de74..865c712 100644 --- a/cmake/LinkBackend.cmake +++ b/cmake/LinkBackend.cmake @@ -1,21 +1,21 @@ # Include framework-specific source files and libraries if (DEFAULT_BACKEND STREQUAL "OPENCV_DNN") - target_include_directories(${PROJECT_NAME} PRIVATE ${INFER_ROOT}/src/opencv-dnn) + target_include_directories(${PROJECT_NAME} PRIVATE ${InferenceEngines_SOURCE_DIR}/src/opencv-dnn) elseif (DEFAULT_BACKEND STREQUAL "ONNX_RUNTIME") - target_include_directories(${PROJECT_NAME} PRIVATE ${ONNX_RUNTIME_DIR}/include ${INFER_ROOT}/src/onnx-runtime) + target_include_directories(${PROJECT_NAME} PRIVATE ${ONNX_RUNTIME_DIR}/include ${InferenceEngines_SOURCE_DIR}/src/onnx-runtime) target_link_directories(${PROJECT_NAME} PRIVATE ${ONNX_RUNTIME_DIR}/lib) target_link_libraries(${PROJECT_NAME} PRIVATE ${ONNX_RUNTIME_DIR}/lib/libonnxruntime.so) elseif (DEFAULT_BACKEND STREQUAL "LIBTORCH") - target_include_directories(${PROJECT_NAME} PRIVATE ${INFER_ROOT}/src/libtorch) + target_include_directories(${PROJECT_NAME} PRIVATE ${InferenceEngines_SOURCE_DIR}/src/libtorch) target_link_libraries(${PROJECT_NAME} PRIVATE ${TORCH_LIBRARIES}) elseif (DEFAULT_BACKEND STREQUAL "TENSORRT") - target_include_directories(${PROJECT_NAME} PRIVATE /usr/local/cuda/include ${TENSORRT_DIR}/include ${INFER_ROOT}/src/tensorrt) + target_include_directories(${PROJECT_NAME} PRIVATE /usr/local/cuda/include ${TENSORRT_DIR}/include ${InferenceEngines_SOURCE_DIR}/src/tensorrt) target_link_directories(${PROJECT_NAME} PRIVATE /usr/local/cuda/lib64 ${TENSORRT_DIR}/lib) target_link_libraries(${PROJECT_NAME} PRIVATE nvinfer nvonnxparser cudart) elseif(DEFAULT_BACKEND STREQUAL "LIBTENSORFLOW" ) - target_include_directories(${PROJECT_NAME} PRIVATE ${TensorFlow_INCLUDE_DIRS} ${INFER_ROOT}/src/libtensorflow) + target_include_directories(${PROJECT_NAME} PRIVATE ${TensorFlow_INCLUDE_DIRS} ${InferenceEngines_SOURCE_DIR}/src/libtensorflow) target_link_libraries(${PROJECT_NAME} PRIVATE ${TensorFlow_LIBRARIES}) elseif(DEFAULT_BACKEND STREQUAL "OPENVINO") - target_include_directories(${PROJECT_NAME} PRIVATE ${InferenceEngine_INCLUDE_DIRS} ${INFER_ROOT}/src/openvino) + target_include_directories(${PROJECT_NAME} PRIVATE ${InferenceEngine_INCLUDE_DIRS} ${InferenceEngines_SOURCE_DIR}/src/openvino) target_link_libraries(${PROJECT_NAME} PRIVATE openvino::runtime ) endif() diff --git a/cmake/ONNXRuntime.cmake b/cmake/ONNXRuntime.cmake deleted file mode 100644 index a6d3467..0000000 --- a/cmake/ONNXRuntime.cmake +++ /dev/null @@ -1,30 +0,0 @@ -# ONNX Runtime Configuration -# Set ONNX Runtime -set(ORT_VERSION "1.15.1" CACHE STRING "Onnx runtime version") # modify accordingly -set(ONNX_RUNTIME_DIR $ENV{HOME}/onnxruntime-linux-x64-gpu-${ORT_VERSION} CACHE PATH "Path to onnxruntime") -message(STATUS "Onnx runtime version: ${ORT_VERSION}") - -# Set ONNX Runtime directory (modify accordingly) -message(STATUS "Onnx runtime directory: ${ONNX_RUNTIME_DIR}") - - -# Find CUDA (if available) -find_package(CUDA) -if (CUDA_FOUND) - message(STATUS "Found CUDA") - set(CUDA_TOOLKIT_ROOT_DIR /usr/local/cuda) -else () - message(WARNING "CUDA not found. GPU support will be disabled.") -endif() - -# Define ONNX Runtime-specific source files -set(ONNX_RUNTIME_SOURCES - ${INFER_ROOT}/src/onnx-runtime/ORTInfer.cpp - # Add more ONNX Runtime source files here if needed -) - -# Append ONNX Runtime sources to the main sources -list(APPEND SOURCES ${ONNX_RUNTIME_SOURCES}) - -# Add compile definition to indicate ONNX Runtime usage -add_compile_definitions(USE_ONNX_RUNTIME) \ No newline at end of file diff --git a/cmake/OpenCVdnn.cmake b/cmake/OpenCVdnn.cmake deleted file mode 100644 index 8e7f38c..0000000 --- a/cmake/OpenCVdnn.cmake +++ /dev/null @@ -1,8 +0,0 @@ -set(OPENCV_DNN_SOURCES -${INFER_ROOT}/src/opencv-dnn/OCVDNNInfer.cpp -# Add more OpenCV DNN source files here if needed -) - -list(APPEND SOURCES ${OPENCV_DNN_SOURCES}) - -add_compile_definitions(USE_OPENCV_DNN) \ No newline at end of file diff --git a/cmake/OpenVino.cmake b/cmake/OpenVino.cmake deleted file mode 100644 index fc93852..0000000 --- a/cmake/OpenVino.cmake +++ /dev/null @@ -1,11 +0,0 @@ -set(OPENVINO_SOURCES -${INFER_ROOT}/src/openvino/OVInfer.cpp -# Add more OPENVINO source files here if needed -) - -find_package(OpenVINO REQUIRED) - - -list(APPEND SOURCES ${OPENVINO_SOURCES}) - -add_compile_definitions(USE_OPENVINO) \ No newline at end of file diff --git a/cmake/TensorRT.cmake b/cmake/QueryGpu.cmake similarity index 74% rename from cmake/TensorRT.cmake rename to cmake/QueryGpu.cmake index bef45e9..32bc96b 100644 --- a/cmake/TensorRT.cmake +++ b/cmake/QueryGpu.cmake @@ -1,13 +1,3 @@ -# TensorRT Configuration - -set(TRT_VERSION "8.6.1.6" CACHE STRING "Tensorrt version") # modify accordingly -# Set TensorRT directory (modify accordingly) -set(TENSORRT_DIR $ENV{HOME}/TensorRT-${TRT_VERSION}/) - -message(STATUS "TENSORRT_DIR: ${TENSORRT_DIR}") - -# Find CUDA -find_package(CUDA REQUIRED) execute_process( COMMAND nvidia-smi --query-gpu=compute_cap --format=csv,noheader OUTPUT_VARIABLE GPU_COMPUTE_CAP @@ -44,15 +34,3 @@ if (GPU_COMPUTE_CAP_RESULT EQUAL 0) else() message("Failed to query GPU compute capability.") endif() - - - - -set(TENSORRT_SOURCES - ${INFER_ROOT}/src/tensorrt/TRTInfer.cpp - # Add more TensorRT source files here if needed -) -list(APPEND SOURCES ${TENSORRT_SOURCES}) - -# Add compile definition to indicate TensorRT usage -add_compile_definitions(USE_TENSORRT) diff --git a/cmake/SelectBackend.cmake b/cmake/SelectBackend.cmake deleted file mode 100644 index 4b72fd6..0000000 --- a/cmake/SelectBackend.cmake +++ /dev/null @@ -1,20 +0,0 @@ -# Unset cache compiler definitions for the selected framework -if(DEFAULT_BACKEND STREQUAL "OPENCV_DNN") - include(OpenCVdnn) -elseif (DEFAULT_BACKEND STREQUAL "ONNX_RUNTIME") - # Set ONNX Runtime - include(ONNXRuntime) -elseif (DEFAULT_BACKEND STREQUAL "LIBTORCH") - # Set libtorch - include(LibTorch) -elseif (DEFAULT_BACKEND STREQUAL "TENSORRT") - # Set tensorrt - include(TensorRT) -elseif (DEFAULT_BACKEND STREQUAL "LIBTENSORFLOW") - # Set TensorFlow - include(LibTensorFlow) -elseif (DEFAULT_BACKEND STREQUAL "OPENVINO") - # Set OpenVino - include(OpenVino) -endif() - diff --git a/cmake/SetCompilerFlags.cmake b/cmake/SetCompilerFlags.cmake index 4af43d1..20b3e17 100644 --- a/cmake/SetCompilerFlags.cmake +++ b/cmake/SetCompilerFlags.cmake @@ -27,4 +27,4 @@ set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CUDA_ARCH_FLAG}") message("CMake CXX Flags Debug: ${CMAKE_CXX_FLAGS_DEBUG}") message("CMake CXX Flags: ${CMAKE_CXX_FLAGS}") -message("CMake CUDA Flags: ${CMAKE_CUDA_FLAGS}") +message("CMake CUDA Flags: ${CMAKE_CUDA_FLAGS}") \ No newline at end of file diff --git a/components/inference-engines/CMakeLists.txt b/components/inference-engines/CMakeLists.txt deleted file mode 100644 index a9a7ff6..0000000 --- a/components/inference-engines/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ - -# Add source files for inference engines -set(INFERENCE_SOURCES src/InferenceInterface.cpp) - -# Create the inference engines library -add_library(InferenceEnginesLib SHARED ${INFERENCE_SOURCES}) - -target_include_directories(InferenceEnginesLib PRIVATE - ${CMAKE_SOURCE_DIR}/inc - ${OpenCV_INCLUDE_DIRS} - ${spdlog_INCLUDE_DIRS} - inference-engines -) - -target_link_libraries(DetectorsLib PRIVATE - ${OpenCV_LIBS} - spdlog::spdlog_header_only -) diff --git a/components/inference-engines/src/InferenceInterface.cpp b/components/inference-engines/src/InferenceInterface.cpp deleted file mode 100644 index 3b0f4a8..0000000 --- a/components/inference-engines/src/InferenceInterface.cpp +++ /dev/null @@ -1,25 +0,0 @@ -#include "InferenceInterface.hpp" - -std::shared_ptr InferenceInterface::logger_; - - -std::vector InferenceInterface::blob2vec(const cv::Mat& input_blob) -{ - - const auto channels = input_blob.size[1]; - const auto network_width = input_blob.size[2]; - const auto network_height = input_blob.size[3]; - size_t img_byte_size = network_width * network_height * channels * sizeof(float); // Allocate a buffer to hold all image elements. - std::vector input_data = std::vector(network_width * network_height * channels); - std::memcpy(input_data.data(), input_blob.data, img_byte_size); - - std::vector chw; - for (size_t i = 0; i < channels; ++i) - { - chw.emplace_back(cv::Mat(cv::Size(network_width, network_height), CV_32FC1, &(input_data[i * network_width * network_height]))); - } - cv::split(input_blob, chw); - - return input_data; -} - diff --git a/components/inference-engines/src/InferenceInterface.hpp b/components/inference-engines/src/InferenceInterface.hpp deleted file mode 100644 index 3a1443a..0000000 --- a/components/inference-engines/src/InferenceInterface.hpp +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once -#include "common.hpp" - -class InferenceInterface{ - - public: - InferenceInterface(const std::string& weights, const std::string& modelConfiguration, - bool use_gpu = false) - { - - } - - - - static void SetLogger(const std::shared_ptr& logger) - { - logger_ = logger; - } - - - virtual std::tuple>, std::vector>> get_infer_results(const cv::Mat& input_blob) = 0; - - protected: - std::vector blob2vec(const cv::Mat& input_blob); - static std::shared_ptr logger_; - - - -}; \ No newline at end of file diff --git a/components/inference-engines/src/libtensorflow/TFDetectionAPI.cpp b/components/inference-engines/src/libtensorflow/TFDetectionAPI.cpp deleted file mode 100644 index 4e62a50..0000000 --- a/components/inference-engines/src/libtensorflow/TFDetectionAPI.cpp +++ /dev/null @@ -1,51 +0,0 @@ -#include "TFDetectionAPI.hpp" - - -std::tuple>, std::vector>> TFDetectionAPI::get_infer_results(const cv::Mat& input_blob) -{ - // Convert the frame to a TensorFlow tensor - tensorflow::Tensor input_tensor(tensorflow::DT_UINT8, tensorflow::TensorShape({1, input_blob.size[1], input_blob.size[2], input_blob.size[3]})); // NHWC - - std::memcpy(input_tensor.flat().data(), input_blob.data, input_blob.total() * input_blob.elemSize()); - - // Run the inference - std::vector> inputs = { - {"serving_default_input_tensor:0", input_tensor} - }; - std::vector outputs; - tensorflow::Status status = session_->Run(inputs, {"StatefulPartitionedCall:0", "StatefulPartitionedCall:1", "StatefulPartitionedCall:2", "StatefulPartitionedCall:3", "StatefulPartitionedCall:4"}, {}, &outputs); - if (!status.ok()) { - std::cout << "Error running session: " << status.ToString() << "\n"; - std::exit(1); - } - - std::vector> convertedOutputs; - std::vector> shapes; - - for (const auto& tensor : outputs) { - std::vector outputData; - if (tensor.dtype() == tensorflow::DataType::DT_FLOAT) { - for (int i = 0; i < tensor.NumElements(); ++i) { - // Convert tensor elements to std::any - outputData.push_back(tensor.flat()(i)); - } - } else if (tensor.dtype() == tensorflow::DataType::DT_INT32) { - for (int i = 0; i < tensor.NumElements(); ++i) { - // Convert tensor elements to std::any - outputData.push_back(tensor.flat()(i)); - } - } else if (tensor.dtype() == tensorflow::DataType::DT_INT64) { - for (int i = 0; i < tensor.NumElements(); ++i) { - // Convert tensor elements to std::any - outputData.push_back(tensor.flat()(i)); - } - } else { - std::cerr << "Unsupported output data type" << std::endl; - } - convertedOutputs.push_back(outputData); - // Assuming all output shapes are the same - std::vector outputShape = { tensor.dim_size(0), tensor.dim_size(1), tensor.dim_size(2), tensor.dim_size(3) }; - shapes.push_back(outputShape); - } - return std::make_tuple(convertedOutputs, shapes); -} \ No newline at end of file diff --git a/components/inference-engines/src/libtensorflow/TFDetectionAPI.hpp b/components/inference-engines/src/libtensorflow/TFDetectionAPI.hpp deleted file mode 100644 index 95dd6de..0000000 --- a/components/inference-engines/src/libtensorflow/TFDetectionAPI.hpp +++ /dev/null @@ -1,43 +0,0 @@ -#pragma once -#include "InferenceInterface.hpp" -#include -#include -#include -#include -#include "opencv2/opencv.hpp" - -class TFDetectionAPI : public InferenceInterface{ - -public: - TFDetectionAPI(const std::string& model_path, bool use_gpu) : InferenceInterface{model_path, "", use_gpu} - { - tensorflow::SessionOptions session_options; - tensorflow::RunOptions run_options; - tensorflow::Status status = LoadSavedModel(session_options, run_options, - model_path, {tensorflow::kSavedModelTagServe}, &bundle_); - - if (!status.ok()) { - std::cout << "Error loading SavedModel: " << status.ToString() << "\n"; - std::exit(1); - } - - // Create a new session and attach the graph - session_.reset(bundle_.session.get()); - - } - - ~TFDetectionAPI() { - tensorflow::Status status = session_->Close(); - if (!status.ok()) { - std::cerr << "Error closing TensorFlow session: " << status.ToString() << std::endl; - } - } - -private: - - std::string model_path_; - tensorflow::SavedModelBundle bundle_; - std::unique_ptr session_; - - std::tuple>, std::vector>> get_infer_results(const cv::Mat& input_blob) override; -}; \ No newline at end of file diff --git a/components/inference-engines/src/libtorch/LibtorchInfer.cpp b/components/inference-engines/src/libtorch/LibtorchInfer.cpp deleted file mode 100644 index 7cf5f12..0000000 --- a/components/inference-engines/src/libtorch/LibtorchInfer.cpp +++ /dev/null @@ -1,108 +0,0 @@ -#include "LibtorchInfer.hpp" - - -LibtorchInfer::LibtorchInfer(const std::string& model_path, bool use_gpu) : InferenceInterface{model_path, "", use_gpu} -{ - if (use_gpu && torch::cuda::is_available()) - { - device_ = torch::kCUDA; - logger_->info("Using CUDA GPU"); - } - else - { - device_ = torch::kCPU; - logger_->info("Using CPU"); - } - - module_ = torch::jit::load(model_path, device_); - -} - -std::tuple>, std::vector>> LibtorchInfer::get_infer_results(const cv::Mat& input_blob) -{ - - // Convert the input tensor to a Torch tensor - torch::Tensor input = torch::from_blob(input_blob.data, { 1, input_blob.size[1], input_blob.size[2], input_blob.size[3] }, torch::kFloat32); - input = input.to(device_); - - // Run inference - std::vector inputs; - inputs.push_back(input); - auto output = module_.forward(inputs); - - std::vector> output_vectors; - std::vector> shape_vectors; - - if (output.isTuple()) { - // Handle the case where the model returns a tuple - auto tuple_outputs = output.toTuple()->elements(); - - for (const auto& output_tensor : tuple_outputs) { - if(!output_tensor.isTensor()) - continue; - torch::Tensor tensor = output_tensor.toTensor().to(torch::kCPU).contiguous(); - - // Get the output data type - torch::ScalarType data_type = tensor.scalar_type(); - - // Store the output data based on its type - std::vector tensor_data; - if (data_type == torch::kFloat32) { - const float* output_data = tensor.data_ptr(); - tensor_data.reserve(tensor.numel()); - for (size_t i = 0; i < tensor.numel(); ++i) { - tensor_data.emplace_back(output_data[i]); - } - } else if (data_type == torch::kInt64) { - const int64_t* output_data = tensor.data_ptr(); - tensor_data.reserve(tensor.numel()); - for (size_t i = 0; i < tensor.numel(); ++i) { - tensor_data.emplace_back(output_data[i]); - } - } else { - // Handle other data types if needed - std::exit(1); - } - - // Store the output data in the outputs vector - output_vectors.push_back(tensor_data); - - // Get the shape of the output tensor - std::vector shape = tensor.sizes().vec(); - shape_vectors.push_back(shape); - } - } else { - torch::Tensor tensor = output.toTensor().to(torch::kCPU).contiguous(); - - // Get the output data type - torch::ScalarType data_type = tensor.scalar_type(); - - // Store the output data based on its type - std::vector tensor_data; - if (data_type == torch::kFloat32) { - const float* output_data = tensor.data_ptr(); - tensor_data.reserve(tensor.numel()); - for (size_t i = 0; i < tensor.numel(); ++i) { - tensor_data.emplace_back(output_data[i]); - } - } else if (data_type == torch::kInt64) { - const int64_t* output_data = tensor.data_ptr(); - tensor_data.reserve(tensor.numel()); - for (size_t i = 0; i < tensor.numel(); ++i) { - tensor_data.emplace_back(output_data[i]); - } - } else { - // Handle other data types if needed - std::exit(1); - } - - // Store the output data in the outputs vector - output_vectors.push_back(tensor_data); - - // Get the shape of the output tensor - std::vector shape = tensor.sizes().vec(); - shape_vectors.push_back(shape); - } - - return std::make_tuple(output_vectors, shape_vectors); -} diff --git a/components/inference-engines/src/libtorch/LibtorchInfer.hpp b/components/inference-engines/src/libtorch/LibtorchInfer.hpp deleted file mode 100644 index 0407f55..0000000 --- a/components/inference-engines/src/libtorch/LibtorchInfer.hpp +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once -#include "InferenceInterface.hpp" -#include -#include - -class LibtorchInfer : public InferenceInterface -{ -protected: - torch::DeviceType device_; - torch::jit::script::Module module_; - -public: - LibtorchInfer(const std::string& model_path, bool use_gpu = true); - - std::tuple>, std::vector>> get_infer_results(const cv::Mat& input_blob) override; - -}; \ No newline at end of file diff --git a/components/inference-engines/src/onnx-runtime/ORTInfer.cpp b/components/inference-engines/src/onnx-runtime/ORTInfer.cpp deleted file mode 100644 index f4b9456..0000000 --- a/components/inference-engines/src/onnx-runtime/ORTInfer.cpp +++ /dev/null @@ -1,221 +0,0 @@ -#include "ORTInfer.hpp" - -ORTInfer::ORTInfer(const std::string& model_path, bool use_gpu) : InferenceInterface{model_path, "", use_gpu} -{ - env_=Ort::Env(ORT_LOGGING_LEVEL_WARNING, "Onnx Runtime Inference"); - - Ort::SessionOptions session_options; - - if (use_gpu) - { - // Check if CUDA GPU is available - std::vector providers = Ort::GetAvailableProviders(); - logger_->info("Available providers:"); - bool is_found = false; - for (const auto& p : providers) - { - logger_->info("{}", p); - if (p.find("CUDA") != std::string::npos) - { - // CUDA GPU is available, use it - logger_->info("Using CUDA GPU"); - OrtCUDAProviderOptions cuda_options; - session_options.AppendExecutionProvider_CUDA(cuda_options); - is_found = true; - break; - } - } - if (!is_found) - { - // CUDA GPU is not available, fall back to CPU - logger_->info("CUDA GPU not available, falling back to CPU"); - session_options = Ort::SessionOptions(); - } - } - else - { - logger_->info("Using CPU"); - session_options = Ort::SessionOptions(); - } - - try - { - session_ = Ort::Session(env_, model_path.c_str(), session_options); - } - catch (const Ort::Exception& ex) - { - logger_->error("Failed to load the ONNX model: {}", ex.what()); - std::exit(1); - } - - Ort::AllocatorWithDefaultOptions allocator; - logger_->info("Input Node Name/Shape ({}):", session_.GetInputCount()); - for (std::size_t i = 0; i < session_.GetInputCount(); i++) - { - input_names_.emplace_back(session_.GetInputNameAllocated(i, allocator).get()); - auto input_shapes = session_.GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape(); - auto input_type = session_.GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetElementType(); - logger_->info("\t{} : {}", input_names_.at(i), print_shape(input_shapes)); - input_shapes[0] = input_shapes[0] == -1 ? 1 : input_shapes[0]; - input_shapes_.emplace_back(input_shapes); - - std::string input_type_str; - switch (input_type) - { - case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: - input_type_str = "Float"; - break; - case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: - input_type_str = "Int64"; - break; - // Handle other data types as needed - default: - input_type_str = "Unknown"; - } - - // Log the type - logger_->info("\tData Type: {}", input_type_str); - } - - const auto network_width = static_cast(input_shapes_[0][3]); - const auto network_height = static_cast(input_shapes_[0][2]); - const auto channels = static_cast(input_shapes_[0][1]); - - logger_->info("channels {}", channels); - logger_->info("width {}", network_width); - logger_->info("height {}", network_height); - - // print name/shape of outputs - logger_->info("Output Node Name/Shape ({}):", session_.GetOutputCount()); - for (std::size_t i = 0; i < session_.GetOutputCount(); i++) - { - output_names_.emplace_back(session_.GetOutputNameAllocated(i, allocator).get()); - auto output_shapes = session_.GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape(); - logger_->info("\t{} : {}", output_names_.at(i), print_shape(output_shapes)); - output_shapes_.emplace_back(output_shapes); - } -} - -std::string ORTInfer::print_shape(const std::vector& v) -{ - std::stringstream ss(""); - for (std::size_t i = 0; i < v.size() - 1; i++) - ss << v[i] << "x"; - ss << v[v.size() - 1]; - return ss.str(); -} - - -size_t ORTInfer::getSizeByDim(const std::vector& dims) -{ - size_t size = 1; - for (size_t i = 0; i < dims.size(); ++i) - { - if(dims[i] == -1 || dims[i] == 0) - { - continue; - } - size *= dims[i]; - } - return size; -} - - -std::tuple>, std::vector>> ORTInfer::get_infer_results(const cv::Mat& input_blob) -{ - std::vector> outputs; - std::vector> shapes; - std::vector> input_tensors(session_.GetInputCount()); - std::vector in_ort_tensors; - Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault); - std::vector orig_target_sizes; - - input_tensors[0] = blob2vec(input_blob); - in_ort_tensors.emplace_back(Ort::Value::CreateTensor( - memory_info, - input_tensors[0].data(), - getSizeByDim(input_shapes_[0]), - input_shapes_[0].data(), - input_shapes_[0].size() - )); - - // RTDETR case, two inputs - if(input_tensors.size() > 1) - { - orig_target_sizes = { static_cast(input_blob.size[2]), static_cast(input_blob.size[3]) }; - // Assuming input_tensors[1] is of type int64 - in_ort_tensors.emplace_back(Ort::Value::CreateTensor( - memory_info, - orig_target_sizes.data(), - getSizeByDim( orig_target_sizes), - input_shapes_[1].data(), - input_shapes_[1].size() - )); - } - - - // Run inference - std::vector input_names_char(input_names_.size()); - std::transform(input_names_.begin(), input_names_.end(), input_names_char.begin(), - [](const std::string& str) { return str.c_str(); }); - - std::vector output_names_char(output_names_.size()); - std::transform(output_names_.begin(), output_names_.end(), output_names_char.begin(), - [](const std::string& str) { return str.c_str(); }); - - std::vector output_ort_tensors = session_.Run( - Ort::RunOptions{ nullptr }, - input_names_char.data(), - in_ort_tensors.data(), - in_ort_tensors.size(), - output_names_char.data(), - output_names_.size() - ); - - // Process output tensors - assert(output_ort_tensors.size() == output_names_.size()); - - for (const Ort::Value& output_tensor : output_ort_tensors) - { - const auto& shape_ref = output_tensor.GetTensorTypeAndShapeInfo().GetShape(); - std::vector shape(shape_ref.begin(), shape_ref.end()); - - size_t num_elements = 1; - for (int64_t dim : shape) { - num_elements *= dim; - } - - std::vector tensor_data; - - // Retrieve tensor data - const int onnx_type = output_tensor.GetTensorTypeAndShapeInfo().GetElementType(); - switch(onnx_type) { - case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: { - const float* output_data_float = output_tensor.GetTensorData(); - tensor_data.reserve(num_elements); - for (size_t i = 0; i < num_elements; ++i) { - tensor_data.emplace_back(output_data_float[i]); - } - break; - } - case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: { - const int64_t* output_data_int64 = output_tensor.GetTensorData(); - tensor_data.reserve(num_elements); - for (size_t i = 0; i < num_elements; ++i) { - tensor_data.emplace_back(output_data_int64[i]); - } - break; - } - // Add cases for other data types as needed - default: - std::exit(1); - } - - // Store the tensor data in outputs - outputs.emplace_back(tensor_data); - shapes.emplace_back(shape); - } - - return std::make_tuple(outputs, shapes); -} - diff --git a/components/inference-engines/src/onnx-runtime/ORTInfer.hpp b/components/inference-engines/src/onnx-runtime/ORTInfer.hpp deleted file mode 100644 index 8d11010..0000000 --- a/components/inference-engines/src/onnx-runtime/ORTInfer.hpp +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once -#include "InferenceInterface.hpp" -#include // for ONNX Runtime C++ API -#include // for CUDA execution provider (if using CUDA) - -class ORTInfer : public InferenceInterface -{ -private: - Ort::Env env_; - Ort::Session session_{ nullptr }; - std::vector input_names_; // Store input layer names - std::vector output_names_; // Store output layer names - std::vector> input_shapes_; - std::vector> output_shapes_; - -public: - std::string print_shape(const std::vector& v); - ORTInfer(const std::string& model_path, bool use_gpu = false); - size_t getSizeByDim(const std::vector& dims); - - std::tuple>, std::vector>> get_infer_results(const cv::Mat& input_blob) override; -}; \ No newline at end of file diff --git a/components/inference-engines/src/opencv-dnn/OCVDNNInfer.cpp b/components/inference-engines/src/opencv-dnn/OCVDNNInfer.cpp deleted file mode 100644 index d2149e5..0000000 --- a/components/inference-engines/src/opencv-dnn/OCVDNNInfer.cpp +++ /dev/null @@ -1,57 +0,0 @@ -#include "OCVDNNInfer.hpp" - -OCVDNNInfer::OCVDNNInfer(const std::string& weights, const std::string& modelConfiguration) : InferenceInterface{weights, modelConfiguration} -{ - - logger_->info("Running {} using OpenCV DNN runtime", weights); - net_ = modelConfiguration.empty() ? cv::dnn::readNet(weights) : cv::dnn::readNetFromDarknet(modelConfiguration, weights); - if (net_.empty()) - { - std::cerr << "Can't load network by using the following files: " << std::endl; - std::cerr << "weights-file: " << weights << std::endl; - exit(-1); - } - outLayers_ = net_.getUnconnectedOutLayers(); - outLayerType_ = net_.getLayer(outLayers_[0])->type; - outNames_ = net_.getUnconnectedOutLayersNames(); - - -} - - -std::tuple>, std::vector>> OCVDNNInfer::get_infer_results(const cv::Mat& input_blob) -{ - - std::vector> outputs; - std::vector> shapes; - std::vector layerTypes; - - std::vector outs; - net_.setInput(input_blob); - net_.forward(outs, outNames_); - - for (size_t i = 0; i < outs.size(); ++i) { - const auto& output = outs[i]; - // Extracting dimensions of the output tensor - std::vector shape; - for (int j = 0; j < output.dims; ++j) { - shape.push_back(output.size[j]); - } - shapes.push_back(shape); - - // Extracting data - if (output.type() == CV_32F) { - const float* data = output.ptr(); - outputs.emplace_back(data, data + output.total()); - } - else if (output.type() == CV_64F) { - const int64_t* data = output.ptr(); - outputs.emplace_back(data, data + output.total()); - } - else { - std::cerr << "Unsupported data type\n"; - } - } - - return std::make_tuple(outputs, shapes); -} \ No newline at end of file diff --git a/components/inference-engines/src/opencv-dnn/OCVDNNInfer.hpp b/components/inference-engines/src/opencv-dnn/OCVDNNInfer.hpp deleted file mode 100644 index d77fd62..0000000 --- a/components/inference-engines/src/opencv-dnn/OCVDNNInfer.hpp +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once -#include "InferenceInterface.hpp" - -class OCVDNNInfer : public InferenceInterface -{ -private: - cv::dnn::Net net_; - std::vector outLayers_; - std::string outLayerType_; - std::vector outNames_; - -public: - OCVDNNInfer(const std::string& weights, const std::string& modelConfiguration = ""); - - std::tuple>, std::vector>> get_infer_results(const cv::Mat& input_blob) override; -}; diff --git a/components/inference-engines/src/openvino/OVInfer.cpp b/components/inference-engines/src/openvino/OVInfer.cpp deleted file mode 100644 index a2621a5..0000000 --- a/components/inference-engines/src/openvino/OVInfer.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include "OVInfer.hpp" - -OVInfer::OVInfer(const std::string& model_path, const std::string& model_config, bool use_gpu) : - InferenceInterface{model_path, model_config, use_gpu} -{ - - model_ = core_.read_model(model_config); - compiled_model_ = core_.compile_model(model_); - infer_request_ = compiled_model_.create_infer_request(); - ov::Shape s = compiled_model_.input().get_shape(); -} - -std::tuple>, std::vector>> OVInfer::get_infer_results(const cv::Mat& input_blob) -{ - - std::vector> outputs; - std::vector> shapes; - - ov::Tensor input_tensor(compiled_model_.input().get_element_type(), compiled_model_.input().get_shape(), input_blob.data); - // Set input tensor for model with one input - infer_request_.set_input_tensor(input_tensor); - infer_request_.infer(); - auto output_tensor = infer_request_.get_output_tensor(); - const float *output_buffer = output_tensor.data(); - std::size_t output_size = output_tensor.get_size(); - std::vectoroutput_shape(output_tensor.get_shape().begin(), output_tensor.get_shape().end()); - std::vector output(output_buffer, output_buffer + output_size); - outputs.emplace_back(output); - shapes.emplace_back(output_shape); - return std::make_tuple(outputs, shapes); -} \ No newline at end of file diff --git a/components/inference-engines/src/openvino/OVInfer.hpp b/components/inference-engines/src/openvino/OVInfer.hpp deleted file mode 100644 index af18bf1..0000000 --- a/components/inference-engines/src/openvino/OVInfer.hpp +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once -#include "InferenceInterface.hpp" -#include - - -class OVInfer : public InferenceInterface -{ -protected: - - -public: - OVInfer(const std::string& model_path = "", const std::string& modelConfiguration = "", bool use_gpu = true); - - std::tuple>, std::vector>> get_infer_results(const cv::Mat& input_blob) override; - - ov::Core core_; - ov::Tensor input_tensor_; - ov::InferRequest infer_request_; - std::shared_ptr model_; - ov::CompiledModel compiled_model_; -}; \ No newline at end of file diff --git a/components/inference-engines/src/tensorrt/Logger.hpp b/components/inference-engines/src/tensorrt/Logger.hpp deleted file mode 100644 index 0a28c90..0000000 --- a/components/inference-engines/src/tensorrt/Logger.hpp +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once -#include // for TensorRT API -class Logger : public nvinfer1::ILogger -{ -public: - void log(Severity severity, const char* msg) noexcept override - { - // Implement logging behavior here, e.g., print the log message - if (severity != Severity::kINFO) - { - std::cout << "TensorRT Logger: " << msg << std::endl; - } - } -}; \ No newline at end of file diff --git a/components/inference-engines/src/tensorrt/TRTInfer.cpp b/components/inference-engines/src/tensorrt/TRTInfer.cpp deleted file mode 100644 index bdcddb9..0000000 --- a/components/inference-engines/src/tensorrt/TRTInfer.cpp +++ /dev/null @@ -1,175 +0,0 @@ -#include "TRTInfer.hpp" - -TRTInfer::TRTInfer(const std::string& model_path) : InferenceInterface{model_path, "", true} -{ - - logger_->info("Initializing TensorRT for model {}", model_path); - initializeBuffers(model_path); -} - -void TRTInfer::initializeBuffers(const std::string& engine_path) -{ - - - // Create TensorRT runtime and deserialize engine - Logger logger; - // Create TensorRT runtime - runtime_ = nvinfer1::createInferRuntime(logger); - - // Load engine file - std::ifstream engine_file(engine_path, std::ios::binary); - if (!engine_file) - { - throw std::runtime_error("Failed to open engine file: " + engine_path); - } - engine_file.seekg(0, std::ios::end); - size_t file_size = engine_file.tellg(); - engine_file.seekg(0, std::ios::beg); - std::vector engine_data(file_size); - engine_file.read(engine_data.data(), file_size); - engine_file.close(); - - // Deserialize engine - engine_.reset( - runtime_->deserializeCudaEngine(engine_data.data(), file_size), - [](nvinfer1::ICudaEngine* engine) { engine->destroy(); }); - - - // Create execution context and allocate input/output buffers - createContextAndAllocateBuffers(); -} - -// calculate size of tensor -size_t TRTInfer::getSizeByDim(const nvinfer1::Dims& dims) -{ - size_t size = 1; - for (size_t i = 0; i < dims.nbDims; ++i) - { - if(dims.d[i] == -1 || dims.d[i] == 0) - { - continue; - } - size *= dims.d[i]; - } - return size; -} - - - -void TRTInfer::createContextAndAllocateBuffers() -{ - nvinfer1::Dims profile_dims = engine_->getProfileDimensions(0, 0, nvinfer1::OptProfileSelector::kMIN); - int max_batch_size = profile_dims.d[0]; - context_ = engine_->createExecutionContext(); - context_->setBindingDimensions(0, profile_dims); - buffers_.resize(engine_->getNbBindings()); - for (int i = 0; i < engine_->getNbBindings(); ++i) - { - nvinfer1::Dims dims = engine_->getBindingDimensions(i); - auto size = getSizeByDim(dims); - size_t binding_size; - switch (engine_->getBindingDataType(i)) - { - case nvinfer1::DataType::kFLOAT: - binding_size = size * sizeof(float); - break; - case nvinfer1::DataType::kINT32: - binding_size = size * sizeof(int); - break; - // Add more cases for other data types if needed - default: - // Handle unsupported data types - std::exit(1); - } - cudaMalloc(&buffers_[i], binding_size); - - if (engine_->bindingIsInput(i)) - { - logger_->info("Input layer {} {}",num_inputs_, engine_->getBindingName(i)); - num_inputs_++; - continue; - } - logger_->info("Output layer {} {}",num_outputs_, engine_->getBindingName(i)); - num_outputs_++; - } -} -std::tuple>, std::vector>> TRTInfer::get_infer_results(const cv::Mat& input_blob) -{ - for (size_t i = 0; i < num_inputs_; ++i) - { - nvinfer1::Dims dims = engine_->getBindingDimensions(i); - size_t size = getSizeByDim(dims); - size_t binding_size = 0; - switch (engine_->getBindingDataType(i)) - { - case nvinfer1::DataType::kFLOAT: - binding_size = size * sizeof(float); - break; - case nvinfer1::DataType::kINT32: - binding_size = size * sizeof(int32_t); - break; - // Add more cases for other data types if needed - default: - // Handle unsupported data types - std::exit(1); - break; - } - - switch(i) - { - case 0: - cudaMemcpy(buffers_[0], input_blob.data, binding_size, cudaMemcpyHostToDevice); - break; - case 1: - // in rtdetr lyuwenyu version we have a second input - std::vector orig_target_sizes = { static_cast(input_blob.size[2]), static_cast(input_blob.size[3]) }; - cudaMemcpy(buffers_[1], orig_target_sizes.data(), binding_size, cudaMemcpyHostToDevice); - break; - } - } - - if(!context_->enqueueV2(buffers_.data(), 0, nullptr)) - { - logger_->error("Forward Error !"); - std::exit(1); - } - - std::vector> output_shapes; - std::vector> outputs; - for (size_t i = 0; i < num_outputs_; ++i) - { - nvinfer1::Dims dims = engine_->getBindingDimensions(i + num_inputs_); // i + 1 to account for the input buffer - auto num_elements = getSizeByDim(dims); - std::vector tensor_data; - switch (engine_->getBindingDataType(i + num_inputs_)) - { - case nvinfer1::DataType::kFLOAT: - { - std::vector output_data_float(num_elements); - cudaMemcpy(output_data_float.data(), buffers_[i + num_inputs_], num_elements * sizeof(float), cudaMemcpyDeviceToHost); - tensor_data = std::vector(output_data_float.begin(), output_data_float.end()); - break; - } - case nvinfer1::DataType::kINT32: - { - std::vector output_data_int(num_elements); - cudaMemcpy(output_data_int.data(), buffers_[i + num_inputs_], num_elements * sizeof(int32_t), cudaMemcpyDeviceToHost); - tensor_data = std::vector(output_data_int.begin(), output_data_int.end()); - break; - } - - // Add more cases for other data types if needed - default: - // Handle unsupported data types - std::exit(1); - break; - } - outputs.emplace_back(std::move(tensor_data)); - - const int64_t curr_batch = dims.d[0] == -1 ? 1 : dims.d[0]; - const auto out_shape = std::vector{curr_batch, dims.d[1], dims.d[2], dims.d[3] }; - output_shapes.emplace_back(out_shape); - } - - return std::make_tuple(std::move(outputs), std::move(output_shapes)); -} diff --git a/components/inference-engines/src/tensorrt/TRTInfer.hpp b/components/inference-engines/src/tensorrt/TRTInfer.hpp deleted file mode 100644 index 07abe60..0000000 --- a/components/inference-engines/src/tensorrt/TRTInfer.hpp +++ /dev/null @@ -1,43 +0,0 @@ -#pragma once -#include "InferenceInterface.hpp" -#include // for TensorRT API -#include // for CUDA runtime API -#include - -#include "Logger.hpp" - -class TRTInfer : public InferenceInterface -{ - protected: - std::shared_ptr engine_{nullptr}; - nvinfer1::IExecutionContext* context_{nullptr}; - std::vector buffers_; - nvinfer1::IRuntime* runtime_{nullptr}; - size_t num_inputs_{0}; - size_t num_outputs_{0}; - - public: - TRTInfer(const std::string& model_path); - - // Create execution context and allocate input/output buffers - void createContextAndAllocateBuffers(); - - void initializeBuffers(const std::string& engine_path); - - // calculate size of tensor - size_t getSizeByDim(const nvinfer1::Dims& dims); - - void infer(); - - std::tuple>, std::vector>> get_infer_results(const cv::Mat& input_blob) override; - - ~TRTInfer() - { - for (void* buffer : buffers_) - { - cudaFree(buffer); - } - } - - -}; \ No newline at end of file diff --git a/inc/InferenceBackendSetup.hpp b/inc/InferenceBackendSetup.hpp deleted file mode 100644 index 81fd232..0000000 --- a/inc/InferenceBackendSetup.hpp +++ /dev/null @@ -1,36 +0,0 @@ -#pragma once -#include "common.hpp" -#include "InferenceInterface.hpp" -#ifdef USE_ONNX_RUNTIME -#include "ORTInfer.hpp" -#elif USE_LIBTORCH -#include "LibtorchInfer.hpp" -#elif USE_LIBTENSORFLOW -#include "TFDetectionAPI.hpp" -#elif USE_OPENCV_DNN -#include "OCVDNNInfer.hpp" -#elif USE_TENSORRT -#include "TRTInfer.hpp" -#elif USE_OPENVINO -#include "OVInfer.hpp" -#endif - -std::unique_ptr setup_inference_engine(const std::string& weights, const std::string& modelConfiguration ) -{ - #ifdef USE_ONNX_RUNTIME - return std::make_unique(weights, false); - #elif USE_LIBTORCH - return std::make_unique(weights, false); - #elif USE_LIBTENSORFLOW - return std::make_unique(weights, false); - #elif USE_OPENCV_DNN - return std::make_unique(weights, modelConfiguration); - #elif USE_TENSORRT - return std::make_unique(weights); - #elif USE_OPENVINO - return std::make_unique("", modelConfiguration, false); - #endif - return nullptr; - - -} \ No newline at end of file