From 2d181019d975fa205bcc820ff0a468ac9f87b7fd Mon Sep 17 00:00:00 2001 From: Attila Krasznahorkay Date: Tue, 12 Apr 2022 10:37:20 +0200 Subject: [PATCH] Added printouts about the CUDA device being used in some places. In order to help with debugging issues on multi-CUDA-GPU systems, added some printouts of the device name wherever it would be possible. At the same time simplified the build configuration of vecmem::cuda a bit, since the library doesn't actually build any CUDA code. --- cuda/CMakeLists.txt | 8 ++-- .../memory/cuda/device_memory_resource.cpp | 9 ++++- cuda/src/utils/get_device_name.cpp | 39 +++++++++++++++++++ cuda/src/utils/get_device_name.hpp | 29 ++++++++++++++ cuda/src/utils/select_device.cpp | 10 ++++- cuda/src/utils/select_device.hpp | 9 ++++- cuda/src/utils/stream_wrapper.cpp | 10 ++++- 7 files changed, 102 insertions(+), 12 deletions(-) create mode 100644 cuda/src/utils/get_device_name.cpp create mode 100644 cuda/src/utils/get_device_name.hpp diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt index 16217eaf..21aa53ec 100644 --- a/cuda/CMakeLists.txt +++ b/cuda/CMakeLists.txt @@ -1,18 +1,14 @@ # VecMem project, part of the ACTS project (R&D line) # -# (c) 2021 CERN for the benefit of the ACTS project +# (c) 2021-2022 CERN for the benefit of the ACTS project # # Mozilla Public License Version 2.0 # CUDAToolkit requires CMake 3.17. cmake_minimum_required( VERSION 3.17 ) -# Enable CUDA as a language. -enable_language( CUDA ) - # Project include(s). include( vecmem-compiler-options-cpp ) -include( vecmem-compiler-options-cuda ) # External dependency/dependencies. find_package( CUDAToolkit REQUIRED ) @@ -35,6 +31,8 @@ vecmem_add_library( vecmem_cuda cuda "src/utils/cuda_error_handling.cpp" "src/utils/cuda_wrappers.hpp" "src/utils/cuda_wrappers.cpp" + "src/utils/get_device_name.hpp" + "src/utils/get_device_name.cpp" "src/utils/select_device.hpp" "src/utils/select_device.cpp" "include/vecmem/utils/cuda/stream_wrapper.hpp" diff --git a/cuda/src/memory/cuda/device_memory_resource.cpp b/cuda/src/memory/cuda/device_memory_resource.cpp index c6431463..05ca2efe 100644 --- a/cuda/src/memory/cuda/device_memory_resource.cpp +++ b/cuda/src/memory/cuda/device_memory_resource.cpp @@ -1,7 +1,7 @@ /* * VecMem project, part of the ACTS project (R&D line) * - * (c) 2021 CERN for the benefit of the ACTS project + * (c) 2021-2022 CERN for the benefit of the ACTS project * * Mozilla Public License Version 2.0 */ @@ -11,6 +11,7 @@ #include "../../utils/cuda_error_handling.hpp" #include "../../utils/cuda_wrappers.hpp" +#include "../../utils/get_device_name.hpp" #include "../../utils/select_device.hpp" #include "vecmem/utils/debug.hpp" @@ -20,7 +21,11 @@ namespace vecmem::cuda { device_memory_resource::device_memory_resource(int device) - : m_device(device >= 0 ? device : details::get_device()) {} + : m_device(device >= 0 ? device : details::get_device()) { + + VECMEM_DEBUG_MSG(2, "Created device memory resource on: %s", + details::get_device_name(m_device).c_str()); +} void *device_memory_resource::do_allocate(std::size_t bytes, std::size_t) { diff --git a/cuda/src/utils/get_device_name.cpp b/cuda/src/utils/get_device_name.cpp new file mode 100644 index 00000000..c376390b --- /dev/null +++ b/cuda/src/utils/get_device_name.cpp @@ -0,0 +1,39 @@ +/* + * VecMem project, part of the ACTS project (R&D line) + * + * (c) 2022 CERN for the benefit of the ACTS project + * + * Mozilla Public License Version 2.0 + */ + +// Local include(s). +#include "get_device_name.hpp" + +// CUDA include(s). +#include + +// System include(s). +#include + +namespace vecmem { +namespace cuda { +namespace details { + +std::string get_device_name(int device) { + + // Get the device's properties. + cudaDeviceProp props; + if (cudaGetDeviceProperties(&props, device) != cudaSuccess) { + return "Unknown"; + } + + // Construct a unique name out of those properties. + std::ostringstream result; + result << props.name << " [id: " << device << ", bus: " << props.pciBusID + << ", device: " << props.pciDeviceID << "]"; + return result.str(); +} + +} // namespace details +} // namespace cuda +} // namespace vecmem diff --git a/cuda/src/utils/get_device_name.hpp b/cuda/src/utils/get_device_name.hpp new file mode 100644 index 00000000..191b346e --- /dev/null +++ b/cuda/src/utils/get_device_name.hpp @@ -0,0 +1,29 @@ +/* + * VecMem project, part of the ACTS project (R&D line) + * + * (c) 2022 CERN for the benefit of the ACTS project + * + * Mozilla Public License Version 2.0 + */ +#pragma once + +// System include(s). +#include + +namespace vecmem { +namespace cuda { +namespace details { + +/// Get a "fully qualified" name for a given CUDA device +/// +/// This function provides a uniform way for printing the names of the +/// devices that various VecMem objects would be interacting with. +/// +/// @param device The device ID that the CUDA runtime assigned +/// @return A user friently name for the device +/// +std::string get_device_name(int device); + +} // namespace details +} // namespace cuda +} // namespace vecmem diff --git a/cuda/src/utils/select_device.cpp b/cuda/src/utils/select_device.cpp index fbb2d4ad..9aed07d9 100644 --- a/cuda/src/utils/select_device.cpp +++ b/cuda/src/utils/select_device.cpp @@ -1,7 +1,7 @@ /* * VecMem project, part of the ACTS project (R&D line) * - * (c) 2021 CERN for the benefit of the ACTS project + * (c) 2021-2022 CERN for the benefit of the ACTS project * * Mozilla Public License Version 2.0 */ @@ -28,4 +28,10 @@ select_device::~select_device() { */ VECMEM_CUDA_ERROR_CHECK(cudaSetDevice(m_device)); } -} // namespace vecmem::cuda::details \ No newline at end of file + +int select_device::device() const { + + return m_device; +} + +} // namespace vecmem::cuda::details diff --git a/cuda/src/utils/select_device.hpp b/cuda/src/utils/select_device.hpp index 770db4bf..c8f0b19c 100644 --- a/cuda/src/utils/select_device.hpp +++ b/cuda/src/utils/select_device.hpp @@ -1,7 +1,7 @@ /* * VecMem project, part of the ACTS project (R&D line) * - * (c) 2021 CERN for the benefit of the ACTS project + * (c) 2021-2022 CERN for the benefit of the ACTS project * * Mozilla Public License Version 2.0 */ @@ -39,6 +39,11 @@ class select_device { */ ~select_device(); + /** + * @brief Identifier for the device being seleced + */ + int device() const; + private: /** * @brief The old device number, this is what we restore when the @@ -46,4 +51,4 @@ class select_device { */ int m_device; }; -} // namespace vecmem::cuda::details \ No newline at end of file +} // namespace vecmem::cuda::details diff --git a/cuda/src/utils/stream_wrapper.cpp b/cuda/src/utils/stream_wrapper.cpp index 816b4721..0a9fe18d 100644 --- a/cuda/src/utils/stream_wrapper.cpp +++ b/cuda/src/utils/stream_wrapper.cpp @@ -1,6 +1,6 @@ /** VecMem project, part of the ACTS project (R&D line) * - * (c) 2021 CERN for the benefit of the ACTS project + * (c) 2021-2022 CERN for the benefit of the ACTS project * * Mozilla Public License Version 2.0 */ @@ -10,9 +10,13 @@ #include "cuda_error_handling.hpp" #include "cuda_wrappers.hpp" +#include "get_device_name.hpp" #include "opaque_stream.hpp" #include "select_device.hpp" +// VecMem include(s). +#include "vecmem/utils/debug.hpp" + // CUDA include(s). #include @@ -28,6 +32,10 @@ stream_wrapper::stream_wrapper(int device) // Construct the stream. m_managedStream = std::make_shared(); m_stream = m_managedStream->m_stream; + + // Tell the user what happened. + VECMEM_DEBUG_MSG(2, "Created stream on device: %s", + details::get_device_name(dev_selector.device()).c_str()); } stream_wrapper::stream_wrapper(void* stream)