test_run_prefetcher_post_commit[device_params={'trace_region_size': 23887872}-num_reader_cores=2-num_tensors=2-input_shapes=[(256, 512), (256, 512)]-dtypes=[<DataType.BFLOAT4_B: 4>, <DataType.BFLOAT4_B: 4>]-num_layers=5] RuntimeError: TT_THROW @ /work/ttnn/cpp/ttnn/operations/global_cb_utils.hpp:36: tt::exception info: Error finding a device for a GlobalCB in MultiDeviceGlobalCircularBuffer backtrace: --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/.local/lib/python3.8/site-packages/ttnn/_ttnn.cpython-38-x86_64-linux-gnu.so(+0xb83ef8) [0x7f57c6197ef8] --- ttnn::operations::dram_prefetcher::DramPrefetcher::validate(std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>> const&) const --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/.local/lib/python3.8/site-packages/ttnn/_ttnn.cpython-38-x86_64-linux-gnu.so(_ZN4ttnn16device_operation6detail21launch_on_mesh_deviceIN2tt8tt_metal9operation23OldInfraDeviceOperationINSt3__16vectorINS4_6TensorENS7_9allocatorIS9_EEEEEENS3_3stl10StrongTypeIhNS_10QueueIdTagEEElNS5_15DeviceOperationISC_EENSD_13tensor_args_tESC_PNS4_11distributed10MeshDeviceEEEvT0_T1_RKT2_RKT3_RT4_RT5_+0x23d) [0x7f57c7ebc52d] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/.local/lib/python3.8/site-packages/ttnn/_ttnn.cpython-38-x86_64-linux-gnu.so(_ZN4ttnn16device_operation6detail23launch_on_single_deviceIN2tt8tt_metal9operation23OldInfraDeviceOperationINSt3__16vectorINS4_6TensorENS7_9allocatorIS9_EEEEEEEENT_21tensor_return_value_tENS3_3stl10StrongTypeIhNS_10QueueIdTagEEERKNSE_22operation_attributes_tERKNSE_13tensor_args_tE+0x86) [0x7f57c7ebc206] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/.local/lib/python3.8/site-packages/ttnn/_ttnn.cpython-38-x86_64-linux-gnu.so(+0x28a813a) [0x7f57c7ebc13a] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/.local/lib/python3.8/site-packages/ttnn/_ttnn.cpython-38-x86_64-linux-gnu.so(_ZN4ttnn16device_operation6detail6invokeIN2tt8tt_metal9operation23OldInfraDeviceOperationINSt3__16vectorINS4_6TensorENS7_9allocatorIS9_EEEEEEEENT_21tensor_return_value_tENS3_3stl10StrongTypeIhNS_10QueueIdTagEEERKNSE_22operation_attributes_tERKNSE_13tensor_args_tE+0x197) [0x7f57c7ebbaf7] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/.local/lib/python3.8/site-packages/ttnn/_ttnn.cpython-38-x86_64-linux-gnu.so(+0x28a749b) [0x7f57c7ebb49b] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/.local/lib/python3.8/site-packages/ttnn/_ttnn.cpython-38-x86_64-linux-gnu.so(+0x28a717b) [0x7f57c7ebb17b] --- std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>> tt::tt_metal::operation::run<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>(tt::tt_metal::operation::DeviceOperation<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>&&, std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>> const&, std::__1::vector<std::__1::optional<tt::tt_metal::Tensor const>, std::__1::allocator<std::__1::optional<tt::tt_metal::Tensor const>>> const&, std::__1::vector<std::__1::optional<tt::tt_metal::Tensor>, std::__1::allocator<std::__1::optional<tt::tt_metal::Tensor>>> const&, tt::stl::StrongType<unsigned char, ttnn::QueueIdTag>) --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/.local/lib/python3.8/site-packages/ttnn/_ttnn.cpython-38-x86_64-linux-gnu.so(+0x12a7275) [0x7f57c68bb275] --- /home/ubuntu/actions-runner/_work/tt-metal/tt-metal/.local/lib/python3.8/site-packages/ttnn/_ttnn.cpython-38-x86_64-linux-gnu.so(+0x12a6e9c) [0x7f57c68bae9c] --- void tt::tt_metal::operation::launch_op_func<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>>>(std::__1::function<std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>> (std::__1::vector<tt::tt_metal::Tensor, std::__1::allocator<tt::tt_metal::Tensor>> const&, std::__1::vector<std::__1::optional<tt::tt_metal::Tensor const>, std::__1::allocator<std::__1::optional<tt::tt_metal::Tensor const>>> const&, std::__1::vector<

[DEPRECATION] This action is deprecated. Please migrate to reading the Docker image from the pipeline.

Disk usage is 67 %

Touching and printing out SMI info

tt-smi reset was successful

Hugepages service found. Command returned with exit code 3. Restarting it so we can ensure hugepages are available