From 1f47e15f8fded390343596d522939a6f2e8d5fa7 Mon Sep 17 00:00:00 2001 From: Dzmitry Malyshau Date: Sun, 26 May 2024 23:08:37 -0700 Subject: [PATCH] vk: grow descriptor pools progressively instead of pre-allocating --- blade-graphics/src/vulkan/command.rs | 31 ++----- blade-graphics/src/vulkan/descriptor.rs | 111 ++++++++++++++++++++++++ blade-graphics/src/vulkan/mod.rs | 34 +++----- docs/CHANGELOG.md | 1 + 4 files changed, 131 insertions(+), 46 deletions(-) create mode 100644 blade-graphics/src/vulkan/descriptor.rs diff --git a/blade-graphics/src/vulkan/command.rs b/blade-graphics/src/vulkan/command.rs index 8833e627..b43f5203 100644 --- a/blade-graphics/src/vulkan/command.rs +++ b/blade-graphics/src/vulkan/command.rs @@ -219,19 +219,14 @@ impl super::CommandEncoder { pub fn start(&mut self) { self.buffers.rotate_left(1); + self.device + .reset_descriptor_pool(&mut self.buffers[0].descriptor_pool); let vk_info = vk::CommandBufferBeginInfo { flags: vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT, ..Default::default() }; unsafe { - self.device - .core - .reset_descriptor_pool( - self.buffers[0].descriptor_pool, - vk::DescriptorPoolResetFlags::empty(), - ) - .unwrap(); self.device .core .begin_command_buffer(self.buffers[0].raw, &vk_info) @@ -357,7 +352,7 @@ impl super::CommandEncoder { self.barrier(); self.mark("pass/compute"); super::ComputeCommandEncoder { - cmd_buf: self.buffers[0], + cmd_buf: self.buffers.first_mut().unwrap(), device: &self.device, update_data: &mut self.update_data, } @@ -407,7 +402,7 @@ impl super::CommandEncoder { }; rendering_info.render_area = render_area; - let cmd_buf = self.buffers[0]; + let cmd_buf = self.buffers.first_mut().unwrap(); unsafe { self.device .core @@ -708,20 +703,12 @@ impl crate::traits::PipelineEncoder for super::PipelineEncoder<'_, '_> { template_offsets: &dsl.template_offsets, }); - let descriptor_set_layouts = [dsl.raw]; - let descriptor_set_info = vk::DescriptorSetAllocateInfo { - descriptor_pool: self.cmd_buf.descriptor_pool, - ..Default::default() - } - .set_layouts(&descriptor_set_layouts); + let vk_set = self + .device + .allocate_descriptor_set(&mut self.cmd_buf.descriptor_pool, dsl); unsafe { - let sets = self - .device - .core - .allocate_descriptor_sets(&descriptor_set_info) - .unwrap(); self.device.core.update_descriptor_set_with_template( - sets[0], + vk_set, dsl.update_template, self.update_data.as_ptr() as *const _, ); @@ -730,7 +717,7 @@ impl crate::traits::PipelineEncoder for super::PipelineEncoder<'_, '_> { self.bind_point, self.layout.raw, group, - &sets, + &[vk_set], &[], ); } diff --git a/blade-graphics/src/vulkan/descriptor.rs b/blade-graphics/src/vulkan/descriptor.rs new file mode 100644 index 00000000..2a0dfa67 --- /dev/null +++ b/blade-graphics/src/vulkan/descriptor.rs @@ -0,0 +1,111 @@ +use ash::vk; + +//TODO: replace by an abstraction in `gpu-descriptor` +// https://github.com/zakarumych/gpu-descriptor/issues/42 +const COUNT_BASE: u32 = 16; + +#[derive(Debug)] +pub struct DescriptorPool { + sub_pools: Vec, +} + +impl super::Device { + fn create_descriptor_sub_pool(&self, max_sets: u32) -> vk::DescriptorPool { + log::info!("Creating a descriptor pool for at most {} sets", max_sets); + let mut descriptor_sizes = vec![ + vk::DescriptorPoolSize { + ty: vk::DescriptorType::INLINE_UNIFORM_BLOCK_EXT, + descriptor_count: max_sets * crate::limits::PLAIN_DATA_SIZE, + }, + vk::DescriptorPoolSize { + ty: vk::DescriptorType::STORAGE_BUFFER, + descriptor_count: max_sets, + }, + vk::DescriptorPoolSize { + ty: vk::DescriptorType::SAMPLED_IMAGE, + descriptor_count: 2 * max_sets, + }, + vk::DescriptorPoolSize { + ty: vk::DescriptorType::SAMPLER, + descriptor_count: max_sets, + }, + vk::DescriptorPoolSize { + ty: vk::DescriptorType::STORAGE_IMAGE, + descriptor_count: max_sets, + }, + ]; + if self.ray_tracing.is_some() { + descriptor_sizes.push(vk::DescriptorPoolSize { + ty: vk::DescriptorType::ACCELERATION_STRUCTURE_KHR, + descriptor_count: max_sets, + }); + } + + let mut inline_uniform_block_info = vk::DescriptorPoolInlineUniformBlockCreateInfoEXT { + max_inline_uniform_block_bindings: max_sets, + ..Default::default() + }; + let descriptor_pool_info = vk::DescriptorPoolCreateInfo::default() + .max_sets(max_sets) + .pool_sizes(&descriptor_sizes) + .push_next(&mut inline_uniform_block_info); + unsafe { + self.core + .create_descriptor_pool(&descriptor_pool_info, None) + .unwrap() + } + } + + pub(super) fn create_descriptor_pool(&self) -> DescriptorPool { + let vk_pool = self.create_descriptor_sub_pool(COUNT_BASE); + DescriptorPool { + sub_pools: vec![vk_pool], + } + } + + pub(super) fn destroy_descriptor_pool(&self, pool: &mut DescriptorPool) { + for sub_pool in pool.sub_pools.drain(..) { + unsafe { self.core.destroy_descriptor_pool(sub_pool, None) }; + } + } + + pub(super) fn allocate_descriptor_set( + &self, + pool: &mut DescriptorPool, + layout: &super::DescriptorSetLayout, + ) -> vk::DescriptorSet { + let descriptor_set_layouts = [layout.raw]; + let mut descriptor_set_info = vk::DescriptorSetAllocateInfo::default() + .descriptor_pool(pool.sub_pools[0]) + .set_layouts(&descriptor_set_layouts); + let result = unsafe { self.core.allocate_descriptor_sets(&descriptor_set_info) }; + match result { + Ok(vk_sets) => return vk_sets[0], + Err(vk::Result::ERROR_OUT_OF_POOL_MEMORY) => {} + Err(other) => panic!("Unexpected descriptor allocation error: {:?}", other), + }; + + let next_max_sets = COUNT_BASE.pow(pool.sub_pools.len() as u32 + 1); + let vk_pool = self.create_descriptor_sub_pool(next_max_sets); + // Always insert in front to avoid expecting any overflows down the road + pool.sub_pools.insert(0, vk_pool); + + descriptor_set_info.descriptor_pool = vk_pool; + let vk_sets = unsafe { + self.core + .allocate_descriptor_sets(&descriptor_set_info) + .unwrap() + }; + vk_sets[0] + } + + pub(super) fn reset_descriptor_pool(&self, pool: &mut DescriptorPool) { + for &vk_pool in pool.sub_pools.iter() { + unsafe { + self.core + .reset_descriptor_pool(vk_pool, vk::DescriptorPoolResetFlags::empty()) + .unwrap(); + } + } + } +} diff --git a/blade-graphics/src/vulkan/mod.rs b/blade-graphics/src/vulkan/mod.rs index bbbd3af4..1ebfcd8d 100644 --- a/blade-graphics/src/vulkan/mod.rs +++ b/blade-graphics/src/vulkan/mod.rs @@ -2,6 +2,7 @@ use ash::{khr, vk}; use std::{num::NonZeroU32, ptr, sync::Mutex}; mod command; +mod descriptor; mod init; mod pipeline; mod resource; @@ -208,10 +209,10 @@ pub struct RenderPipeline { layout: PipelineLayout, } -#[derive(Clone, Copy, Debug)] +#[derive(Debug)] struct CommandBuffer { raw: vk::CommandBuffer, - descriptor_pool: vk::DescriptorPool, + descriptor_pool: descriptor::DescriptorPool, } #[derive(Debug, PartialEq)] @@ -244,17 +245,17 @@ pub struct AccelerationStructureCommandEncoder<'a> { device: &'a Device, } pub struct ComputeCommandEncoder<'a> { - cmd_buf: CommandBuffer, + cmd_buf: &'a mut CommandBuffer, device: &'a Device, update_data: &'a mut Vec, } pub struct RenderCommandEncoder<'a> { - cmd_buf: CommandBuffer, + cmd_buf: &'a mut CommandBuffer, device: &'a Device, update_data: &'a mut Vec, } pub struct PipelineEncoder<'a, 'p> { - cmd_buf: CommandBuffer, + cmd_buf: &'a mut CommandBuffer, layout: &'p PipelineLayout, bind_point: vk::PipelineBindPoint, device: &'a Device, @@ -332,21 +333,7 @@ impl crate::traits::CommandDevice for Context { if !desc.name.is_empty() { self.set_object_name(raw, desc.name); }; - let mut inline_uniform_block_info = - vk::DescriptorPoolInlineUniformBlockCreateInfoEXT { - max_inline_uniform_block_bindings: ROUGH_SET_COUNT, - ..Default::default() - }; - let descriptor_pool_info = vk::DescriptorPoolCreateInfo::default() - .max_sets(ROUGH_SET_COUNT) - .pool_sizes(&descriptor_sizes) - .push_next(&mut inline_uniform_block_info); - let descriptor_pool = unsafe { - self.device - .core - .create_descriptor_pool(&descriptor_pool_info, None) - .unwrap() - }; + let descriptor_pool = self.device.create_descriptor_pool(); CommandBuffer { raw, descriptor_pool, @@ -380,16 +367,15 @@ impl crate::traits::CommandDevice for Context { } fn destroy_command_encoder(&self, command_encoder: &mut CommandEncoder) { - for cmd_buf in command_encoder.buffers.iter() { + for cmd_buf in command_encoder.buffers.iter_mut() { let raw_cmd_buffers = [cmd_buf.raw]; unsafe { self.device .core .free_command_buffers(command_encoder.pool, &raw_cmd_buffers); - self.device - .core - .destroy_descriptor_pool(cmd_buf.descriptor_pool, None); } + self.device + .destroy_descriptor_pool(&mut cmd_buf.descriptor_pool); } unsafe { self.device diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 39ca6cbf..5c93ebc3 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -10,6 +10,7 @@ Changelog for Blade - window API switched to raw-window-handle-0.6 - GLES: support for storage buffer and compute - GLES: scissor rects, able to run "particle" example +- fixed initial RAM consumption ## blade-graphics-0.4, blade-render-0.3, blade-0.2 (22 Mar 2024)