612 lines
24 KiB
Rust
612 lines
24 KiB
Rust
use {
|
|
crate::{
|
|
align_down,
|
|
block::{MemoryBlock, MemoryBlockFlavor},
|
|
buddy::{BuddyAllocator, BuddyBlock},
|
|
config::Config,
|
|
error::AllocationError,
|
|
freelist::{FreeListAllocator, FreeListBlock},
|
|
heap::Heap,
|
|
usage::{MemoryForUsage, UsageFlags},
|
|
MemoryBounds, Request,
|
|
},
|
|
alloc::boxed::Box,
|
|
core::convert::TryFrom as _,
|
|
gpu_alloc_types::{
|
|
AllocationFlags, DeviceProperties, MemoryDevice, MemoryPropertyFlags, MemoryType,
|
|
OutOfMemory,
|
|
},
|
|
};
|
|
|
|
/// Memory allocator for Vulkan-like APIs.
|
|
#[derive(Debug)]
|
|
pub struct GpuAllocator<M> {
|
|
dedicated_threshold: u64,
|
|
preferred_dedicated_threshold: u64,
|
|
transient_dedicated_threshold: u64,
|
|
max_memory_allocation_size: u64,
|
|
memory_for_usage: MemoryForUsage,
|
|
memory_types: Box<[MemoryType]>,
|
|
memory_heaps: Box<[Heap]>,
|
|
allocations_remains: u32,
|
|
non_coherent_atom_mask: u64,
|
|
starting_free_list_chunk: u64,
|
|
final_free_list_chunk: u64,
|
|
minimal_buddy_size: u64,
|
|
initial_buddy_dedicated_size: u64,
|
|
buffer_device_address: bool,
|
|
|
|
buddy_allocators: Box<[Option<BuddyAllocator<M>>]>,
|
|
freelist_allocators: Box<[Option<FreeListAllocator<M>>]>,
|
|
}
|
|
|
|
/// Hints for allocator to decide on allocation strategy.
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
|
#[non_exhaustive]
|
|
pub enum Dedicated {
|
|
/// Allocation directly from device.\
|
|
/// Very slow.
|
|
/// Count of allocations is limited.\
|
|
/// Use with caution.\
|
|
/// Must be used if resource has to be bound to dedicated memory object.
|
|
Required,
|
|
|
|
/// Hint for allocator that dedicated memory object is preferred.\
|
|
/// Should be used if it is known that resource placed in dedicated memory object
|
|
/// would allow for better performance.\
|
|
/// Implementation is allowed to return block to shared memory object.
|
|
Preferred,
|
|
}
|
|
|
|
impl<M> GpuAllocator<M>
|
|
where
|
|
M: MemoryBounds + 'static,
|
|
{
|
|
/// Creates new instance of `GpuAllocator`.
|
|
/// Provided `DeviceProperties` should match properties of `MemoryDevice` that will be used
|
|
/// with created `GpuAllocator` instance.
|
|
#[cfg_attr(feature = "tracing", tracing::instrument)]
|
|
pub fn new(config: Config, props: DeviceProperties<'_>) -> Self {
|
|
assert!(
|
|
props.non_coherent_atom_size.is_power_of_two(),
|
|
"`non_coherent_atom_size` must be power of two"
|
|
);
|
|
|
|
assert!(
|
|
isize::try_from(props.non_coherent_atom_size).is_ok(),
|
|
"`non_coherent_atom_size` must fit host address space"
|
|
);
|
|
|
|
GpuAllocator {
|
|
dedicated_threshold: config.dedicated_threshold,
|
|
preferred_dedicated_threshold: config
|
|
.preferred_dedicated_threshold
|
|
.min(config.dedicated_threshold),
|
|
|
|
transient_dedicated_threshold: config
|
|
.transient_dedicated_threshold
|
|
.max(config.dedicated_threshold),
|
|
|
|
max_memory_allocation_size: props.max_memory_allocation_size,
|
|
|
|
memory_for_usage: MemoryForUsage::new(props.memory_types.as_ref()),
|
|
|
|
memory_types: props.memory_types.as_ref().iter().copied().collect(),
|
|
memory_heaps: props
|
|
.memory_heaps
|
|
.as_ref()
|
|
.iter()
|
|
.map(|heap| Heap::new(heap.size))
|
|
.collect(),
|
|
|
|
buffer_device_address: props.buffer_device_address,
|
|
|
|
allocations_remains: props.max_memory_allocation_count,
|
|
non_coherent_atom_mask: props.non_coherent_atom_size - 1,
|
|
|
|
starting_free_list_chunk: config.starting_free_list_chunk,
|
|
final_free_list_chunk: config.final_free_list_chunk,
|
|
minimal_buddy_size: config.minimal_buddy_size,
|
|
initial_buddy_dedicated_size: config.initial_buddy_dedicated_size,
|
|
|
|
buddy_allocators: props.memory_types.as_ref().iter().map(|_| None).collect(),
|
|
freelist_allocators: props.memory_types.as_ref().iter().map(|_| None).collect(),
|
|
}
|
|
}
|
|
|
|
/// Allocates memory block from specified `device` according to the `request`.
|
|
///
|
|
/// # Safety
|
|
///
|
|
/// * `device` must be one with `DeviceProperties` that were provided to create this `GpuAllocator` instance.
|
|
/// * Same `device` instance must be used for all interactions with one `GpuAllocator` instance
|
|
/// and memory blocks allocated from it.
|
|
#[cfg_attr(feature = "tracing", tracing::instrument(skip(self, device)))]
|
|
pub unsafe fn alloc(
|
|
&mut self,
|
|
device: &impl MemoryDevice<M>,
|
|
request: Request,
|
|
) -> Result<MemoryBlock<M>, AllocationError> {
|
|
self.alloc_internal(device, request, None)
|
|
}
|
|
|
|
/// Allocates memory block from specified `device` according to the `request`.
|
|
/// This function allows user to force specific allocation strategy.
|
|
/// Improper use can lead to suboptimal performance or too large overhead.
|
|
/// Prefer `GpuAllocator::alloc` if doubt.
|
|
///
|
|
/// # Safety
|
|
///
|
|
/// * `device` must be one with `DeviceProperties` that were provided to create this `GpuAllocator` instance.
|
|
/// * Same `device` instance must be used for all interactions with one `GpuAllocator` instance
|
|
/// and memory blocks allocated from it.
|
|
#[cfg_attr(feature = "tracing", tracing::instrument(skip(self, device)))]
|
|
pub unsafe fn alloc_with_dedicated(
|
|
&mut self,
|
|
device: &impl MemoryDevice<M>,
|
|
request: Request,
|
|
dedicated: Dedicated,
|
|
) -> Result<MemoryBlock<M>, AllocationError> {
|
|
self.alloc_internal(device, request, Some(dedicated))
|
|
}
|
|
|
|
unsafe fn alloc_internal(
|
|
&mut self,
|
|
device: &impl MemoryDevice<M>,
|
|
mut request: Request,
|
|
dedicated: Option<Dedicated>,
|
|
) -> Result<MemoryBlock<M>, AllocationError> {
|
|
enum Strategy {
|
|
Buddy,
|
|
Dedicated,
|
|
FreeList,
|
|
}
|
|
|
|
request.usage = with_implicit_usage_flags(request.usage);
|
|
|
|
if request.usage.contains(UsageFlags::DEVICE_ADDRESS) {
|
|
assert!(self.buffer_device_address, "`DEVICE_ADDRESS` cannot be requested when `DeviceProperties::buffer_device_address` is false");
|
|
}
|
|
|
|
if request.size > self.max_memory_allocation_size {
|
|
return Err(AllocationError::OutOfDeviceMemory);
|
|
}
|
|
|
|
if let Some(Dedicated::Required) = dedicated {
|
|
if self.allocations_remains == 0 {
|
|
return Err(AllocationError::TooManyObjects);
|
|
}
|
|
}
|
|
|
|
if 0 == self.memory_for_usage.mask(request.usage) & request.memory_types {
|
|
#[cfg(feature = "tracing")]
|
|
tracing::error!(
|
|
"Cannot serve request {:?}, no memory among bitset `{}` support usage {:?}",
|
|
request,
|
|
request.memory_types,
|
|
request.usage
|
|
);
|
|
|
|
return Err(AllocationError::NoCompatibleMemoryTypes);
|
|
}
|
|
|
|
let transient = request.usage.contains(UsageFlags::TRANSIENT);
|
|
|
|
for &index in self.memory_for_usage.types(request.usage) {
|
|
if 0 == request.memory_types & (1 << index) {
|
|
// Skip memory type incompatible with the request.
|
|
continue;
|
|
}
|
|
|
|
let memory_type = &self.memory_types[index as usize];
|
|
let heap = memory_type.heap;
|
|
let heap = &mut self.memory_heaps[heap as usize];
|
|
|
|
if request.size > heap.size() {
|
|
// Impossible to use memory type from this heap.
|
|
continue;
|
|
}
|
|
|
|
let atom_mask = if host_visible_non_coherent(memory_type.props) {
|
|
self.non_coherent_atom_mask
|
|
} else {
|
|
0
|
|
};
|
|
|
|
let flags = if self.buffer_device_address {
|
|
AllocationFlags::DEVICE_ADDRESS
|
|
} else {
|
|
AllocationFlags::empty()
|
|
};
|
|
|
|
let strategy = match (dedicated, transient) {
|
|
(Some(Dedicated::Required), _) => Strategy::Dedicated,
|
|
(Some(Dedicated::Preferred), _)
|
|
if request.size >= self.preferred_dedicated_threshold =>
|
|
{
|
|
Strategy::Dedicated
|
|
}
|
|
(_, true) => {
|
|
let threshold = self.transient_dedicated_threshold.min(heap.size() / 32);
|
|
|
|
if request.size < threshold {
|
|
Strategy::FreeList
|
|
} else {
|
|
Strategy::Dedicated
|
|
}
|
|
}
|
|
(_, false) => {
|
|
let threshold = self.dedicated_threshold.min(heap.size() / 32);
|
|
|
|
if request.size < threshold {
|
|
Strategy::Buddy
|
|
} else {
|
|
Strategy::Dedicated
|
|
}
|
|
}
|
|
};
|
|
|
|
match strategy {
|
|
Strategy::Dedicated => {
|
|
#[cfg(feature = "tracing")]
|
|
tracing::debug!(
|
|
"Allocating memory object `{}@{:?}`",
|
|
request.size,
|
|
memory_type
|
|
);
|
|
|
|
match device.allocate_memory(request.size, index, flags) {
|
|
Ok(memory) => {
|
|
self.allocations_remains -= 1;
|
|
heap.alloc(request.size);
|
|
|
|
return Ok(MemoryBlock::new(
|
|
index,
|
|
memory_type.props,
|
|
0,
|
|
request.size,
|
|
atom_mask,
|
|
MemoryBlockFlavor::Dedicated { memory },
|
|
));
|
|
}
|
|
Err(OutOfMemory::OutOfDeviceMemory) => continue,
|
|
Err(OutOfMemory::OutOfHostMemory) => {
|
|
return Err(AllocationError::OutOfHostMemory)
|
|
}
|
|
}
|
|
}
|
|
Strategy::FreeList => {
|
|
let allocator = match &mut self.freelist_allocators[index as usize] {
|
|
Some(allocator) => allocator,
|
|
slot => {
|
|
let starting_free_list_chunk = match align_down(
|
|
self.starting_free_list_chunk.min(heap.size() / 32),
|
|
atom_mask,
|
|
) {
|
|
0 => atom_mask,
|
|
other => other,
|
|
};
|
|
|
|
let final_free_list_chunk = match align_down(
|
|
self.final_free_list_chunk
|
|
.max(self.starting_free_list_chunk)
|
|
.max(self.transient_dedicated_threshold)
|
|
.min(heap.size() / 32),
|
|
atom_mask,
|
|
) {
|
|
0 => atom_mask,
|
|
other => other,
|
|
};
|
|
|
|
slot.get_or_insert(FreeListAllocator::new(
|
|
starting_free_list_chunk,
|
|
final_free_list_chunk,
|
|
index,
|
|
memory_type.props,
|
|
if host_visible_non_coherent(memory_type.props) {
|
|
self.non_coherent_atom_mask
|
|
} else {
|
|
0
|
|
},
|
|
))
|
|
}
|
|
};
|
|
let result = allocator.alloc(
|
|
device,
|
|
request.size,
|
|
request.align_mask,
|
|
flags,
|
|
heap,
|
|
&mut self.allocations_remains,
|
|
);
|
|
|
|
match result {
|
|
Ok(block) => {
|
|
return Ok(MemoryBlock::new(
|
|
index,
|
|
memory_type.props,
|
|
block.offset,
|
|
block.size,
|
|
atom_mask,
|
|
MemoryBlockFlavor::FreeList {
|
|
chunk: block.chunk,
|
|
ptr: block.ptr,
|
|
memory: block.memory,
|
|
},
|
|
))
|
|
}
|
|
Err(AllocationError::OutOfDeviceMemory) => continue,
|
|
Err(err) => return Err(err),
|
|
}
|
|
}
|
|
|
|
Strategy::Buddy => {
|
|
let allocator = match &mut self.buddy_allocators[index as usize] {
|
|
Some(allocator) => allocator,
|
|
slot => {
|
|
let minimal_buddy_size = self
|
|
.minimal_buddy_size
|
|
.min(heap.size() / 1024)
|
|
.next_power_of_two();
|
|
|
|
let initial_buddy_dedicated_size = self
|
|
.initial_buddy_dedicated_size
|
|
.min(heap.size() / 32)
|
|
.next_power_of_two();
|
|
|
|
slot.get_or_insert(BuddyAllocator::new(
|
|
minimal_buddy_size,
|
|
initial_buddy_dedicated_size,
|
|
index,
|
|
memory_type.props,
|
|
if host_visible_non_coherent(memory_type.props) {
|
|
self.non_coherent_atom_mask
|
|
} else {
|
|
0
|
|
},
|
|
))
|
|
}
|
|
};
|
|
let result = allocator.alloc(
|
|
device,
|
|
request.size,
|
|
request.align_mask,
|
|
flags,
|
|
heap,
|
|
&mut self.allocations_remains,
|
|
);
|
|
|
|
match result {
|
|
Ok(block) => {
|
|
return Ok(MemoryBlock::new(
|
|
index,
|
|
memory_type.props,
|
|
block.offset,
|
|
block.size,
|
|
atom_mask,
|
|
MemoryBlockFlavor::Buddy {
|
|
chunk: block.chunk,
|
|
ptr: block.ptr,
|
|
index: block.index,
|
|
memory: block.memory,
|
|
},
|
|
))
|
|
}
|
|
Err(AllocationError::OutOfDeviceMemory) => continue,
|
|
Err(err) => return Err(err),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Err(AllocationError::OutOfDeviceMemory)
|
|
}
|
|
|
|
/// Creates a memory block from an existing memory allocation, transferring ownership to the allocator.
|
|
///
|
|
/// This function allows the [`GpuAllocator`] to manage memory allocated outside of the typical
|
|
/// [`GpuAllocator::alloc`] family of functions.
|
|
///
|
|
/// # Usage
|
|
///
|
|
/// If you need to import external memory, such as a Win32 `HANDLE` or a Linux `dmabuf`, import the device
|
|
/// memory using the graphics api and platform dependent functions. Once that is done, call this function
|
|
/// to make the [`GpuAllocator`] take ownership of the imported memory.
|
|
///
|
|
/// When calling this function, you **must** ensure there are [enough remaining allocations](GpuAllocator::remaining_allocations).
|
|
///
|
|
/// # Safety
|
|
///
|
|
/// - The `memory` must be allocated with the same device that was provided to create this [`GpuAllocator`]
|
|
/// instance.
|
|
/// - The `memory` must be valid.
|
|
/// - The `props`, `offset` and `size` must match the properties, offset and size of the memory allocation.
|
|
/// - The memory must have been allocated with the specified `memory_type`.
|
|
/// - There must be enough remaining allocations.
|
|
/// - The memory allocation must not come from an existing memory block created by this allocator.
|
|
/// - The underlying memory object must be deallocated using the returned [`MemoryBlock`] with
|
|
/// [`GpuAllocator::dealloc`].
|
|
pub unsafe fn import_memory(
|
|
&mut self,
|
|
memory: M,
|
|
memory_type: u32,
|
|
props: MemoryPropertyFlags,
|
|
offset: u64,
|
|
size: u64,
|
|
) -> MemoryBlock<M> {
|
|
// Get the heap which the imported memory is from.
|
|
let heap = self
|
|
.memory_types
|
|
.get(memory_type as usize)
|
|
.expect("Invalid memory type specified when importing memory")
|
|
.heap;
|
|
let heap = &mut self.memory_heaps[heap as usize];
|
|
|
|
#[cfg(feature = "tracing")]
|
|
tracing::debug!(
|
|
"Importing memory object {:?} `{}@{:?}`",
|
|
memory,
|
|
size,
|
|
memory_type
|
|
);
|
|
|
|
assert_ne!(
|
|
self.allocations_remains, 0,
|
|
"Out of allocations when importing a memory block. Ensure you check GpuAllocator::remaining_allocations before import."
|
|
);
|
|
self.allocations_remains -= 1;
|
|
|
|
let atom_mask = if host_visible_non_coherent(props) {
|
|
self.non_coherent_atom_mask
|
|
} else {
|
|
0
|
|
};
|
|
|
|
heap.alloc(size);
|
|
|
|
MemoryBlock::new(
|
|
memory_type,
|
|
props,
|
|
offset,
|
|
size,
|
|
atom_mask,
|
|
MemoryBlockFlavor::Dedicated { memory },
|
|
)
|
|
}
|
|
|
|
/// Deallocates memory block previously allocated from this `GpuAllocator` instance.
|
|
///
|
|
/// # Safety
|
|
///
|
|
/// * Memory block must have been allocated by this `GpuAllocator` instance
|
|
/// * `device` must be one with `DeviceProperties` that were provided to create this `GpuAllocator` instance
|
|
/// * Same `device` instance must be used for all interactions with one `GpuAllocator` instance
|
|
/// and memory blocks allocated from it
|
|
#[cfg_attr(feature = "tracing", tracing::instrument(skip(self, device)))]
|
|
pub unsafe fn dealloc(&mut self, device: &impl MemoryDevice<M>, block: MemoryBlock<M>) {
|
|
let memory_type = block.memory_type();
|
|
let offset = block.offset();
|
|
let size = block.size();
|
|
let flavor = block.deallocate();
|
|
match flavor {
|
|
MemoryBlockFlavor::Dedicated { memory } => {
|
|
let heap = self.memory_types[memory_type as usize].heap;
|
|
device.deallocate_memory(memory);
|
|
self.allocations_remains += 1;
|
|
self.memory_heaps[heap as usize].dealloc(size);
|
|
}
|
|
MemoryBlockFlavor::Buddy {
|
|
chunk,
|
|
ptr,
|
|
index,
|
|
memory,
|
|
} => {
|
|
let heap = self.memory_types[memory_type as usize].heap;
|
|
let heap = &mut self.memory_heaps[heap as usize];
|
|
|
|
let allocator = self.buddy_allocators[memory_type as usize]
|
|
.as_mut()
|
|
.expect("Allocator should exist");
|
|
|
|
allocator.dealloc(
|
|
device,
|
|
BuddyBlock {
|
|
memory,
|
|
ptr,
|
|
offset,
|
|
size,
|
|
chunk,
|
|
index,
|
|
},
|
|
heap,
|
|
&mut self.allocations_remains,
|
|
);
|
|
}
|
|
MemoryBlockFlavor::FreeList { chunk, ptr, memory } => {
|
|
let heap = self.memory_types[memory_type as usize].heap;
|
|
let heap = &mut self.memory_heaps[heap as usize];
|
|
|
|
let allocator = self.freelist_allocators[memory_type as usize]
|
|
.as_mut()
|
|
.expect("Allocator should exist");
|
|
|
|
allocator.dealloc(
|
|
device,
|
|
FreeListBlock {
|
|
memory,
|
|
ptr,
|
|
chunk,
|
|
offset,
|
|
size,
|
|
},
|
|
heap,
|
|
&mut self.allocations_remains,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Returns the maximum allocation size supported.
|
|
pub fn max_allocation_size(&self) -> u64 {
|
|
self.max_memory_allocation_size
|
|
}
|
|
|
|
/// Returns the number of remaining available allocations.
|
|
///
|
|
/// This may be useful if you need know if the allocator can allocate a number of allocations ahead of
|
|
/// time. This function is also useful for ensuring you do not allocate too much memory outside allocator
|
|
/// (such as external memory).
|
|
pub fn remaining_allocations(&self) -> u32 {
|
|
self.allocations_remains
|
|
}
|
|
|
|
/// Sets the number of remaining available allocations.
|
|
///
|
|
/// # Safety
|
|
///
|
|
/// The caller is responsible for ensuring the number of remaining allocations does not exceed how many
|
|
/// remaining allocations there actually are on the memory device.
|
|
pub unsafe fn set_remaining_allocations(&mut self, remaining: u32) {
|
|
self.allocations_remains = remaining;
|
|
}
|
|
|
|
/// Deallocates leftover memory objects.
|
|
/// Should be used before dropping.
|
|
///
|
|
/// # Safety
|
|
///
|
|
/// * `device` must be one with `DeviceProperties` that were provided to create this `GpuAllocator` instance
|
|
/// * Same `device` instance must be used for all interactions with one `GpuAllocator` instance
|
|
/// and memory blocks allocated from it
|
|
#[cfg_attr(feature = "tracing", tracing::instrument(skip(self, device)))]
|
|
pub unsafe fn cleanup(&mut self, device: &impl MemoryDevice<M>) {
|
|
for (index, allocator) in self
|
|
.freelist_allocators
|
|
.iter_mut()
|
|
.enumerate()
|
|
.filter_map(|(index, allocator)| Some((index, allocator.as_mut()?)))
|
|
{
|
|
let memory_type = &self.memory_types[index];
|
|
let heap = memory_type.heap;
|
|
let heap = &mut self.memory_heaps[heap as usize];
|
|
|
|
allocator.cleanup(device, heap, &mut self.allocations_remains);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn host_visible_non_coherent(props: MemoryPropertyFlags) -> bool {
|
|
(props & (MemoryPropertyFlags::HOST_COHERENT | MemoryPropertyFlags::HOST_VISIBLE))
|
|
== MemoryPropertyFlags::HOST_VISIBLE
|
|
}
|
|
|
|
fn with_implicit_usage_flags(usage: UsageFlags) -> UsageFlags {
|
|
if usage.is_empty() {
|
|
UsageFlags::FAST_DEVICE_ACCESS
|
|
} else if usage.intersects(UsageFlags::DOWNLOAD | UsageFlags::UPLOAD) {
|
|
usage | UsageFlags::HOST_ACCESS
|
|
} else {
|
|
usage
|
|
}
|
|
}
|