mpv/video/out/vulkan/malloc.c

472 lines
16 KiB
C

#include "malloc.h"
#include "utils.h"
#include "osdep/timer.h"
#if HAVE_WIN32_DESKTOP
#include <versionhelpers.h>
#endif
// Controls the multiplication factor for new slab allocations. The new slab
// will always be allocated such that the size of the slab is this factor times
// the previous slab. Higher values make it grow faster.
#define MPVK_HEAP_SLAB_GROWTH_RATE 4
// Controls the minimum slab size, to reduce the frequency at which very small
// slabs would need to get allocated when allocating the first few buffers.
// (Default: 1 MB)
#define MPVK_HEAP_MINIMUM_SLAB_SIZE (1 << 20)
// Controls the maximum slab size, to reduce the effect of unbounded slab
// growth exhausting memory. If the application needs a single allocation
// that's bigger than this value, it will be allocated directly from the
// device. (Default: 512 MB)
#define MPVK_HEAP_MAXIMUM_SLAB_SIZE (1 << 29)
// Controls the minimum free region size, to reduce thrashing the free space
// map with lots of small buffers during uninit. (Default: 1 KB)
#define MPVK_HEAP_MINIMUM_REGION_SIZE (1 << 10)
// Represents a region of available memory
struct vk_region {
size_t start; // first offset in region
size_t end; // first offset *not* in region
};
static inline size_t region_len(struct vk_region r)
{
return r.end - r.start;
}
// A single slab represents a contiguous region of allocated memory. Actual
// allocations are served as slices of this. Slabs are organized into linked
// lists, which represent individual heaps.
struct vk_slab {
VkDeviceMemory mem; // underlying device allocation
size_t size; // total size of `slab`
size_t used; // number of bytes actually in use (for GC accounting)
bool dedicated; // slab is allocated specifically for one object
// free space map: a sorted list of memory regions that are available
struct vk_region *regions;
int num_regions;
// optional, depends on the memory type:
VkBuffer buffer; // buffer spanning the entire slab
void *data; // mapped memory corresponding to `mem`
};
// Represents a single memory heap. We keep track of a vk_heap for each
// combination of buffer type and memory selection parameters. This shouldn't
// actually be that many in practice, because some combinations simply never
// occur, and others will generally be the same for the same objects.
struct vk_heap {
VkBufferUsageFlags usage; // the buffer usage type (or 0)
VkMemoryPropertyFlags flags; // the memory type flags (or 0)
uint32_t typeBits; // the memory type index requirements (or 0)
bool exportable; // whether memory is exportable to other APIs
struct vk_slab **slabs; // array of slabs sorted by size
int num_slabs;
};
// The overall state of the allocator, which keeps track of a vk_heap for each
// memory type.
struct vk_malloc {
VkPhysicalDeviceMemoryProperties props;
struct vk_heap *heaps;
int num_heaps;
};
static void slab_free(struct mpvk_ctx *vk, struct vk_slab *slab)
{
if (!slab)
return;
assert(slab->used == 0);
int64_t start = mp_time_us();
vkDestroyBuffer(vk->dev, slab->buffer, MPVK_ALLOCATOR);
// also implicitly unmaps the memory if needed
vkFreeMemory(vk->dev, slab->mem, MPVK_ALLOCATOR);
int64_t stop = mp_time_us();
MP_VERBOSE(vk, "Freeing slab of size %zu took %lld μs.\n",
slab->size, (long long)(stop - start));
talloc_free(slab);
}
static bool find_best_memtype(struct mpvk_ctx *vk, uint32_t typeBits,
VkMemoryPropertyFlags flags,
VkMemoryType *out_type, int *out_index)
{
struct vk_malloc *ma = vk->alloc;
// The vulkan spec requires memory types to be sorted in the "optimal"
// order, so the first matching type we find will be the best/fastest one.
for (int i = 0; i < ma->props.memoryTypeCount; i++) {
// The memory type flags must include our properties
if ((ma->props.memoryTypes[i].propertyFlags & flags) != flags)
continue;
// The memory type must be supported by the requirements (bitfield)
if (typeBits && !(typeBits & (1 << i)))
continue;
*out_type = ma->props.memoryTypes[i];
*out_index = i;
return true;
}
MP_ERR(vk, "Found no memory type matching property flags 0x%x and type "
"bits 0x%x!\n", (unsigned)flags, (unsigned)typeBits);
return false;
}
static struct vk_slab *slab_alloc(struct mpvk_ctx *vk, struct vk_heap *heap,
size_t size)
{
struct vk_slab *slab = talloc_ptrtype(NULL, slab);
*slab = (struct vk_slab) {
.size = size,
};
MP_TARRAY_APPEND(slab, slab->regions, slab->num_regions, (struct vk_region) {
.start = 0,
.end = slab->size,
});
VkExportMemoryAllocateInfoKHR eminfo = {
.sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO_KHR,
#if HAVE_WIN32_DESKTOP
.handleTypes = IsWindows8OrGreater()
? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT_KHR
: VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT_KHR,
#else
.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT_KHR,
#endif
};
VkMemoryAllocateInfo minfo = {
.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
.pNext = heap->exportable ? &eminfo : NULL,
.allocationSize = slab->size,
};
uint32_t typeBits = heap->typeBits ? heap->typeBits : UINT32_MAX;
if (heap->usage) {
// FIXME: Since we can't keep track of queue family ownership properly,
// and we don't know in advance what types of queue families this buffer
// will belong to, we're forced to share all of our buffers between all
// command pools.
uint32_t qfs[3] = {0};
for (int i = 0; i < vk->num_pools; i++)
qfs[i] = vk->pools[i]->qf;
VkExternalMemoryBufferCreateInfoKHR ebinfo = {
.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO_KHR,
.handleTypes = eminfo.handleTypes,
};
VkBufferCreateInfo binfo = {
.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
.pNext = heap->exportable ? &ebinfo : NULL,
.size = slab->size,
.usage = heap->usage,
.sharingMode = vk->num_pools > 1 ? VK_SHARING_MODE_CONCURRENT
: VK_SHARING_MODE_EXCLUSIVE,
.queueFamilyIndexCount = vk->num_pools,
.pQueueFamilyIndices = qfs,
};
VK(vkCreateBuffer(vk->dev, &binfo, MPVK_ALLOCATOR, &slab->buffer));
VkMemoryRequirements reqs;
vkGetBufferMemoryRequirements(vk->dev, slab->buffer, &reqs);
minfo.allocationSize = reqs.size; // this can be larger than slab->size
typeBits &= reqs.memoryTypeBits; // this can restrict the types
}
VkMemoryType type;
int index;
if (!find_best_memtype(vk, typeBits, heap->flags, &type, &index))
goto error;
MP_VERBOSE(vk, "Allocating %zu memory of type 0x%x (id %d) in heap %d.\n",
slab->size, (unsigned)type.propertyFlags, index, (int)type.heapIndex);
minfo.memoryTypeIndex = index;
VK(vkAllocateMemory(vk->dev, &minfo, MPVK_ALLOCATOR, &slab->mem));
if (heap->flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)
VK(vkMapMemory(vk->dev, slab->mem, 0, VK_WHOLE_SIZE, 0, &slab->data));
if (slab->buffer)
VK(vkBindBufferMemory(vk->dev, slab->buffer, slab->mem, 0));
return slab;
error:
slab_free(vk, slab);
return NULL;
}
static void insert_region(struct vk_slab *slab, struct vk_region region)
{
if (region.start == region.end)
return;
bool big_enough = region_len(region) >= MPVK_HEAP_MINIMUM_REGION_SIZE;
// Find the index of the first region that comes after this
for (int i = 0; i < slab->num_regions; i++) {
struct vk_region *r = &slab->regions[i];
// Check for a few special cases which can be coalesced
if (r->end == region.start) {
// The new region is at the tail of this region. In addition to
// modifying this region, we also need to coalesce all the following
// regions for as long as possible
r->end = region.end;
struct vk_region *next = &slab->regions[i+1];
while (i+1 < slab->num_regions && r->end == next->start) {
r->end = next->end;
MP_TARRAY_REMOVE_AT(slab->regions, slab->num_regions, i+1);
}
return;
}
if (r->start == region.end) {
// The new region is at the head of this region. We don't need to
// do anything special here - because if this could be further
// coalesced backwards, the previous loop iteration would already
// have caught it.
r->start = region.start;
return;
}
if (r->start > region.start) {
// The new region comes somewhere before this region, so insert
// it into this index in the array.
if (big_enough) {
MP_TARRAY_INSERT_AT(slab, slab->regions, slab->num_regions,
i, region);
}
return;
}
}
// If we've reached the end of this loop, then all of the regions
// come before the new region, and are disconnected - so append it
if (big_enough)
MP_TARRAY_APPEND(slab, slab->regions, slab->num_regions, region);
}
static void heap_uninit(struct mpvk_ctx *vk, struct vk_heap *heap)
{
for (int i = 0; i < heap->num_slabs; i++)
slab_free(vk, heap->slabs[i]);
talloc_free(heap->slabs);
*heap = (struct vk_heap){0};
}
void vk_malloc_init(struct mpvk_ctx *vk)
{
assert(vk->physd);
vk->alloc = talloc_zero(NULL, struct vk_malloc);
vkGetPhysicalDeviceMemoryProperties(vk->physd, &vk->alloc->props);
}
void vk_malloc_uninit(struct mpvk_ctx *vk)
{
struct vk_malloc *ma = vk->alloc;
if (!ma)
return;
for (int i = 0; i < ma->num_heaps; i++)
heap_uninit(vk, &ma->heaps[i]);
talloc_free(ma);
vk->alloc = NULL;
}
void vk_free_memslice(struct mpvk_ctx *vk, struct vk_memslice slice)
{
struct vk_slab *slab = slice.priv;
if (!slab)
return;
assert(slab->used >= slice.size);
slab->used -= slice.size;
MP_DBG(vk, "Freeing slice %zu + %zu from slab with size %zu\n",
slice.offset, slice.size, slab->size);
if (slab->dedicated) {
// If the slab was purpose-allocated for this memslice, we can just
// free it here
slab_free(vk, slab);
} else {
// Return the allocation to the free space map
insert_region(slab, (struct vk_region) {
.start = slice.offset,
.end = slice.offset + slice.size,
});
}
}
// reqs: can be NULL
static struct vk_heap *find_heap(struct mpvk_ctx *vk, VkBufferUsageFlags usage,
VkMemoryPropertyFlags flags,
VkMemoryRequirements *reqs,
bool exportable)
{
struct vk_malloc *ma = vk->alloc;
int typeBits = reqs ? reqs->memoryTypeBits : 0;
for (int i = 0; i < ma->num_heaps; i++) {
if (ma->heaps[i].usage != usage)
continue;
if (ma->heaps[i].flags != flags)
continue;
if (ma->heaps[i].typeBits != typeBits)
continue;
if (ma->heaps[i].exportable != exportable)
continue;
return &ma->heaps[i];
}
// Not found => add it
MP_TARRAY_GROW(ma, ma->heaps, ma->num_heaps + 1);
struct vk_heap *heap = &ma->heaps[ma->num_heaps++];
*heap = (struct vk_heap) {
.usage = usage,
.flags = flags,
.typeBits = typeBits,
.exportable = exportable,
};
return heap;
}
static inline bool region_fits(struct vk_region r, size_t size, size_t align)
{
return MP_ALIGN_UP(r.start, align) + size <= r.end;
}
// Finds the best-fitting region in a heap. If the heap is too small or too
// fragmented, a new slab will be allocated under the hood.
static bool heap_get_region(struct mpvk_ctx *vk, struct vk_heap *heap,
size_t size, size_t align,
struct vk_slab **out_slab, int *out_index)
{
struct vk_slab *slab = NULL;
// If the allocation is very big, serve it directly instead of bothering
// with the heap
if (size > MPVK_HEAP_MAXIMUM_SLAB_SIZE) {
slab = slab_alloc(vk, heap, size);
*out_slab = slab;
*out_index = 0;
return !!slab;
}
for (int i = 0; i < heap->num_slabs; i++) {
slab = heap->slabs[i];
if (slab->size < size)
continue;
// Attempt a best fit search
int best = -1;
for (int n = 0; n < slab->num_regions; n++) {
struct vk_region r = slab->regions[n];
if (!region_fits(r, size, align))
continue;
if (best >= 0 && region_len(r) > region_len(slab->regions[best]))
continue;
best = n;
}
if (best >= 0) {
*out_slab = slab;
*out_index = best;
return true;
}
}
// Otherwise, allocate a new vk_slab and append it to the list.
size_t cur_size = MPMAX(size, slab ? slab->size : 0);
size_t slab_size = MPVK_HEAP_SLAB_GROWTH_RATE * cur_size;
slab_size = MPMAX(MPVK_HEAP_MINIMUM_SLAB_SIZE, slab_size);
slab_size = MPMIN(MPVK_HEAP_MAXIMUM_SLAB_SIZE, slab_size);
assert(slab_size >= size);
slab = slab_alloc(vk, heap, slab_size);
if (!slab)
return false;
MP_TARRAY_APPEND(NULL, heap->slabs, heap->num_slabs, slab);
// Return the only region there is in a newly allocated slab
assert(slab->num_regions == 1);
*out_slab = slab;
*out_index = 0;
return true;
}
static bool slice_heap(struct mpvk_ctx *vk, struct vk_heap *heap, size_t size,
size_t alignment, struct vk_memslice *out)
{
struct vk_slab *slab;
int index;
alignment = MP_ALIGN_UP(alignment, vk->limits.bufferImageGranularity);
if (!heap_get_region(vk, heap, size, alignment, &slab, &index))
return false;
struct vk_region reg = slab->regions[index];
MP_TARRAY_REMOVE_AT(slab->regions, slab->num_regions, index);
*out = (struct vk_memslice) {
.vkmem = slab->mem,
.offset = MP_ALIGN_UP(reg.start, alignment),
.size = size,
.slab_size = slab->size,
.priv = slab,
};
MP_DBG(vk, "Sub-allocating slice %zu + %zu from slab with size %zu\n",
out->offset, out->size, slab->size);
size_t out_end = out->offset + out->size;
insert_region(slab, (struct vk_region) { reg.start, out->offset });
insert_region(slab, (struct vk_region) { out_end, reg.end });
slab->used += size;
return true;
}
bool vk_malloc_generic(struct mpvk_ctx *vk, VkMemoryRequirements reqs,
VkMemoryPropertyFlags flags, struct vk_memslice *out)
{
struct vk_heap *heap = find_heap(vk, 0, flags, &reqs, false);
return slice_heap(vk, heap, reqs.size, reqs.alignment, out);
}
bool vk_malloc_buffer(struct mpvk_ctx *vk, VkBufferUsageFlags bufFlags,
VkMemoryPropertyFlags memFlags, VkDeviceSize size,
VkDeviceSize alignment, bool exportable,
struct vk_bufslice *out)
{
if (exportable) {
if (!vk->has_ext_external_memory_export) {
MP_ERR(vk, "Exportable memory requires the %s extension\n",
MP_VK_EXTERNAL_MEMORY_EXPORT_EXTENSION_NAME);
return false;
}
}
struct vk_heap *heap = find_heap(vk, bufFlags, memFlags, NULL, exportable);
if (!slice_heap(vk, heap, size, alignment, &out->mem))
return false;
struct vk_slab *slab = out->mem.priv;
out->buf = slab->buffer;
if (slab->data)
out->data = (void *)((uintptr_t)slab->data + (ptrdiff_t)out->mem.offset);
return true;
}