Halide 22.0.0 Halide compiler and libraries

vulkan_memory.h
Go to the documentation of this file.
1#ifndef HALIDE_RUNTIME_VULKAN_MEMORY_H
2#define HALIDE_RUNTIME_VULKAN_MEMORY_H
3
4#include "internal/block_allocator.h"
5#include "internal/memory_resources.h"
6#include "vulkan_internal.h"
7
8// Uncomment to enable verbose memory allocation debugging
9// #define HL_VK_DEBUG_MEM 1
10
11 namespace Halide {
12 namespace Runtime {
13 namespace Internal {
14 namespace Vulkan {
15
16// --------------------------------------------------------------------------
17
18// Enable external client to override Vulkan allocation callbacks (if they so desire)
19 WEAK ScopedSpinLock::AtomicFlag custom_allocation_callbacks_lock = 0;
20 WEAK const VkAllocationCallbacks *custom_allocation_callbacks = nullptr; // nullptr => use Vulkan runtime implementation
21
22// --------------------------------------------------------------------------
23
24// Runtime configuration parameters to adjust the behaviour of the block allocator
25 struct VulkanMemoryConfig {
26 size_t maximum_pool_size = 0; //< Maximum number of bytes to allocate for the entire pool (including all blocks). Specified in bytes. Zero means no constraint
27 size_t minimum_block_size = 4 * 1024; //< Default block size is 4KB
28 size_t maximum_block_size = 0; //< Specified in bytes. Zero means no constraint
29 size_t maximum_block_count = 0; //< Maximum number of blocks to allocate. Zero means no constraint
30 size_t nearest_multiple = 32; //< Always round up the requested region sizes to the given integer value. Zero means no constraint
31};
32 WEAK VulkanMemoryConfig memory_allocator_config;
33
34// --------------------------------------------------------------------------
35
36/** Vulkan Memory Allocator class interface for managing large
37 * memory requests stored as contiguous blocks of memory, which
38 * are then sub-allocated into smaller regions of
39 * memory to avoid the excessive cost of vkAllocate and the limited
40 * number of available allocation calls through the API.
41 */
42 class VulkanMemoryAllocator {
43 public:
44 // disable copy constructors and assignment
45 VulkanMemoryAllocator(const VulkanMemoryAllocator &) = delete;
46 VulkanMemoryAllocator &operator=(const VulkanMemoryAllocator &) = delete;
47
48 // disable non-factory constrction
49 VulkanMemoryAllocator() = delete;
50 ~VulkanMemoryAllocator() = delete;
51
52 // Factory methods for creation / destruction
53 static VulkanMemoryAllocator *create(void *user_context, const VulkanMemoryConfig &config,
54 VkDevice dev, VkPhysicalDevice phys_dev,
55 const SystemMemoryAllocatorFns &system_allocator,
56 const VkAllocationCallbacks *alloc_callbacks = nullptr);
57
58 static int destroy(void *user_context, VulkanMemoryAllocator *allocator);
59
60 // Public interface methods
61 MemoryRegion *reserve(void *user_context, const MemoryRequest &request);
62 int conform(void *user_context, MemoryRequest *request); //< conforms the given memory request into one that can be allocated
63 int release(void *user_context, MemoryRegion *region); //< unmark and cache the region for reuse
64 int reclaim(void *user_context, MemoryRegion *region); //< free the region and consolidate
65 int retain(void *user_context, MemoryRegion *region); //< retain the region and increase its use count
66 bool collect(void *user_context); //< returns true if any blocks were removed
67 int release(void *user_context);
68 int destroy(void *user_context);
69
70 void *map(void *user_context, MemoryRegion *region);
71 int unmap(void *user_context, MemoryRegion *region);
72 MemoryRegion *create_crop(void *user_context, MemoryRegion *region, uint64_t offset);
73 int destroy_crop(void *user_context, MemoryRegion *region);
74 MemoryRegion *owner_of(void *user_context, MemoryRegion *region);
75
76 VkDevice current_device() const {
77 return this->device;
78 }
79 VkPhysicalDevice current_physical_device() const {
80 return this->physical_device;
81 }
82 VkPhysicalDeviceLimits current_physical_device_limits() const {
83 return this->physical_device_limits;
84 }
85 const VkAllocationCallbacks *callbacks() const {
86 return this->alloc_callbacks;
87 }
88
89 static const VulkanMemoryConfig &default_config();
90
91 static int allocate_block(void *instance_ptr, MemoryBlock *block);
92 static int deallocate_block(void *instance_ptr, MemoryBlock *block);
93 static int conform_block_request(void *instance_ptr, MemoryRequest *request);
94
95 static int allocate_region(void *instance_ptr, MemoryRegion *region);
96 static int deallocate_region(void *instance_ptr, MemoryRegion *region);
97 static int conform_region_request(void *instance_ptr, MemoryRequest *request);
98
99 size_t bytes_allocated_for_blocks() const;
100 size_t blocks_allocated() const;
101
102 size_t bytes_allocated_for_regions() const;
103 size_t regions_allocated() const;
104
105 private:
106 static constexpr uint32_t invalid_memory_heap = uint32_t(-1);
107 static constexpr uint32_t invalid_usage_flags = uint32_t(-1);
108 static constexpr uint32_t invalid_memory_type = uint32_t(VK_MAX_MEMORY_TYPES);
109
110 // Initializes a new instance
111 int initialize(void *user_context, const VulkanMemoryConfig &config,
112 VkDevice dev, VkPhysicalDevice phys_dev,
113 const SystemMemoryAllocatorFns &system_allocator,
114 const VkAllocationCallbacks *alloc_callbacks = nullptr);
115
116 uint32_t select_memory_usage(void *user_context, MemoryProperties properties) const;
117
118 uint32_t preferred_flags_for_memory_properties(void *user_context, MemoryProperties properties) const;
119 uint32_t required_flags_for_memory_properties(void *user_context, MemoryProperties properties) const;
120
121 bool is_valid_memory_type_for_heap(void *user_context,
122 VkPhysicalDeviceMemoryProperties *memory_properties,
123 uint32_t memory_type_index,
124 uint32_t heap_index) const;
125
126 bool is_preferred_memory_type_for_flags(void *user_context,
127 VkPhysicalDeviceMemoryProperties *memory_properties,
128 uint32_t memory_type_index,
129 uint32_t preferred_flags,
130 uint32_t required_flags) const;
131
132 bool is_valid_memory_type_for_flags(void *user_context,
133 VkPhysicalDeviceMemoryProperties *memory_properties,
134 uint32_t memory_type_index,
135 uint32_t required_flags) const;
136
137 int lookup_requirements(void *user_context, size_t size, uint32_t usage_flags, VkMemoryRequirements *memory_requirements);
138
139 size_t block_byte_count = 0;
140 size_t block_count = 0;
141 size_t region_byte_count = 0;
142 size_t region_count = 0;
143 void *owner_context = nullptr;
144 VulkanMemoryConfig config;
145 VkDevice device = nullptr;
146 VkPhysicalDevice physical_device = nullptr;
147 VkPhysicalDeviceLimits physical_device_limits = {};
148 const VkAllocationCallbacks *alloc_callbacks = nullptr;
149 BlockAllocator *block_allocator = nullptr;
150};
151
152 VulkanMemoryAllocator *VulkanMemoryAllocator::create(void *user_context,
153 const VulkanMemoryConfig &cfg, VkDevice dev, VkPhysicalDevice phys_dev,
154 const SystemMemoryAllocatorFns &system_allocator,
155 const VkAllocationCallbacks *alloc_callbacks) {
156
157 if (system_allocator.allocate == nullptr) {
158 error(user_context) << "VulkanBlockAllocator: Unable to create instance! Missing system allocator interface!\n";
159 return nullptr;
160 }
161
162 VulkanMemoryAllocator *result = reinterpret_cast<VulkanMemoryAllocator *>(
163 system_allocator.allocate(user_context, sizeof(VulkanMemoryAllocator)));
164
165 if (result == nullptr) {
166 error(user_context) << "VulkanMemoryAllocator: Failed to create instance! Out of memory!\n";
167 return nullptr; // caller must handle error case for out-of-memory
168 }
169
170 result->initialize(user_context, cfg, dev, phys_dev, system_allocator, alloc_callbacks);
171 return result;
172}
173
174 int VulkanMemoryAllocator::destroy(void *user_context, VulkanMemoryAllocator *instance) {
175 if (instance == nullptr) {
176 error(user_context) << "VulkanBlockAllocator: Unable to destroy instance! Invalide instance pointer!\n";
177 return halide_error_code_internal_error;
178 }
179 BlockAllocator::MemoryAllocators allocators = instance->block_allocator->current_allocators();
180 instance->destroy(user_context);
181 if (allocators.system.deallocate == nullptr) {
182 error(user_context) << "VulkanBlockAllocator: Unable to destroy instance! Missing system allocator interface!\n";
183 return halide_error_code_internal_error;
184 }
185 allocators.system.deallocate(user_context, instance);
186 return halide_error_code_success;
187}
188
189 int VulkanMemoryAllocator::initialize(void *user_context,
190 const VulkanMemoryConfig &cfg, VkDevice dev, VkPhysicalDevice phys_dev,
191 const SystemMemoryAllocatorFns &system_allocator,
192 const VkAllocationCallbacks *callbacks) {
193
194 owner_context = user_context;
195 config = cfg;
196 device = dev;
197 physical_device = phys_dev;
198 alloc_callbacks = callbacks;
199 region_count = 0;
200 region_byte_count = 0;
201 block_count = 0;
202 block_byte_count = 0;
203 BlockAllocator::MemoryAllocators allocators;
204 allocators.system = system_allocator;
205 allocators.block = {VulkanMemoryAllocator::allocate_block, VulkanMemoryAllocator::deallocate_block, VulkanMemoryAllocator::conform_block_request};
206 allocators.region = {VulkanMemoryAllocator::allocate_region, VulkanMemoryAllocator::deallocate_region, VulkanMemoryAllocator::conform_region_request};
207 BlockAllocator::Config block_allocator_config = {0};
208 block_allocator_config.maximum_pool_size = cfg.maximum_pool_size;
209 block_allocator_config.maximum_block_count = cfg.maximum_block_count;
210 block_allocator_config.maximum_block_size = cfg.maximum_block_size;
211 block_allocator_config.minimum_block_size = cfg.minimum_block_size;
212 block_allocator_config.nearest_multiple = cfg.nearest_multiple;
213 block_allocator = BlockAllocator::create(user_context, block_allocator_config, allocators);
214 if (block_allocator == nullptr) {
215 error(user_context) << "VulkanMemoryAllocator: Failed to create BlockAllocator! Out of memory?!\n";
216 return halide_error_code_out_of_memory;
217 }
218
219 // get the physical device properties to determine limits and allocation requirements
220 VkPhysicalDeviceProperties physical_device_properties = {0};
221 memset(&physical_device_limits, 0, sizeof(VkPhysicalDeviceLimits));
222 vkGetPhysicalDeviceProperties(physical_device, &physical_device_properties);
223 memcpy(&physical_device_limits, &(physical_device_properties.limits), sizeof(VkPhysicalDeviceLimits));
224 return halide_error_code_success;
225}
226
227 MemoryRegion *VulkanMemoryAllocator::reserve(void *user_context, const MemoryRequest &request) {
228#if defined(HL_VK_DEBUG_MEM)
229 debug(nullptr) << "VulkanMemoryAllocator: Reserving memory ("
230 << "user_context=" << user_context << " "
231 << "block_allocator=" << (void *)(block_allocator) << " "
232 << "request_size=" << (uint32_t)(request.size) << " "
233 << "device=" << (void *)(device) << " "
234 << "physical_device=" << (void *)(physical_device) << ") ...\n";
235#endif
236
237 if ((device == nullptr) || (physical_device == nullptr)) {
238 error(user_context) << "VulkanMemoryAllocator: Unable to reserve memory! Invalid device handle!\n";
239 return nullptr;
240 }
241
242 if (block_allocator == nullptr) {
243 error(user_context) << "VulkanMemoryAllocator: Unable to reserve memory! Invalid block allocator!\n";
244 return nullptr;
245 }
246
247 return block_allocator->reserve(this, request);
248}
249
250 void *VulkanMemoryAllocator::map(void *user_context, MemoryRegion *region) {
251#if defined(HL_VK_DEBUG_MEM)
252 debug(nullptr) << "VulkanMemoryAllocator: Mapping region ("
253 << "user_context=" << user_context << " "
254 << "device=" << (void *)(device) << " "
255 << "physical_device=" << (void *)(physical_device) << " "
256 << "region=" << (void *)(region) << " "
257 << "region_size=" << (uint32_t)region->size << " "
258 << "region_offset=" << (uint32_t)region->offset << " "
259 << "crop_offset=" << (uint32_t)region->range.head_offset << ") ...\n";
260#endif
261 if ((device == nullptr) || (physical_device == nullptr)) {
262 error(user_context) << "VulkanMemoryAllocator: Unable to map memory! Invalid device handle!\n";
263 return nullptr;
264 }
265
266 if (block_allocator == nullptr) {
267 error(user_context) << "VulkanMemoryAllocator: Unable to map memory! Invalid block allocator!\n";
268 return nullptr;
269 }
270
271 MemoryRegion *owner = owner_of(user_context, region);
272 RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
273 if (region_allocator == nullptr) {
274 error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid region allocator handle!\n";
275 return nullptr; // NOTE: caller must handle nullptr
276 }
277
278 BlockResource *block_resource = region_allocator->block_resource();
279 if (block_resource == nullptr) {
280 error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid block resource handle!\n";
281 return nullptr; // NOTE: caller must handle nullptr
282 }
283
284 VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
285 if (device_memory == nullptr) {
286 error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid device memory handle!\n";
287 return nullptr; // NOTE: caller must handle nullptr
288 }
289
290 void *mapped_ptr = nullptr;
291 VkDeviceSize memory_offset = region->offset + region->range.head_offset;
292 VkDeviceSize memory_size = region->size - region->range.tail_offset - region->range.head_offset;
293 if (((double)region->size - (double)region->range.tail_offset - (double)region->range.head_offset) <= 0.0) {
294 error(user_context) << "VulkanMemoryAllocator: Unable to map region! Invalid memory range !\n";
295 return nullptr;
296 }
297#if defined(HL_VK_DEBUG_MEM)
298 debug(nullptr) << "VulkanMemoryAllocator: MapMemory ("
299 << "user_context=" << user_context << "\n"
300 << " region_size=" << (uint32_t)region->size << "\n"
301 << " region_offset=" << (uint32_t)region->offset << "\n"
302 << " region_range.head_offset=" << (uint32_t)region->range.head_offset << "\n"
303 << " region_range.tail_offset=" << (uint32_t)region->range.tail_offset << "\n"
304 << " memory_offset=" << (uint32_t)memory_offset << "\n"
305 << " memory_size=" << (uint32_t)memory_size << "\n)\n";
306#endif
307 VkResult result = vkMapMemory(device, *device_memory, memory_offset, memory_size, 0, (void **)(&mapped_ptr));
308 if (result != VK_SUCCESS) {
309 error(user_context) << "VulkanMemoryAllocator: Mapping region failed! vkMapMemory returned error code: " << vk_get_error_name(result) << "\n";
310 return nullptr;
311 }
312
313 return mapped_ptr;
314}
315
316 int VulkanMemoryAllocator::unmap(void *user_context, MemoryRegion *region) {
317#if defined(HL_VK_DEBUG_MEM)
318 debug(nullptr) << "VulkanMemoryAllocator: Unmapping region ("
319 << "user_context=" << user_context << " "
320 << "device=" << (void *)(device) << " "
321 << "physical_device=" << (void *)(physical_device) << " "
322 << "region=" << (void *)(region) << " "
323 << "region_size=" << (uint32_t)region->size << " "
324 << "region_offset=" << (uint32_t)region->offset << " "
325 << "crop_offset=" << (uint32_t)region->range.head_offset << ") ...\n";
326#endif
327 if ((device == nullptr) || (physical_device == nullptr)) {
328 error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid device handle!\n";
329 return halide_error_code_generic_error;
330 }
331
332 MemoryRegion *owner = owner_of(user_context, region);
333 RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
334 if (region_allocator == nullptr) {
335 error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid region allocator handle!\n";
336 return halide_error_code_internal_error;
337 }
338
339 BlockResource *block_resource = region_allocator->block_resource();
340 if (block_resource == nullptr) {
341 error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid block resource handle!\n";
342 return halide_error_code_internal_error;
343 }
344
345 VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
346 if (device_memory == nullptr) {
347 error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid device memory handle!\n";
348 return halide_error_code_internal_error;
349 }
350
351 vkUnmapMemory(device, *device_memory);
352 return halide_error_code_success;
353}
354
355 MemoryRegion *VulkanMemoryAllocator::create_crop(void *user_context, MemoryRegion *region, uint64_t offset) {
356#if defined(HL_VK_DEBUG_MEM)
357 debug(nullptr) << "VulkanMemoryAllocator: Cropping region ("
358 << "user_context=" << user_context << " "
359 << "device=" << (void *)(device) << " "
360 << "physical_device=" << (void *)(physical_device) << " "
361 << "region=" << (void *)(region) << " "
362 << "region_size=" << (uint32_t)region->size << " "
363 << "region_offset=" << (uint32_t)region->offset << " "
364 << "crop_offset=" << (int64_t)offset << ") ...\n";
365#endif
366 if ((device == nullptr) || (physical_device == nullptr)) {
367 error(user_context) << "VulkanMemoryAllocator: Unable to crop region! Invalid device handle!\n";
368 return nullptr;
369 }
370
371 MemoryRegion *owner = owner_of(user_context, region);
372 RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
373 if (region_allocator == nullptr) {
374 error(user_context) << "VulkanMemoryAllocator: Unable to unmap region! Invalid region allocator handle!\n";
375 return nullptr; // NOTE: caller must handle nullptr
376 }
377
378 // increment usage count
379 int error_code = region_allocator->retain(this, owner);
380 if (error_code != halide_error_code_success) {
381 error(user_context) << "VulkanMemoryAllocator: Unable to crop region! Failed to retain memory region!\n";
382 return nullptr; // NOTE: caller must handle nullptr
383 }
384
385 // create a new region to return, and copy all the other region's properties
386 const BlockAllocator::MemoryAllocators &allocators = block_allocator->current_allocators();
387 if (allocators.system.allocate == nullptr) {
388 error(user_context) << "VulkanMemoryAllocator: Unable to create crop! Missing system allocator interface!\n";
389 return nullptr;
390 }
391
392 MemoryRegion *memory_region = reinterpret_cast<MemoryRegion *>(
393 allocators.system.allocate(user_context, sizeof(MemoryRegion)));
394
395 if (memory_region == nullptr) {
396 error(user_context) << "VulkanMemoryAllocator: Failed to allocate memory region! Out of memory!\n";
397 return nullptr;
398 }
399 memcpy(memory_region, owner, sizeof(MemoryRegion));
400
401 // point the handle to the owner of the allocated region, and update the head offset
402 memory_region->is_owner = false;
403 memory_region->handle = (void *)owner;
404 memory_region->range.head_offset = owner->range.head_offset + offset;
405 return memory_region;
406}
407
408 int VulkanMemoryAllocator::destroy_crop(void *user_context, MemoryRegion *region) {
409 if (region == nullptr) {
410 error(user_context) << "VulkanMemoryAllocator: Failed to destroy crop! Invalid memory region!\n";
411 return halide_error_code_generic_error;
412 }
413
414 MemoryRegion *owner = owner_of(user_context, region);
415 RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, owner);
416 if (region_allocator == nullptr) {
417 error(user_context) << "VulkanMemoryAllocator: Unable to destroy crop region! Invalid region allocator handle!\n";
418 return halide_error_code_internal_error;
419 }
420
421 // decrement usage count
422 int error_code = region_allocator->release(this, owner);
423 if (error_code != halide_error_code_success) {
424 error(user_context) << "VulkanBlockAllocator: Unable to destroy crop region! Region allocator failed to release memory region!\n";
425 return error_code;
426 }
427
428 // discard the copied region struct
429 const BlockAllocator::MemoryAllocators &allocators = block_allocator->current_allocators();
430 if (allocators.system.deallocate == nullptr) {
431 error(user_context) << "VulkanBlockAllocator: Unable to destroy crop region! Missing system allocator interface!\n";
432 return halide_error_code_internal_error;
433 }
434 allocators.system.deallocate(user_context, region);
435 return halide_error_code_success;
436}
437
438 MemoryRegion *VulkanMemoryAllocator::owner_of(void *user_context, MemoryRegion *region) {
439 if (region->is_owner) {
440 return region;
441 } else {
442 // If this is a cropped region, use the handle to retrieve the owner of the allocation
443 return reinterpret_cast<MemoryRegion *>(region->handle);
444 }
445}
446
447 int VulkanMemoryAllocator::release(void *user_context, MemoryRegion *region) {
448#if defined(HL_VK_DEBUG_MEM)
449 debug(nullptr) << "VulkanMemoryAllocator: Releasing region ("
450 << "user_context=" << user_context << " "
451 << "region=" << (void *)(region) << " "
452 << "size=" << (uint32_t)region->size << " "
453 << "offset=" << (uint32_t)region->offset << ") ...\n";
454#endif
455 if ((device == nullptr) || (physical_device == nullptr)) {
456 error(user_context) << "VulkanMemoryAllocator: Unable to release region! Invalid device handle!\n";
457 return halide_error_code_generic_error;
458 }
459 if (block_allocator == nullptr) {
460 error(user_context) << "VulkanMemoryAllocator: Unable to release region! Invalid block allocator!\n";
461 return halide_error_code_generic_error;
462 }
463 return block_allocator->release(this, region);
464}
465
466 int VulkanMemoryAllocator::reclaim(void *user_context, MemoryRegion *region) {
467#if defined(HL_VK_DEBUG_MEM)
468 debug(nullptr) << "VulkanMemoryAllocator: Reclaiming region ("
469 << "user_context=" << user_context << " "
470 << "region=" << (void *)(region) << " "
471 << "size=" << (uint32_t)region->size << " "
472 << "offset=" << (uint32_t)region->offset << ") ...\n";
473#endif
474 if ((device == nullptr) || (physical_device == nullptr)) {
475 error(user_context) << "VulkanMemoryAllocator: Unable to reclaim region! Invalid device handle!\n";
476 return halide_error_code_generic_error;
477 }
478 if (block_allocator == nullptr) {
479 error(user_context) << "VulkanMemoryAllocator: Unable to reclaim region! Invalid block allocator!\n";
480 return halide_error_code_generic_error;
481 }
482 return block_allocator->reclaim(this, region);
483}
484
485 int VulkanMemoryAllocator::retain(void *user_context, MemoryRegion *region) {
486#if defined(HL_VK_DEBUG_MEM)
487 debug(nullptr) << "VulkanMemoryAllocator: Retaining region ("
488 << "user_context=" << user_context << " "
489 << "region=" << (void *)(region) << " "
490 << "size=" << (uint32_t)region->size << " "
491 << "offset=" << (uint32_t)region->offset << ") ...\n";
492#endif
493 if ((device == nullptr) || (physical_device == nullptr)) {
494 error(user_context) << "VulkanMemoryAllocator: Unable to retain region! Invalid device handle!\n";
495 return halide_error_code_generic_error;
496 }
497 if (block_allocator == nullptr) {
498 error(user_context) << "VulkanMemoryAllocator: Unable to retain region! Invalid block allocator!\n";
499 return halide_error_code_generic_error;
500 }
501 return block_allocator->retain(this, region);
502}
503
504 bool VulkanMemoryAllocator::collect(void *user_context) {
505#if defined(HL_VK_DEBUG_MEM)
506 debug(nullptr) << "VulkanMemoryAllocator: Collecting unused memory ("
507 << "user_context=" << user_context << ") ... \n";
508#endif
509 if ((device == nullptr) || (physical_device == nullptr) || (block_allocator == nullptr)) {
510 return false;
511 }
512 return block_allocator->collect(this);
513}
514
515 int VulkanMemoryAllocator::release(void *user_context) {
516#if defined(HL_VK_DEBUG_MEM)
517 debug(nullptr) << "VulkanMemoryAllocator: Releasing block allocator ("
518 << "user_context=" << user_context << ") ... \n";
519#endif
520 if ((device == nullptr) || (physical_device == nullptr)) {
521 error(user_context) << "VulkanMemoryAllocator: Unable to release allocator! Invalid device handle!\n";
522 return halide_error_code_generic_error;
523 }
524 if (block_allocator == nullptr) {
525 error(user_context) << "VulkanMemoryAllocator: Unable to release allocator! Invalid block allocator!\n";
526 return halide_error_code_generic_error;
527 }
528
529 return block_allocator->release(this);
530}
531
532 int VulkanMemoryAllocator::destroy(void *user_context) {
533#if defined(HL_VK_DEBUG_MEM)
534 debug(nullptr) << "VulkanMemoryAllocator: Destroying allocator ("
535 << "user_context=" << user_context << ") ... \n";
536#endif
537 if (block_allocator != nullptr) {
538 BlockAllocator::destroy(user_context, block_allocator);
539 block_allocator = nullptr;
540 }
541 region_count = 0;
542 region_byte_count = 0;
543 block_count = 0;
544 block_byte_count = 0;
545 return halide_error_code_success;
546}
547
548 const VulkanMemoryConfig &
549 VulkanMemoryAllocator::default_config() {
550 static VulkanMemoryConfig result;
551 return result;
552}
553
554// --
555 int VulkanMemoryAllocator::lookup_requirements(void *user_context, size_t size, uint32_t usage_flags, VkMemoryRequirements *memory_requirements) {
556#if defined(HL_VK_DEBUG_MEM)
557 debug(nullptr) << "VulkanMemoryAllocator: Looking up requirements ("
558 << "user_context=" << user_context << " "
559 << "size=" << (uint32_t)block->size << ", "
560 << "usage_flags=" << usage_flags << ") ... \n";
561#endif
562 VkBufferCreateInfo create_info = {
563 VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, // struct type
564 nullptr, // struct extending this
565 0, // create flags
566 size, // buffer size (in bytes)
567 usage_flags, // buffer usage flags
568 VK_SHARING_MODE_EXCLUSIVE, // sharing mode
569 0, nullptr};
570
571 // Create a buffer to determine alignment requirements
572 VkBuffer buffer = VK_NULL_HANDLE;
573 VkResult result = vkCreateBuffer(this->device, &create_info, this->alloc_callbacks, &buffer);
574 if (result != VK_SUCCESS) {
575#if defined(HL_VK_DEBUG_MEM)
576 debug(nullptr) << "VulkanMemoryAllocator: Failed to create buffer to find requirements!\n\t"
577 << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
578#endif
579 error(user_context) << "VulkanRegionAllocator: Failed to create buffer to gather memory requirements!\n";
580 return halide_error_code_device_malloc_failed;
581 }
582
583 vkGetBufferMemoryRequirements(this->device, buffer, memory_requirements);
584 vkDestroyBuffer(this->device, buffer, this->alloc_callbacks);
585 return halide_error_code_success;
586}
587
588 int VulkanMemoryAllocator::conform_block_request(void *instance_ptr, MemoryRequest *request) {
589
590 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
591 if (instance == nullptr) {
592 return halide_error_code_success;
593 }
594
595 void *user_context = instance->owner_context;
596#if defined(HL_VK_DEBUG_MEM)
597 debug(nullptr) << "VulkanMemoryAllocator: Conforming block request ("
598 << "user_context=" << user_context << " "
599 << "request=" << (void *)(request) << ") ... \n";
600#endif
601
602 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
603 error(user_context) << "VulkanRegionAllocator: Unable to conform block request! Invalid device handle!\n";
604 return halide_error_code_internal_error;
605 }
606
607 VkMemoryRequirements memory_requirements = {0};
608 uint32_t usage_flags = instance->select_memory_usage(user_context, request->properties);
609 int error_code = instance->lookup_requirements(user_context, request->size, usage_flags, &memory_requirements);
610 if (error_code != halide_error_code_success) {
611 error(user_context) << "VulkanRegionAllocator: Failed to conform block request! Unable to lookup requirements!\n";
612 return error_code;
613 }
614
615#if defined(HL_VK_DEBUG_MEM)
616 debug(nullptr) << "VulkanMemoryAllocator: Block allocated ("
617 << "size=" << (uint32_t)request->size << ", "
618 << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
619 << "required_size=" << (uint32_t)memory_requirements.size << ", "
620 << "uniform_buffer_offset_alignment=" << (uint32_t)instance->physical_device_limits.minUniformBufferOffsetAlignment << ", "
621 << "storage_buffer_offset_alignment=" << (uint32_t)instance->physical_device_limits.minStorageBufferOffsetAlignment << ", "
622 << "dedicated=" << (request->dedicated ? "true" : "false") << ")\n";
623#endif
624
625 request->size = memory_requirements.size;
626 request->properties.alignment = memory_requirements.alignment;
627 return halide_error_code_success;
628}
629
630 int VulkanMemoryAllocator::allocate_block(void *instance_ptr, MemoryBlock *block) {
631 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
632 if (instance == nullptr) {
633 return halide_error_code_success;
634 }
635
636 void *user_context = instance->owner_context;
637 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
638 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device handle!\n";
639 return halide_error_code_internal_error;
640 }
641
642 if (block == nullptr) {
643 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid pointer!\n";
644 return halide_error_code_internal_error;
645 }
646
647#if defined(HL_VK_DEBUG_MEM)
648 debug(nullptr) << "VulkanMemoryAllocator: Allocating block ("
649 << "user_context=" << user_context << " "
650 << "block=" << (void *)(block) << " "
651 << "size=" << (uint64_t)block->size << ", "
652 << "dedicated=" << (block->dedicated ? "true" : "false") << " "
653 << "usage=" << halide_memory_usage_name(block->properties.usage) << " "
654 << "caching=" << halide_memory_caching_name(block->properties.caching) << " "
655 << "visibility=" << halide_memory_visibility_name(block->properties.visibility) << ")\n";
656#endif
657
658 VkDeviceMemory *device_memory = (VkDeviceMemory *)vk_host_malloc(nullptr, sizeof(VkDeviceMemory), 0, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT, instance->alloc_callbacks);
659 if (device_memory == nullptr) {
660 error(user_context) << "VulkanBlockAllocator: Unable to allocate block! Failed to allocate device memory handle!\n";
661 return halide_error_code_out_of_memory;
662 }
663
664 VkPhysicalDeviceMemoryProperties memory_properties;
665 vkGetPhysicalDeviceMemoryProperties(instance->physical_device, &memory_properties);
666
667 // Attempt to allocate device memory with the requested flags, trying each type until successful
668 uint32_t selected_type = invalid_memory_type;
669 uint32_t preferred_flags = instance->preferred_flags_for_memory_properties(user_context, block->properties);
670 uint32_t required_flags = instance->required_flags_for_memory_properties(user_context, block->properties);
671
672 // Try preferred flags first
673 for (uint32_t type_index = 0; type_index < memory_properties.memoryTypeCount; type_index++) {
674 if (!instance->is_preferred_memory_type_for_flags(user_context, &memory_properties, type_index, preferred_flags, required_flags)) {
675 continue;
676 }
677
678 // Allocate memory
679 VkMemoryAllocateInfo alloc_info = {
680 VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, // struct type
681 nullptr, // struct extending this
682 block->size, // size of allocation in bytes
683 type_index // memory type index from physical device
684 };
685
686 VkResult result = vkAllocateMemory(instance->device, &alloc_info, instance->alloc_callbacks, device_memory);
687 if (result == VK_SUCCESS) {
688 selected_type = type_index;
689 break;
690 }
691
692#if defined(HL_VK_DEBUG_MEM)
693 debug(user_context) << "VulkanMemoryAllocator: Allocation request for preferred flags was not successful (\n"
694 << "\tblock=" << (void *)(block) << "\n"
695 << "\ttype_index=" << type_index << "\n"
696 << "\tsize=" << (uint64_t)block->size << "\n"
697 << "\tdedicated=" << (block->dedicated ? "true" : "false") << "\n"
698 << "\tusage=" << halide_memory_usage_name(block->properties.usage) << "\n"
699 << "\tcaching=" << halide_memory_caching_name(block->properties.caching) << "\n"
700 << "\tvisibility=" << halide_memory_visibility_name(block->properties.visibility) << "\n"
701 << "\tblocks_allocated=" << (uint64_t)instance->blocks_allocated() << "\n"
702 << "\tbytes_allocated_for_blocks=" << (uint64_t)instance->bytes_allocated_for_blocks() << "\n"
703 << "\tregions_allocated=" << (uint64_t)instance->regions_allocated() << "\n"
704 << "\tbytes_allocated_for_regions=" << (uint64_t)instance->bytes_allocated_for_regions() << "\n)\n"
705 << "vkAllocateMemory returned: "
706 << vk_get_error_name(result) << "\n";
707#endif
708 }
709
710 // try any valid type on any valid pool
711 if (selected_type == invalid_memory_type) {
712 for (uint32_t type_index = 0; type_index < memory_properties.memoryTypeCount; type_index++) {
713 if (!instance->is_valid_memory_type_for_flags(user_context, &memory_properties, type_index, required_flags)) {
714 continue;
715 }
716
717 // Allocate memory
718 VkMemoryAllocateInfo alloc_info = {
719 VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, // struct type
720 nullptr, // struct extending this
721 block->size, // size of allocation in bytes
722 type_index // memory type index from physical device
723 };
724
725 VkResult result = vkAllocateMemory(instance->device, &alloc_info, instance->alloc_callbacks, device_memory);
726 if (result == VK_SUCCESS) {
727 selected_type = type_index;
728 break;
729 }
730#if defined(HL_VK_DEBUG_MEM)
731 debug(user_context) << "VulkanMemoryAllocator: Allocation request for valid flags was not successful (\n"
732 << "\tblock=" << (void *)(block) << "\n"
733 << "\ttype_index=" << type_index << "\n"
734 << "\tsize=" << (uint64_t)block->size << "\n"
735 << "\tdedicated=" << (block->dedicated ? "true" : "false") << "\n"
736 << "\tusage=" << halide_memory_usage_name(block->properties.usage) << "\n"
737 << "\tcaching=" << halide_memory_caching_name(block->properties.caching) << "\n"
738 << "\tvisibility=" << halide_memory_visibility_name(block->properties.visibility) << "\n"
739 << "\tblocks_allocated=" << (uint64_t)instance->blocks_allocated() << "\n"
740 << "\tbytes_allocated_for_blocks=" << (uint64_t)instance->bytes_allocated_for_blocks() << "\n"
741 << "\tregions_allocated=" << (uint64_t)instance->regions_allocated() << "\n"
742 << "\tbytes_allocated_for_regions=" << (uint64_t)instance->bytes_allocated_for_regions() << "\n)\n"
743 << "vkAllocateMemory returned: "
744 << vk_get_error_name(result) << "\n";
745#endif
746 }
747 }
748
749 if (selected_type == invalid_memory_type) {
750 vk_host_free(nullptr, device_memory, instance->alloc_callbacks);
751 device_memory = VK_NULL_HANDLE;
752 error(user_context) << "VulkanMemoryAllocator: Allocation failed for block (\n"
753 << "\tblock=" << (void *)(block) << "\n"
754 << "\tsize=" << (uint64_t)block->size << "\n"
755 << "\tdedicated=" << (block->dedicated ? "true" : "false") << "\n"
756 << "\tusage=" << halide_memory_usage_name(block->properties.usage) << "\n"
757 << "\tcaching=" << halide_memory_caching_name(block->properties.caching) << "\n"
758 << "\tvisibility=" << halide_memory_visibility_name(block->properties.visibility) << "\n"
759 << "\tblocks_allocated=" << (uint64_t)instance->blocks_allocated() << "\n"
760 << "\tbytes_allocated_for_blocks=" << (uint64_t)instance->bytes_allocated_for_blocks() << "\n"
761 << "\tregions_allocated=" << (uint64_t)instance->regions_allocated() << "\n"
762 << "\tbytes_allocated_for_regions=" << (uint64_t)instance->bytes_allocated_for_regions() << "\n)\n";
763 return halide_error_code_device_malloc_failed;
764 }
765
766#ifdef DEBUG_RUNTIME
767 debug(nullptr) << "vkAllocateMemory: Allocated memory for device region (" << (uint64_t)block->size << " bytes) ...\n";
768#endif
769
770 block->handle = (void *)device_memory;
771 instance->block_byte_count += block->size;
772 instance->block_count++;
773 return halide_error_code_success;
774}
775
776 int VulkanMemoryAllocator::deallocate_block(void *instance_ptr, MemoryBlock *block) {
777 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
778 if (instance == nullptr) {
779 return halide_error_code_success;
780 }
781
782 void *user_context = instance->owner_context;
783#if defined(HL_VK_DEBUG_MEM)
784 debug(nullptr) << "VulkanMemoryAllocator: Deallocating block ("
785 << "user_context=" << user_context << " "
786 << "block=" << (void *)(block) << ") ... \n";
787#endif
788
789 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
790 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device handle!\n";
791 return halide_error_code_internal_error;
792 }
793
794 if (block == nullptr) {
795 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid pointer!\n";
796 return halide_error_code_internal_error;
797 }
798
799#if defined(HL_VK_DEBUG_MEM)
800 debug(nullptr) << "VulkanBlockAllocator: deallocating block ("
801 << "size=" << (uint32_t)block->size << ", "
802 << "dedicated=" << (block->dedicated ? "true" : "false") << " "
803 << "usage=" << halide_memory_usage_name(block->properties.usage) << " "
804 << "caching=" << halide_memory_caching_name(block->properties.caching) << " "
805 << "visibility=" << halide_memory_visibility_name(block->properties.visibility) << ")\n";
806#endif
807
808 // Nothing to do if device memory was already freed
809 if (block->handle == nullptr) {
810 return halide_error_code_success;
811 }
812
813 VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block->handle);
814 if (device_memory == nullptr) {
815 error(user_context) << "VulkanBlockAllocator: Unable to deallocate block! Invalid device memory handle!\n";
816 return halide_error_code_internal_error;
817 }
818
819 vkFreeMemory(instance->device, *device_memory, instance->alloc_callbacks);
820#ifdef DEBUG_RUNTIME
821 debug(nullptr) << "vkFreeMemory: Deallocated memory for device region (" << (uint64_t)block->size << " bytes) ...\n";
822#endif
823
824 if (instance->block_count > 0) {
825 instance->block_count--;
826 } else {
827 error(nullptr) << "VulkanRegionAllocator: Block counter invalid ... reseting to zero!\n";
828 instance->block_count = 0;
829 }
830
831 if (int64_t(instance->block_byte_count) - int64_t(block->size) >= 0) {
832 instance->block_byte_count -= block->size;
833 } else {
834 error(nullptr) << "VulkanRegionAllocator: Block byte counter invalid ... reseting to zero!\n";
835 instance->block_byte_count = 0;
836 }
837
838 block->handle = nullptr;
839 vk_host_free(nullptr, device_memory, instance->alloc_callbacks);
840 device_memory = nullptr;
841 return halide_error_code_success;
842}
843
844 size_t VulkanMemoryAllocator::blocks_allocated() const {
845 return block_count;
846}
847
848 size_t VulkanMemoryAllocator::bytes_allocated_for_blocks() const {
849 return block_byte_count;
850}
851
852 uint32_t VulkanMemoryAllocator::required_flags_for_memory_properties(
853 void *user_context, MemoryProperties properties) const {
854
855 uint32_t required_flags = 0; //< must have in order to enable requested access
856 switch (properties.visibility) {
857 case MemoryVisibility::DeviceOnly:
858 required_flags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
859 break;
860 case MemoryVisibility::DeviceToHost:
861 required_flags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
862 break;
863 case MemoryVisibility::HostToDevice:
864 required_flags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
865 break;
866 case MemoryVisibility::DefaultVisibility:
867 case MemoryVisibility::HostOnly:
868 break;
869 case MemoryVisibility::InvalidVisibility:
870 error(user_context) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory visibility request!\n\t"
871 << "caching=" << halide_memory_caching_name(properties.caching) << "\n";
872 return invalid_memory_type;
873 };
874 return required_flags;
875}
876
877 uint32_t VulkanMemoryAllocator::preferred_flags_for_memory_properties(
878 void *user_context, MemoryProperties properties) const {
879
880 uint32_t preferred_flags = 0; //< preferred memory flags for requested access type
881 switch (properties.visibility) {
882 case MemoryVisibility::HostOnly:
883 preferred_flags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
884 break;
885 case MemoryVisibility::DeviceToHost:
886 preferred_flags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
887 break;
888 case MemoryVisibility::HostToDevice:
889 case MemoryVisibility::DefaultVisibility:
890 case MemoryVisibility::DeviceOnly:
891 break;
892 case MemoryVisibility::InvalidVisibility:
893 error(user_context) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory visibility request!\n\t"
894 << "caching=" << halide_memory_caching_name(properties.caching) << "\n";
895 return invalid_memory_type;
896 };
897
898 uint32_t required_flags = required_flags_for_memory_properties(user_context, properties);
899 switch (properties.caching) {
900 case MemoryCaching::CachedCoherent:
901 if (required_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
902 preferred_flags |= VK_MEMORY_PROPERTY_HOST_CACHED_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
903 }
904 break;
905 case MemoryCaching::UncachedCoherent:
906 if (required_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
907 preferred_flags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
908 }
909 break;
910 case MemoryCaching::Cached:
911 if (required_flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
912 preferred_flags |= VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
913 }
914 break;
915 case MemoryCaching::Uncached:
916 case MemoryCaching::DefaultCaching:
917 break;
918 case MemoryCaching::InvalidCaching:
919 default:
920 error(user_context) << "VulkanMemoryAllocator: Unable to convert type! Invalid memory caching request!\n\t"
921 << "caching=" << halide_memory_caching_name(properties.caching) << "\n";
922 return invalid_memory_type;
923 };
924 return preferred_flags;
925}
926
927 bool VulkanMemoryAllocator::is_valid_memory_type_for_heap(void *user_context,
928 VkPhysicalDeviceMemoryProperties *memory_properties,
929 uint32_t memory_type_index,
930 uint32_t heap_index) const {
931
932 if (memory_type_index < memory_properties->memoryTypeCount) {
933 if (memory_properties->memoryTypes[memory_type_index].heapIndex == heap_index) {
934 return true;
935 }
936 }
937 return false;
938}
939
940 bool VulkanMemoryAllocator::is_valid_memory_type_for_flags(void *user_context,
941 VkPhysicalDeviceMemoryProperties *memory_properties,
942 uint32_t memory_type_index,
943 uint32_t required_flags) const {
944
945 if (memory_type_index < memory_properties->memoryTypeCount) {
946 const VkMemoryPropertyFlags property_flags = memory_properties->memoryTypes[memory_type_index].propertyFlags;
947 if (required_flags) {
948 if ((property_flags & required_flags) != required_flags) {
949 return false;
950 }
951 }
952 return true;
953 }
954 return false;
955}
956
957 bool VulkanMemoryAllocator::is_preferred_memory_type_for_flags(void *user_context,
958 VkPhysicalDeviceMemoryProperties *memory_properties,
959 uint32_t memory_type_index,
960 uint32_t preferred_flags,
961 uint32_t required_flags) const {
962
963 if (memory_type_index < memory_properties->memoryTypeCount) {
964 const VkMemoryPropertyFlags property_flags = memory_properties->memoryTypes[memory_type_index].propertyFlags;
965 if (required_flags) {
966 if ((property_flags & required_flags) != required_flags) {
967 return false;
968 }
969 }
970
971 if (preferred_flags) {
972 if ((property_flags & preferred_flags) != preferred_flags) {
973 return false;
974 }
975 }
976 return true;
977 }
978 return false;
979}
980
981// --
982
983 int VulkanMemoryAllocator::conform(void *user_context, MemoryRequest *request) {
984
985 // NOTE: Vulkan will only allow us to bind device memory to a buffer if the memory requirements are met.
986 // So now we have to check those (on every allocation) and potentially recreate the buffer if the requirements
987 // don't match the requested VkBuffer's properties. Note that this is the internal storage for the driver,
988 // whose size may be required to larger than our requested size (even though we will only ever touch the
989 // size of the region we're managing as within our block)
990
991 VkMemoryRequirements memory_requirements = {0};
992 uint32_t usage_flags = select_memory_usage(user_context, request->properties);
993 int error_code = lookup_requirements(user_context, request->size, usage_flags, &memory_requirements);
994 if (error_code != halide_error_code_success) {
995 error(user_context) << "VulkanRegionAllocator: Failed to conform block request! Unable to lookup requirements!\n";
996 return error_code;
997 }
998
999#if defined(HL_VK_DEBUG_MEM)
1000 debug(nullptr) << "VulkanMemoryAllocator: Buffer requirements ("
1001 << "requested_size=" << (uint32_t)region->size << ", "
1002 << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
1003 << "required_size=" << (uint32_t)memory_requirements.size << ")\n";
1004#endif
1005
1006 // Enforce any alignment constraints reported by the device limits for each usage type
1007 if (usage_flags & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) {
1008 if ((request->alignment % this->physical_device_limits.minStorageBufferOffsetAlignment) != 0) {
1009 request->alignment = this->physical_device_limits.minStorageBufferOffsetAlignment;
1010 }
1011 } else if (usage_flags & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) {
1012 if ((request->alignment % this->physical_device_limits.minUniformBufferOffsetAlignment) != 0) {
1013 request->alignment = this->physical_device_limits.minUniformBufferOffsetAlignment;
1014 }
1015 }
1016
1017 // Ensure the request ends on an aligned address
1018 if (request->alignment > config.nearest_multiple) {
1019 request->properties.nearest_multiple = request->alignment;
1020 }
1021
1022 size_t actual_alignment = conform_alignment(request->alignment, memory_requirements.alignment);
1023 size_t actual_offset = aligned_offset(request->offset, actual_alignment);
1024 size_t actual_size = conform_size(actual_offset, memory_requirements.size, actual_alignment, request->properties.nearest_multiple);
1025
1026#if defined(HL_VK_DEBUG_MEM)
1027 if ((request->size != actual_size) || (request->alignment != actual_alignment) || (request->offset != actual_offset)) {
1028 debug(nullptr) << "VulkanMemoryAllocator: Adjusting request to match requirements (\n"
1029 << " size = " << (uint64_t)request->size << " => " << (uint64_t)actual_size << ",\n"
1030 << " alignment = " << (uint64_t)request->alignment << " => " << (uint64_t)actual_alignment << ",\n"
1031 << " offset = " << (uint64_t)request->offset << " => " << (uint64_t)actual_offset << ",\n"
1032 << " required.size = " << (uint64_t)memory_requirements.size << ",\n"
1033 << " required.alignment = " << (uint64_t)memory_requirements.alignment << "\n)\n";
1034 }
1035#endif
1036 request->size = actual_size;
1037 request->alignment = actual_alignment;
1038 request->offset = actual_offset;
1039
1040 return halide_error_code_success;
1041}
1042
1043 int VulkanMemoryAllocator::conform_region_request(void *instance_ptr, MemoryRequest *request) {
1044
1045 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
1046 if (instance == nullptr) {
1047 return halide_error_code_success;
1048 }
1049
1050 void *user_context = instance->owner_context;
1051#if defined(HL_VK_DEBUG_MEM)
1052 debug(nullptr) << "VulkanMemoryAllocator: Conforming region request ("
1053 << "user_context=" << user_context << " "
1054 << "request=" << (void *)(region) << ") ... \n";
1055#endif
1056
1057 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
1058 error(user_context) << "VulkanRegionAllocator: Unable to conform region request! Invalid device handle!\n";
1059 return halide_error_code_internal_error;
1060 }
1061
1062#if defined(HL_VK_DEBUG_MEM)
1063 debug(nullptr) << "VulkanRegionAllocator: Conforming region request ("
1064 << "size=" << (uint32_t)request->size << ", "
1065 << "offset=" << (uint32_t)request->offset << ", "
1066 << "dedicated=" << (request->dedicated ? "true" : "false") << " "
1067 << "usage=" << halide_memory_usage_name(request->properties.usage) << " "
1068 << "caching=" << halide_memory_caching_name(request->properties.caching) << " "
1069 << "visibility=" << halide_memory_visibility_name(request->properties.visibility) << ")\n";
1070#endif
1071
1072 return instance->conform(user_context, request);
1073}
1074
1075 int VulkanMemoryAllocator::allocate_region(void *instance_ptr, MemoryRegion *region) {
1076
1077 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
1078 if (instance == nullptr) {
1079 return halide_error_code_success;
1080 }
1081
1082 void *user_context = instance->owner_context;
1083#if defined(HL_VK_DEBUG_MEM)
1084 debug(nullptr) << "VulkanMemoryAllocator: Allocating region ("
1085 << "user_context=" << user_context << " "
1086 << "region=" << (void *)(region) << ") ... \n";
1087#endif
1088
1089 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
1090 error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Invalid device handle!\n";
1091 return halide_error_code_internal_error;
1092 }
1093
1094 if (region == nullptr) {
1095 error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Invalid pointer!\n";
1096 return halide_error_code_internal_error;
1097 }
1098
1099#if defined(HL_VK_DEBUG_MEM)
1100 debug(nullptr) << "VulkanRegionAllocator: Allocating region ("
1101 << "size=" << (uint32_t)region->size << ", "
1102 << "offset=" << (uint32_t)region->offset << ", "
1103 << "dedicated=" << (region->dedicated ? "true" : "false") << " "
1104 << "usage=" << halide_memory_usage_name(region->properties.usage) << " "
1105 << "caching=" << halide_memory_caching_name(region->properties.caching) << " "
1106 << "visibility=" << halide_memory_visibility_name(region->properties.visibility) << ")\n";
1107#endif
1108
1109 uint32_t usage_flags = instance->select_memory_usage(user_context, region->properties);
1110
1111 VkBufferCreateInfo create_info = {
1112 VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, // struct type
1113 nullptr, // struct extending this
1114 0, // create flags
1115 region->size, // buffer size (in bytes)
1116 usage_flags, // buffer usage flags
1117 VK_SHARING_MODE_EXCLUSIVE, // sharing mode
1118 0, nullptr};
1119
1120 VkBuffer *buffer = (VkBuffer *)vk_host_malloc(nullptr, sizeof(VkBuffer), 0, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT, instance->alloc_callbacks);
1121 if (buffer == nullptr) {
1122 error(user_context) << "VulkanRegionAllocator: Unable to allocate region! Failed to allocate buffer handle!\n";
1123 return halide_error_code_out_of_memory;
1124 }
1125
1126 VkResult result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
1127 if (result != VK_SUCCESS) {
1128 // Allocation failed ... collect unused regions and try again ...
1129 instance->collect(user_context);
1130 result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
1131 }
1132 if (result != VK_SUCCESS) {
1133 vk_host_free(nullptr, buffer, instance->alloc_callbacks);
1134 buffer = nullptr;
1135 error(user_context) << "VulkanRegionAllocator: Failed to create buffer!\n\t"
1136 << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
1137 return halide_error_code_device_malloc_failed;
1138 }
1139
1140 // NOTE: Vulkan will only allow us to bind device memory to a buffer if the memory requirements are met.
1141 // So now we have to check those (on every allocation) and potentially recreate the buffer if the requirements
1142 // don't match the requested VkBuffer's properties. Note that this is the internal storage for the driver,
1143 // whose size may be required to larger than our requested size (even though we will only ever touch the
1144 // size of the region we're managing as within our block)
1145 VkMemoryRequirements memory_requirements = {0};
1146 vkGetBufferMemoryRequirements(instance->device, *buffer, &memory_requirements);
1147
1148#if defined(HL_VK_DEBUG_MEM)
1149 debug(nullptr) << "VulkanMemoryAllocator: Buffer requirements ("
1150 << "requested_size=" << (uint32_t)region->size << ", "
1151 << "required_alignment=" << (uint32_t)memory_requirements.alignment << ", "
1152 << "required_size=" << (uint32_t)memory_requirements.size << ")\n";
1153#endif
1154
1155 if (memory_requirements.size > region->size) {
1156 vkDestroyBuffer(instance->device, *buffer, instance->alloc_callbacks);
1157#ifdef DEBUG_RUNTIME
1158 debug(nullptr) << "VulkanMemoryAllocator: Reallocating buffer to match required size ("
1159 << (uint64_t)region->size << " => " << (uint64_t)memory_requirements.size << " bytes) ...\n";
1160#endif
1161 create_info.size = memory_requirements.size;
1162 VkResult result = vkCreateBuffer(instance->device, &create_info, instance->alloc_callbacks, buffer);
1163 if (result != VK_SUCCESS) {
1164 error(user_context) << "VulkanRegionAllocator: Failed to recreate buffer!\n\t"
1165 << "vkCreateBuffer returned: " << vk_get_error_name(result) << "\n";
1166 return halide_error_code_device_malloc_failed;
1167 }
1168 }
1169
1170#ifdef DEBUG_RUNTIME
1171 debug(nullptr) << "vkCreateBuffer: Created buffer for device region (" << (uint64_t)region->size << " bytes) ...\n";
1172#endif
1173
1174 RegionAllocator *region_allocator = RegionAllocator::find_allocator(user_context, region);
1175 if (region_allocator == nullptr) {
1176 error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid region allocator!\n";
1177 return halide_error_code_internal_error;
1178 }
1179
1180 BlockResource *block_resource = region_allocator->block_resource();
1181 if (block_resource == nullptr) {
1182 error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid block resource handle!\n";
1183 return halide_error_code_internal_error;
1184 }
1185
1186 VkDeviceMemory *device_memory = reinterpret_cast<VkDeviceMemory *>(block_resource->memory.handle);
1187 if (device_memory == nullptr) {
1188 error(user_context) << "VulkanBlockAllocator: Unable to allocate region! Invalid device memory handle!\n";
1189 return halide_error_code_internal_error;
1190 }
1191
1192 // Finally, bind buffer to the device memory
1193 result = vkBindBufferMemory(instance->device, *buffer, *device_memory, region->offset);
1194 if (result != VK_SUCCESS) {
1195 error(user_context) << "VulkanRegionAllocator: Failed to bind buffer!\n\t"
1196 << "vkBindBufferMemory returned: " << vk_get_error_name(result) << "\n";
1197 return halide_error_code_generic_error;
1198 }
1199
1200 region->handle = (void *)buffer;
1201 region->is_owner = true;
1202 instance->region_byte_count += region->size;
1203 instance->region_count++;
1204 return halide_error_code_success;
1205}
1206
1207 int VulkanMemoryAllocator::deallocate_region(void *instance_ptr, MemoryRegion *region) {
1208 VulkanMemoryAllocator *instance = reinterpret_cast<VulkanMemoryAllocator *>(instance_ptr);
1209 if (instance == nullptr) {
1210 return halide_error_code_success;
1211 }
1212
1213 void *user_context = instance->owner_context;
1214#if defined(HL_VK_DEBUG_MEM)
1215 debug(nullptr) << "VulkanMemoryAllocator: Deallocating region ("
1216 << "user_context=" << user_context << " "
1217 << "region=" << (void *)(region) << ") ... \n";
1218#endif
1219
1220 if ((instance->device == nullptr) || (instance->physical_device == nullptr)) {
1221 error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid device handle!\n";
1222 return halide_error_code_internal_error;
1223 }
1224
1225 if (region == nullptr) {
1226 error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid pointer!\n";
1227 return halide_error_code_internal_error;
1228 }
1229
1230#if defined(HL_VK_DEBUG_MEM)
1231 debug(nullptr) << "VulkanRegionAllocator: Deallocating region ("
1232 << "size=" << (uint32_t)region->size << ", "
1233 << "offset=" << (uint32_t)region->offset << ", "
1234 << "dedicated=" << (region->dedicated ? "true" : "false") << " "
1235 << "usage=" << halide_memory_usage_name(region->properties.usage) << " "
1236 << "caching=" << halide_memory_caching_name(region->properties.caching) << " "
1237 << "visibility=" << halide_memory_visibility_name(region->properties.visibility) << ")\n";
1238#endif
1239
1240 if (region->handle == nullptr) {
1241 error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid handle!\n";
1242 return halide_error_code_internal_error;
1243 }
1244
1245 VkBuffer *buffer = reinterpret_cast<VkBuffer *>(region->handle);
1246 if (buffer == nullptr) {
1247 error(user_context) << "VulkanRegionAllocator: Unable to deallocate region! Invalid buffer handle!\n";
1248 return halide_error_code_internal_error;
1249 }
1250
1251 vkDestroyBuffer(instance->device, *buffer, instance->alloc_callbacks);
1252#ifdef DEBUG_RUNTIME
1253 debug(nullptr) << "vkDestroyBuffer: Destroyed buffer for device region (" << (uint64_t)region->size << " bytes) ...\n";
1254#endif
1255 halide_error_code_t error_code = halide_error_code_success;
1256 region->handle = nullptr;
1257 if (instance->region_count > 0) {
1258 instance->region_count--;
1259 } else {
1260 error(user_context) << "VulkanRegionAllocator: Region counter invalid ... reseting to zero!\n";
1261 instance->region_count = 0;
1262 error_code = halide_error_code_internal_error;
1263 }
1264
1265 if (int64_t(instance->region_byte_count) - int64_t(region->size) >= 0) {
1266 instance->region_byte_count -= region->size;
1267 } else {
1268 error(user_context) << "VulkanRegionAllocator: Region byte counter invalid ... reseting to zero!\n";
1269 instance->region_byte_count = 0;
1270 error_code = halide_error_code_internal_error;
1271 }
1272 vk_host_free(nullptr, buffer, instance->alloc_callbacks);
1273 buffer = nullptr;
1274 return error_code;
1275}
1276
1277 size_t VulkanMemoryAllocator::regions_allocated() const {
1278 return region_count;
1279}
1280
1281 size_t VulkanMemoryAllocator::bytes_allocated_for_regions() const {
1282 return region_byte_count;
1283}
1284
1285 uint32_t VulkanMemoryAllocator::select_memory_usage(void *user_context, MemoryProperties properties) const {
1286 uint32_t result = 0;
1287 switch (properties.usage) {
1288 case MemoryUsage::UniformStorage:
1289 result |= VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT;
1290 break;
1291 case MemoryUsage::DynamicStorage:
1292 case MemoryUsage::StaticStorage:
1293 result |= VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
1294 break;
1295 case MemoryUsage::TransferSrc:
1296 result |= VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
1297 break;
1298 case MemoryUsage::TransferDst:
1299 result |= VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
1300 break;
1301 case MemoryUsage::TransferSrcDst:
1302 result |= VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
1303 break;
1304 case MemoryUsage::DefaultUsage:
1305 case MemoryUsage::InvalidUsage:
1306 default:
1307 error(user_context) << "VulkanRegionAllocator: Unable to convert type! Invalid memory usage request!\n\t"
1308 << "usage=" << halide_memory_usage_name(properties.usage) << "\n";
1309 return invalid_usage_flags;
1310 };
1311
1312 if (result == invalid_usage_flags) {
1313 error(user_context) << "VulkanRegionAllocator: Failed to find appropriate memory usage for given properties:\n\t"
1314 << "usage=" << halide_memory_usage_name(properties.usage) << " "
1315 << "caching=" << halide_memory_caching_name(properties.caching) << " "
1316 << "visibility=" << halide_memory_visibility_name(properties.visibility) << "\n";
1317 return invalid_usage_flags;
1318 }
1319
1320 return result;
1321}
1322
1323// --------------------------------------------------------------------------
1324
1325 namespace {
1326
1327// --------------------------------------------------------------------------
1328// Halide System allocator for host allocations
1329 void *vk_system_malloc(void *user_context, size_t size) {
1330 return malloc(size);
1331}
1332
1333 void vk_system_free(void *user_context, void *ptr) {
1334 free(ptr);
1335}
1336
1337// Vulkan host-side allocation
1338 void *vk_host_malloc(void *user_context, size_t size, size_t alignment, VkSystemAllocationScope scope, const VkAllocationCallbacks *callbacks) {
1339 if (callbacks) {
1340 return callbacks->pfnAllocation(user_context, size, alignment, scope);
1341 } else {
1342 return vk_system_malloc(user_context, size);
1343 }
1344}
1345
1346 void vk_host_free(void *user_context, void *ptr, const VkAllocationCallbacks *callbacks) {
1347 if (callbacks) {
1348 return callbacks->pfnFree(user_context, ptr);
1349 } else {
1350 return vk_system_free(user_context, ptr);
1351 }
1352}
1353
1354VulkanMemoryAllocator *vk_create_memory_allocator(void *user_context,
1355 VkDevice device,
1356 VkPhysicalDevice physical_device,
1357 const VkAllocationCallbacks *alloc_callbacks) {
1358
1359 SystemMemoryAllocatorFns system_allocator = {vk_system_malloc, vk_system_free};
1360 VulkanMemoryConfig config = memory_allocator_config;
1361
1362 // Parse the allocation config string (if specified).
1363 //
1364 // `HL_VK_ALLOC_CONFIG=N:N:N` will tell Halide to configure the Vulkan memory
1365 // allocator use the given constraints specified as three integer values
1366 // separated by a `:` or `;`. These values correspond to `minimum_block_size`,
1367 // `maximum_block_size` and `maximum_block_count`.
1368 //
1369 const char *alloc_config = vk_get_alloc_config_internal(user_context);
1370 if (!StringUtils::is_empty(alloc_config)) {
1371 StringTable alloc_config_values;
1372 alloc_config_values.parse(user_context, alloc_config, HL_VK_ENV_DELIM);
1373 if (alloc_config_values.size() > 0) {
1374 config.maximum_pool_size = atoi(alloc_config_values[0]) * 1024 * 1024;
1375 print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_pool_size << " for maximum pool size (in bytes)\n";
1376 }
1377 if (alloc_config_values.size() > 1) {
1378 config.minimum_block_size = atoi(alloc_config_values[1]) * 1024 * 1024;
1379 print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.minimum_block_size << " for minimum block size (in bytes)\n";
1380 }
1381 if (alloc_config_values.size() > 2) {
1382 config.maximum_block_size = atoi(alloc_config_values[2]) * 1024 * 1024;
1383 print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_block_size << " for maximum block size (in bytes)\n";
1384 }
1385 if (alloc_config_values.size() > 3) {
1386 config.maximum_block_count = atoi(alloc_config_values[3]);
1387 print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.maximum_block_count << " for maximum block count\n";
1388 }
1389 if (alloc_config_values.size() > 4) {
1390 config.nearest_multiple = atoi(alloc_config_values[4]);
1391 print(user_context) << "Vulkan: Configuring allocator with " << (uint32_t)config.nearest_multiple << " for nearest multiple\n";
1392 }
1393 }
1394
1395 return VulkanMemoryAllocator::create(user_context,
1396 config, device, physical_device,
1397 system_allocator, alloc_callbacks);
1398}
1399
1400 int vk_destroy_memory_allocator(void *user_context, VulkanMemoryAllocator *allocator) {
1401 if (allocator != nullptr) {
1402 allocator->collect(user_context);
1403 VulkanMemoryAllocator::destroy(user_context, allocator);
1404 allocator = nullptr;
1405 }
1406 return halide_error_code_success;
1407}
1408
1409// --------------------------------------------------------------------------
1410
1411 int vk_clear_device_buffer(void *user_context,
1412 VulkanMemoryAllocator *allocator,
1413 VkCommandBuffer command_buffer,
1414 VkQueue command_queue,
1415 VkBuffer device_buffer) {
1416
1417#ifdef DEBUG_RUNTIME
1418 debug(user_context)
1419 << " vk_clear_device_buffer (user_context: " << user_context << ", "
1420 << "allocator: " << (void *)allocator << ", "
1421 << "command_buffer: " << (void *)command_buffer << ", "
1422 << "command_queue: " << (void *)command_queue << ", "
1423 << "device_buffer: " << (void *)device_buffer << ")\n";
1424#endif
1425
1426 // begin the command buffer
1427 VkCommandBufferBeginInfo command_buffer_begin_info =
1428 {
1429 VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, // struct type
1430 nullptr, // pointer to struct extending this
1431 VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT, // flags
1432 nullptr // pointer to parent command buffer
1433 };
1434
1435 VkResult result = vkBeginCommandBuffer(command_buffer, &command_buffer_begin_info);
1436 if (result != VK_SUCCESS) {
1437 error(user_context) << "Vulkan: vkBeginCommandBuffer returned " << vk_get_error_name(result) << "\n";
1438 return halide_error_code_generic_error;
1439 }
1440
1441 // fill buffer with zero values up to the size of the buffer
1442 vkCmdFillBuffer(command_buffer, device_buffer, 0, VK_WHOLE_SIZE, 0);
1443
1444 // end the command buffer
1445 result = vkEndCommandBuffer(command_buffer);
1446 if (result != VK_SUCCESS) {
1447 error(user_context) << "Vulkan: vkEndCommandBuffer returned " << vk_get_error_name(result) << "\n";
1448 return halide_error_code_generic_error;
1449 }
1450
1451 // submit the command buffer
1452 VkSubmitInfo submit_info =
1453 {
1454 VK_STRUCTURE_TYPE_SUBMIT_INFO, // struct type
1455 nullptr, // pointer to struct extending this
1456 0, // wait semaphore count
1457 nullptr, // semaphores
1458 nullptr, // pipeline stages where semaphore waits occur
1459 1, // how many command buffers to execute
1460 &command_buffer, // the command buffers
1461 0, // number of semaphores to signal
1462 nullptr // the semaphores to signal
1463 };
1464
1465 result = vkQueueSubmit(command_queue, 1, &submit_info, VK_NULL_HANDLE);
1466 if (result != VK_SUCCESS) {
1467 error(user_context) << "Vulkan: vkQueueSubmit returned " << vk_get_error_name(result) << "\n";
1468 return halide_error_code_generic_error;
1469 }
1470
1471 // wait for memset to finish
1472 result = vkQueueWaitIdle(command_queue);
1473 if (result != VK_SUCCESS) {
1474 error(user_context) << "Vulkan: vkQueueWaitIdle returned " << vk_get_error_name(result) << "\n";
1475 return halide_error_code_generic_error;
1476 }
1477
1478 return halide_error_code_success;
1479}
1480
1481// --------------------------------------------------------------------------
1482
1483} // namespace
1484} // namespace Vulkan
1485} // namespace Internal
1486} // namespace Runtime
1487} // namespace Halide
1488
1489// --------------------------------------------------------------------------
1490
1491 extern "C" {
1492
1493// --------------------------------------------------------------------------
1494
1495 WEAK void halide_vulkan_set_allocation_callbacks(const VkAllocationCallbacks *callbacks) {
1496 using namespace Halide::Runtime::Internal::Vulkan;
1497 ScopedSpinLock lock(&custom_allocation_callbacks_lock);
1498 custom_allocation_callbacks = callbacks;
1499}
1500
1501 WEAK const VkAllocationCallbacks *halide_vulkan_get_allocation_callbacks(void *user_context) {
1502 using namespace Halide::Runtime::Internal::Vulkan;
1503 ScopedSpinLock lock(&custom_allocation_callbacks_lock);
1504 return custom_allocation_callbacks;
1505}
1506
1507// --------------------------------------------------------------------------
1508
1509} // extern "C"
1510
1511#endif // HALIDE_RUNTIME_VULKAN_MEMORY_H
debug
#define debug(n)
For optional debugging during codegen, use the debug macro as follows:
Definition: Debug.h:52
halide_error_code_t
halide_error_code_t
The error codes that may be returned by a Halide pipeline.
Definition: HalideRuntime.h:1074
halide_error_code_internal_error
@ halide_error_code_internal_error
There is a bug in the Halide compiler.
Definition: HalideRuntime.h:1164
halide_error_code_generic_error
@ halide_error_code_generic_error
An uncategorized error occurred.
Definition: HalideRuntime.h:1079
halide_error_code_success
@ halide_error_code_success
There was no error.
Definition: HalideRuntime.h:1076
halide_error_code_device_malloc_failed
@ halide_error_code_device_malloc_failed
The Halide runtime encountered an error while trying to allocate memory on device.
Definition: HalideRuntime.h:1141
halide_error_code_out_of_memory
@ halide_error_code_out_of_memory
A call to halide_malloc returned NULL.
Definition: HalideRuntime.h:1119
block_allocator.h
Halide::Runtime::Internal::BlockAllocator
Allocator class interface for managing large contiguous blocks of memory, which are then sub-allocate...
Definition: block_allocator.h:25
Halide::Runtime::Internal::BlockAllocator::collect
bool collect(void *user_context)
Definition: block_allocator.h:233
Halide::Runtime::Internal::BlockAllocator::reserve
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
Definition: block_allocator.h:155
Halide::Runtime::Internal::BlockAllocator::release
int release(void *user_context, MemoryRegion *region)
Definition: block_allocator.h:200
Halide::Runtime::Internal::BlockAllocator::retain
int retain(void *user_context, MemoryRegion *region)
Definition: block_allocator.h:222
Halide::Runtime::Internal::BlockAllocator::destroy
static void destroy(void *user_context, BlockAllocator *block_allocator)
Definition: block_allocator.h:138
Halide::Runtime::Internal::BlockAllocator::create
static BlockAllocator * create(void *user_context, const Config &config, const MemoryAllocators &allocators)
Definition: block_allocator.h:124
Halide::Runtime::Internal::BlockAllocator::reclaim
int reclaim(void *user_context, MemoryRegion *region)
Definition: block_allocator.h:211
Halide::Runtime::Internal::BlockAllocator::current_allocators
const MemoryAllocators & current_allocators() const
Definition: block_allocator.h:629
Halide::Runtime::Internal::RegionAllocator
Allocator class interface for sub-allocating a contiguous memory block into smaller regions of memory...
Definition: region_allocator.h:24
Halide::Runtime::Internal::RegionAllocator::find_allocator
static RegionAllocator * find_allocator(void *user_context, MemoryRegion *memory_region)
Definition: region_allocator.h:240
Halide::Runtime::Internal::RegionAllocator::retain
int retain(void *user_context, MemoryRegion *memory_region)
Definition: region_allocator.h:232
Halide::Runtime::Internal::RegionAllocator::release
int release(void *user_context, MemoryRegion *memory_region)
Definition: region_allocator.h:210
Halide::Runtime::Internal::RegionAllocator::block_resource
BlockResource * block_resource() const
Definition: region_allocator.h:832
Halide::Runtime::Internal::StringTable
Definition: string_table.h:15
Halide::Runtime::Internal::StringTable::parse
size_t parse(void *user_context, const char *str, const char *delim)
Definition: string_table.h:159
Halide::Runtime::Internal::StringTable::size
size_t size() const
Definition: string_table.h:54
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator
Vulkan Memory Allocator class interface for managing large memory requests stored as contiguous block...
Definition: vulkan_memory.h:42
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::reclaim
int reclaim(void *user_context, MemoryRegion *region)
Definition: vulkan_memory.h:466
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::default_config
static const VulkanMemoryConfig & default_config()
Definition: vulkan_memory.h:549
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::reserve
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
Definition: vulkan_memory.h:227
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::VulkanMemoryAllocator
VulkanMemoryAllocator(const VulkanMemoryAllocator &)=delete
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::release
int release(void *user_context, MemoryRegion *region)
Definition: vulkan_memory.h:447
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::allocate_block
static int allocate_block(void *instance_ptr, MemoryBlock *block)
Definition: vulkan_memory.h:630
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::conform
int conform(void *user_context, MemoryRequest *request)
Definition: vulkan_memory.h:983
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::deallocate_region
static int deallocate_region(void *instance_ptr, MemoryRegion *region)
Definition: vulkan_memory.h:1207
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::operator=
VulkanMemoryAllocator & operator=(const VulkanMemoryAllocator &)=delete
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::destroy
static int destroy(void *user_context, VulkanMemoryAllocator *allocator)
Definition: vulkan_memory.h:174
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::unmap
int unmap(void *user_context, MemoryRegion *region)
Definition: vulkan_memory.h:316
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::conform_region_request
static int conform_region_request(void *instance_ptr, MemoryRequest *request)
Definition: vulkan_memory.h:1043
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::~VulkanMemoryAllocator
~VulkanMemoryAllocator()=delete
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::create_crop
MemoryRegion * create_crop(void *user_context, MemoryRegion *region, uint64_t offset)
Definition: vulkan_memory.h:355
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::destroy_crop
int destroy_crop(void *user_context, MemoryRegion *region)
Definition: vulkan_memory.h:408
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::regions_allocated
size_t regions_allocated() const
Definition: vulkan_memory.h:1277
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::blocks_allocated
size_t blocks_allocated() const
Definition: vulkan_memory.h:844
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::allocate_region
static int allocate_region(void *instance_ptr, MemoryRegion *region)
Definition: vulkan_memory.h:1075
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::conform_block_request
static int conform_block_request(void *instance_ptr, MemoryRequest *request)
Definition: vulkan_memory.h:588
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::owner_of
MemoryRegion * owner_of(void *user_context, MemoryRegion *region)
Definition: vulkan_memory.h:438
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::current_device
VkDevice current_device() const
Definition: vulkan_memory.h:76
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::current_physical_device_limits
VkPhysicalDeviceLimits current_physical_device_limits() const
Definition: vulkan_memory.h:82
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::collect
bool collect(void *user_context)
Definition: vulkan_memory.h:504
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::bytes_allocated_for_blocks
size_t bytes_allocated_for_blocks() const
Definition: vulkan_memory.h:848
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::callbacks
const VkAllocationCallbacks * callbacks() const
Definition: vulkan_memory.h:85
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::deallocate_block
static int deallocate_block(void *instance_ptr, MemoryBlock *block)
Definition: vulkan_memory.h:776
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::VulkanMemoryAllocator
VulkanMemoryAllocator()=delete
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::map
void * map(void *user_context, MemoryRegion *region)
Definition: vulkan_memory.h:250
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::current_physical_device
VkPhysicalDevice current_physical_device() const
Definition: vulkan_memory.h:79
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::bytes_allocated_for_regions
size_t bytes_allocated_for_regions() const
Definition: vulkan_memory.h:1281
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::retain
int retain(void *user_context, MemoryRegion *region)
Definition: vulkan_memory.h:485
Halide::Runtime::Internal::Vulkan::VulkanMemoryAllocator::create
static VulkanMemoryAllocator * create(void *user_context, const VulkanMemoryConfig &config, VkDevice dev, VkPhysicalDevice phys_dev, const SystemMemoryAllocatorFns &system_allocator, const VkAllocationCallbacks *alloc_callbacks=nullptr)
Definition: vulkan_memory.h:152
memory_resources.h
halide_memory_caching_name
WEAK const char * halide_memory_caching_name(MemoryCaching value)
Definition: memory_resources.h:293
halide_memory_usage_name
WEAK const char * halide_memory_usage_name(MemoryUsage value)
Definition: memory_resources.h:260
halide_memory_visibility_name
WEAK const char * halide_memory_visibility_name(MemoryVisibility value)
Definition: memory_resources.h:233
Halide::Internal::destroy
void destroy(const T *t)
Halide::Runtime::Internal::Vulkan
Definition: vulkan_context.h:17
Halide::Runtime::Internal::Vulkan::memory_allocator_config
WEAK VulkanMemoryConfig memory_allocator_config
Definition: vulkan_memory.h:32
Halide::Runtime::Internal::Vulkan::custom_allocation_callbacks
WEAK const VkAllocationCallbacks * custom_allocation_callbacks
Definition: vulkan_memory.h:20
Halide::Runtime::Internal::Vulkan::custom_allocation_callbacks_lock
WEAK ScopedSpinLock::AtomicFlag custom_allocation_callbacks_lock
Definition: vulkan_memory.h:19
Halide::Runtime::Internal::Vulkan::alloc_config
WEAK char alloc_config[1024]
Definition: vulkan_extensions.h:31
Halide::Runtime::Internal::conform_alignment
ALWAYS_INLINE size_t conform_alignment(size_t requested, size_t required)
Definition: memory_resources.h:135
Halide::Runtime::Internal::MemoryUsage::TransferSrcDst
@ TransferSrcDst
Halide::Runtime::Internal::MemoryUsage::DefaultUsage
@ DefaultUsage
Halide::Runtime::Internal::MemoryUsage::DynamicStorage
@ DynamicStorage
Halide::Runtime::Internal::MemoryUsage::UniformStorage
@ UniformStorage
Halide::Runtime::Internal::MemoryUsage::InvalidUsage
@ InvalidUsage
Halide::Runtime::Internal::MemoryUsage::StaticStorage
@ StaticStorage
Halide::Runtime::Internal::MemoryUsage::TransferSrc
@ TransferSrc
Halide::Runtime::Internal::MemoryUsage::TransferDst
@ TransferDst
Halide::Runtime::Internal::conform_size
ALWAYS_INLINE size_t conform_size(size_t offset, size_t size, size_t alignment, size_t nearest_multiple)
Definition: memory_resources.h:151
Halide::Runtime::Internal::MemoryVisibility::DefaultVisibility
@ DefaultVisibility
Halide::Runtime::Internal::MemoryVisibility::DeviceToHost
@ DeviceToHost
Halide::Runtime::Internal::MemoryVisibility::InvalidVisibility
@ InvalidVisibility
Halide::Runtime::Internal::MemoryVisibility::HostToDevice
@ HostToDevice
Halide::Runtime::Internal::MemoryVisibility::HostOnly
@ HostOnly
Halide::Runtime::Internal::MemoryVisibility::DeviceOnly
@ DeviceOnly
Halide::Runtime::Internal::MemoryCaching::Cached
@ Cached
Halide::Runtime::Internal::MemoryCaching::InvalidCaching
@ InvalidCaching
Halide::Runtime::Internal::MemoryCaching::UncachedCoherent
@ UncachedCoherent
Halide::Runtime::Internal::MemoryCaching::DefaultCaching
@ DefaultCaching
Halide::Runtime::Internal::MemoryCaching::Uncached
@ Uncached
Halide::Runtime::Internal::MemoryCaching::CachedCoherent
@ CachedCoherent
Halide::Runtime::Internal::aligned_offset
ALWAYS_INLINE size_t aligned_offset(size_t offset, size_t alignment)
Definition: memory_resources.h:128
Halide
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
Definition: AbstractGenerator.h:19
Halide::LinkageType::Internal
@ Internal
Not visible externally, similar to 'static' linkage in C.
Halide::DeviceAPI::Vulkan
@ Vulkan
Halide::print
Expr print(const std::vector< Expr > &values)
Create an Expr that prints out its value whenever it is evaluated.
uint64_t
unsigned __INT64_TYPE__ uint64_t
Definition: runtime_internal.h:23
int64_t
signed __INT64_TYPE__ int64_t
Definition: runtime_internal.h:22
atoi
int atoi(const char *)
malloc
void * malloc(size_t)
memcpy
void * memcpy(void *s1, const void *s2, size_t n)
memset
void * memset(void *s, int val, size_t n)
uint32_t
unsigned __INT32_TYPE__ uint32_t
Definition: runtime_internal.h:25
WEAK
#define WEAK
Definition: runtime_internal.h:52
free
void free(void *)
Halide::Runtime::Internal::BlockAllocator::Config
Definition: block_allocator.h:43
Halide::Runtime::Internal::BlockAllocator::Config::maximum_block_count
size_t maximum_block_count
Definition: block_allocator.h:48
Halide::Runtime::Internal::BlockAllocator::Config::nearest_multiple
size_t nearest_multiple
Definition: block_allocator.h:49
Halide::Runtime::Internal::BlockAllocator::Config::minimum_block_size
size_t minimum_block_size
Definition: block_allocator.h:46
Halide::Runtime::Internal::BlockAllocator::Config::maximum_pool_size
size_t maximum_pool_size
Definition: block_allocator.h:45
Halide::Runtime::Internal::BlockAllocator::Config::maximum_block_size
size_t maximum_block_size
Definition: block_allocator.h:47
Halide::Runtime::Internal::BlockAllocator::MemoryAllocators
Definition: block_allocator.h:36
Halide::Runtime::Internal::BlockAllocator::MemoryAllocators::region
MemoryRegionAllocatorFns region
Definition: block_allocator.h:39
Halide::Runtime::Internal::BlockAllocator::MemoryAllocators::block
MemoryBlockAllocatorFns block
Definition: block_allocator.h:38
Halide::Runtime::Internal::BlockAllocator::MemoryAllocators::system
SystemMemoryAllocatorFns system
Definition: block_allocator.h:37
Halide::Runtime::Internal::BlockResource
Definition: memory_resources.h:103
Halide::Runtime::Internal::BlockResource::memory
MemoryBlock memory
Definition: memory_resources.h:104
Halide::Runtime::Internal::MemoryBlock
Definition: memory_resources.h:65
Halide::Runtime::Internal::MemoryBlock::handle
void * handle
Definition: memory_resources.h:66
Halide::Runtime::Internal::MemoryBlock::dedicated
bool dedicated
Definition: memory_resources.h:68
Halide::Runtime::Internal::MemoryBlock::properties
MemoryProperties properties
Definition: memory_resources.h:69
Halide::Runtime::Internal::MemoryBlock::size
size_t size
Definition: memory_resources.h:67
Halide::Runtime::Internal::MemoryProperties
Definition: memory_resources.h:56
Halide::Runtime::Internal::MemoryProperties::caching
MemoryCaching caching
Definition: memory_resources.h:59
Halide::Runtime::Internal::MemoryProperties::alignment
size_t alignment
Definition: memory_resources.h:60
Halide::Runtime::Internal::MemoryProperties::nearest_multiple
size_t nearest_multiple
Definition: memory_resources.h:61
Halide::Runtime::Internal::MemoryProperties::visibility
MemoryVisibility visibility
Definition: memory_resources.h:57
Halide::Runtime::Internal::MemoryProperties::usage
MemoryUsage usage
Definition: memory_resources.h:58
Halide::Runtime::Internal::MemoryRange::head_offset
size_t head_offset
Definition: memory_resources.h:74
Halide::Runtime::Internal::MemoryRange::tail_offset
size_t tail_offset
Definition: memory_resources.h:75
Halide::Runtime::Internal::MemoryRegion
Definition: memory_resources.h:79
Halide::Runtime::Internal::MemoryRegion::handle
void * handle
Definition: memory_resources.h:80
Halide::Runtime::Internal::MemoryRegion::dedicated
bool dedicated
Definition: memory_resources.h:84
Halide::Runtime::Internal::MemoryRegion::is_owner
bool is_owner
Definition: memory_resources.h:85
Halide::Runtime::Internal::MemoryRegion::size
size_t size
Definition: memory_resources.h:82
Halide::Runtime::Internal::MemoryRegion::range
MemoryRange range
Definition: memory_resources.h:83
Halide::Runtime::Internal::MemoryRegion::properties
MemoryProperties properties
Definition: memory_resources.h:86
Halide::Runtime::Internal::MemoryRegion::offset
size_t offset
Definition: memory_resources.h:81
Halide::Runtime::Internal::MemoryRequest
Definition: memory_resources.h:90
Halide::Runtime::Internal::MemoryRequest::size
size_t size
Definition: memory_resources.h:92
Halide::Runtime::Internal::MemoryRequest::offset
size_t offset
Definition: memory_resources.h:91
Halide::Runtime::Internal::MemoryRequest::dedicated
bool dedicated
Definition: memory_resources.h:94
Halide::Runtime::Internal::MemoryRequest::alignment
size_t alignment
Definition: memory_resources.h:93
Halide::Runtime::Internal::MemoryRequest::properties
MemoryProperties properties
Definition: memory_resources.h:95
Halide::Runtime::Internal::ScopedSpinLock
Definition: scoped_spin_lock.h:9
Halide::Runtime::Internal::ScopedSpinLock::AtomicFlag
char AtomicFlag
Definition: scoped_spin_lock.h:11
Halide::Runtime::Internal::StringUtils::is_empty
static bool is_empty(const char *str)
Definition: string_storage.h:13
Halide::Runtime::Internal::SystemMemoryAllocatorFns
Definition: memory_resources.h:193
Halide::Runtime::Internal::SystemMemoryAllocatorFns::deallocate
DeallocateSystemFn deallocate
Definition: memory_resources.h:195
Halide::Runtime::Internal::SystemMemoryAllocatorFns::allocate
AllocateSystemFn allocate
Definition: memory_resources.h:194
Halide::Runtime::Internal::Vulkan::VulkanMemoryConfig
Definition: vulkan_memory.h:25
Halide::Runtime::Internal::Vulkan::VulkanMemoryConfig::maximum_block_size
size_t maximum_block_size
Definition: vulkan_memory.h:28
Halide::Runtime::Internal::Vulkan::VulkanMemoryConfig::nearest_multiple
size_t nearest_multiple
Definition: vulkan_memory.h:30
Halide::Runtime::Internal::Vulkan::VulkanMemoryConfig::maximum_pool_size
size_t maximum_pool_size
Definition: vulkan_memory.h:26
Halide::Runtime::Internal::Vulkan::VulkanMemoryConfig::minimum_block_size
size_t minimum_block_size
Definition: vulkan_memory.h:27
Halide::Runtime::Internal::Vulkan::VulkanMemoryConfig::maximum_block_count
size_t maximum_block_count
Definition: vulkan_memory.h:29
HL_VK_ENV_DELIM
#define HL_VK_ENV_DELIM
Definition: vulkan_interface.h:14
vulkan_internal.h
halide_vulkan_set_allocation_callbacks
WEAK void halide_vulkan_set_allocation_callbacks(const VkAllocationCallbacks *callbacks)
Definition: vulkan_memory.h:1495
halide_vulkan_get_allocation_callbacks
WEAK const VkAllocationCallbacks * halide_vulkan_get_allocation_callbacks(void *user_context)
Definition: vulkan_memory.h:1501
error_code
int error_code
Definition: vulkan_resources.h:182
user_context
void * user_context
Definition: vulkan_resources.h:178
command_buffer
VkCommandBuffer command_buffer
Definition: vulkan_resources.h:181
allocator
VulkanMemoryAllocator * allocator
Definition: vulkan_resources.h:179