Halide 22.0.0 Halide compiler and libraries

region_allocator.h
Go to the documentation of this file.
1#ifndef HALIDE_RUNTIME_REGION_ALLOCATOR_H
2#define HALIDE_RUNTIME_REGION_ALLOCATOR_H
3
4#include "../HalideRuntime.h"
5#include "../printer.h"
6#include "memory_arena.h"
7#include "memory_resources.h"
8
9 namespace Halide {
10 namespace Runtime {
11 namespace Internal {
12
13// --
14
15/** Allocator class interface for sub-allocating a contiguous
16 * memory block into smaller regions of memory. This class only
17 * manages the address creation for the regions -- allocation
18 * callback functions are used to request the memory from the
19 * necessary system or API calls. This class is intended to be
20 * used inside of a higher level memory management class that
21 * provides thread safety, policy management and API
22 * integration for a specific runtime API (eg Vulkan, OpenCL, etc)
23 */
24 class RegionAllocator {
25 public:
26 // disable copy constructors and assignment
27 RegionAllocator(const RegionAllocator &) = delete;
28 RegionAllocator &operator=(const RegionAllocator &) = delete;
29
30 // disable non-factory based construction
31 RegionAllocator() = delete;
32 ~RegionAllocator() = delete;
33
34 // Allocators for the different types of memory we need to allocate
35 struct MemoryAllocators {
36 SystemMemoryAllocatorFns system;
37 MemoryRegionAllocatorFns region;
38 };
39
40 // Factory methods for creation / destruction
41 static RegionAllocator *create(void *user_context, BlockResource *block, const MemoryAllocators &ma);
42 static int destroy(void *user_context, RegionAllocator *region_allocator);
43
44 // Returns the allocator class instance for the given allocation (or nullptr)
45 static RegionAllocator *find_allocator(void *user_context, MemoryRegion *memory_region);
46
47 // Public interface methods
48 MemoryRegion *reserve(void *user_context, const MemoryRequest &request);
49 int conform(void *user_context, MemoryRequest *request) const; //< conform the given request into a suitable allocation
50 int release(void *user_context, MemoryRegion *memory_region); //< unmark and cache the region for reuse
51 int reclaim(void *user_context, MemoryRegion *memory_region); //< free the region and consolidate
52 int retain(void *user_context, MemoryRegion *memory_region); //< retain the region and increase usage count
53 bool collect(void *user_context); //< returns true if any blocks were removed
54 int release(void *user_context);
55 int destroy(void *user_context);
56
57 // Returns the currently managed block resource
58 BlockResource *block_resource() const;
59
60 private:
61 // Initializes a new instance
62 int initialize(void *user_context, BlockResource *block, const MemoryAllocators &ma);
63
64 // Search through allocated block regions (Best-Fit)
65 BlockRegion *find_block_region(void *user_context, const MemoryRequest &request);
66
67 // Returns true if block region is unused and available
68 bool is_available(const BlockRegion *region) const;
69
70 // Returns true if neighbouring block regions to the given region can be coalesced into one
71 bool can_coalesce(const BlockRegion *region) const;
72
73 // Merges available neighbouring block regions into the given region
74 BlockRegion *coalesce_block_regions(void *user_context, BlockRegion *region);
75
76 // Returns true if the given region can be split to accomodate the given size
77 bool can_split(const BlockRegion *region, const MemoryRequest &request) const;
78
79 // Splits the given block region into a smaller region to accomodate the given size, followed by empty space for the remaining
80 BlockRegion *split_block_region(void *user_context, BlockRegion *region, const MemoryRequest &request);
81
82 // Creates a new block region and adds it to the region list
83 BlockRegion *create_block_region(void *user_context, const MemoryRequest &request);
84
85 // Creates a new block region and adds it to the region list
86 int destroy_block_region(void *user_context, BlockRegion *region);
87
88 // Invokes the allocation callback to allocate memory for the block region
89 int alloc_block_region(void *user_context, BlockRegion *region);
90
91 // Releases a block region and leaves it in the list for further allocations
92 int release_block_region(void *user_context, BlockRegion *region);
93
94 // Invokes the deallocation callback to free memory for the block region
95 int free_block_region(void *user_context, BlockRegion *region);
96
97 // Returns true if the given block region is the last region in the list
98 bool is_last_block_region(void *user_context, const BlockRegion *region) const;
99
100 // Returns true if the given block region is compatible with the given properties
101 bool is_compatible_block_region(const BlockRegion *region, const MemoryProperties &properties) const;
102
103 // Returns true if the given block region is suitable for the requested allocation
104 bool is_block_region_suitable_for_request(void *user_context, const BlockRegion *region, const MemoryRequest &request) const;
105
106 // Returns the number of active regions for the block;
107 size_t region_count(void *user_context) const;
108
109 BlockResource *block = nullptr;
110 MemoryArena *arena = nullptr;
111 MemoryAllocators allocators;
112};
113
114 RegionAllocator *RegionAllocator::create(void *user_context, BlockResource *block_resource, const MemoryAllocators &allocators) {
115 halide_abort_if_false(user_context, allocators.system.allocate != nullptr);
116 RegionAllocator *result = reinterpret_cast<RegionAllocator *>(
117 allocators.system.allocate(user_context, sizeof(RegionAllocator)));
118
119 if (result == nullptr) {
120 return nullptr;
121 }
122
123 result->initialize(user_context, block_resource, allocators);
124 return result;
125}
126
127 int RegionAllocator::destroy(void *user_context, RegionAllocator *instance) {
128 halide_abort_if_false(user_context, instance != nullptr);
129 MemoryAllocators allocators = instance->allocators;
130 instance->destroy(user_context);
131 halide_abort_if_false(user_context, allocators.system.deallocate != nullptr);
132 allocators.system.deallocate(user_context, instance);
133 return 0;
134}
135
136 int RegionAllocator::initialize(void *user_context, BlockResource *mb, const MemoryAllocators &ma) {
137 block = mb;
138 allocators = ma;
139 arena = MemoryArena::create(user_context, {sizeof(BlockRegion), MemoryArena::default_capacity, 0}, allocators.system);
140 halide_abort_if_false(user_context, arena != nullptr);
141 MemoryRequest block_request = {};
142 block_request.size = block->memory.size;
143 block_request.offset = 0;
144 block_request.alignment = block->memory.properties.alignment;
145 block_request.properties = block->memory.properties;
146 block_request.dedicated = block->memory.dedicated;
147 block->allocator = this;
148 block->regions = create_block_region(user_context, block_request);
149 return 0;
150}
151
152 int RegionAllocator::conform(void *user_context, MemoryRequest *request) const {
153 if (allocators.region.conform) {
154 return allocators.region.conform(user_context, request);
155 } else {
156 size_t actual_alignment = conform_alignment(request->alignment, block->memory.properties.alignment);
157 size_t actual_offset = aligned_offset(request->offset, actual_alignment);
158 size_t actual_size = conform_size(actual_offset, request->size, actual_alignment, block->memory.properties.nearest_multiple);
159 request->alignment = actual_alignment;
160 request->offset = actual_offset;
161 request->size = actual_size;
162 }
163 return 0;
164}
165
166 MemoryRegion *RegionAllocator::reserve(void *user_context, const MemoryRequest &request) {
167 halide_abort_if_false(user_context, request.size > 0);
168
169 MemoryRequest region_request = request;
170
171 int error_code = conform(user_context, &region_request);
172 if (error_code) {
173#ifdef DEBUG_RUNTIME_INTERNAL
174 debug(user_context) << "RegionAllocator: Failed to conform region request! Unable to reserve memory ...\n";
175#endif
176 return nullptr;
177 }
178
179 size_t remaining = block->memory.size - block->reserved;
180 if (remaining < region_request.size) {
181#ifdef DEBUG_RUNTIME_INTERNAL
182 debug(user_context) << "RegionAllocator: Unable to reserve more memory from block "
183 << "-- requested size (" << (int32_t)(region_request.size) << " bytes) "
184 << "greater than available (" << (int32_t)(remaining) << " bytes)";
185#endif
186 return nullptr;
187 }
188
189 BlockRegion *block_region = find_block_region(user_context, region_request);
190 if (block_region == nullptr) {
191#ifdef DEBUG_RUNTIME_INTERNAL
192 debug(user_context) << "RegionAllocator: Failed to locate region for requested size ("
193 << (int32_t)(request.size) << " bytes)";
194#endif
195 return nullptr;
196 }
197
198 if (can_split(block_region, region_request)) {
199#ifdef DEBUG_RUNTIME_INTERNAL
200 debug(user_context) << "RegionAllocator: Splitting region of size ( " << (int32_t)(block_region->memory.size) << ") "
201 << "to accomodate requested size (" << (int32_t)(region_request.size) << " bytes)";
202#endif
203 split_block_region(user_context, block_region, region_request);
204 }
205
206 alloc_block_region(user_context, block_region);
207 return reinterpret_cast<MemoryRegion *>(block_region);
208}
209
210 int RegionAllocator::release(void *user_context, MemoryRegion *memory_region) {
211 BlockRegion *block_region = reinterpret_cast<BlockRegion *>(memory_region);
212 halide_abort_if_false(user_context, block_region != nullptr);
213 halide_abort_if_false(user_context, block_region->block_ptr == block);
214 if (block_region->usage_count > 0) {
215 block_region->usage_count--;
216 }
217 return release_block_region(user_context, block_region);
218}
219
220 int RegionAllocator::reclaim(void *user_context, MemoryRegion *memory_region) {
221 BlockRegion *block_region = reinterpret_cast<BlockRegion *>(memory_region);
222 halide_abort_if_false(user_context, block_region != nullptr);
223 halide_abort_if_false(user_context, block_region->block_ptr == block);
224 if (block_region->usage_count > 0) {
225 block_region->usage_count--;
226 }
227 release_block_region(user_context, block_region);
228 free_block_region(user_context, block_region);
229 return 0;
230}
231
232 int RegionAllocator::retain(void *user_context, MemoryRegion *memory_region) {
233 BlockRegion *block_region = reinterpret_cast<BlockRegion *>(memory_region);
234 halide_abort_if_false(user_context, block_region != nullptr);
235 halide_abort_if_false(user_context, block_region->block_ptr == block);
236 block_region->usage_count++;
237 return 0;
238}
239
240 RegionAllocator *RegionAllocator::find_allocator(void *user_context, MemoryRegion *memory_region) {
241 BlockRegion *block_region = reinterpret_cast<BlockRegion *>(memory_region);
242 if (block_region == nullptr) {
243 return nullptr;
244 }
245 if (block_region->block_ptr == nullptr) {
246 return nullptr;
247 }
248 return block_region->block_ptr->allocator;
249}
250
251 bool RegionAllocator::is_last_block_region(void *user_context, const BlockRegion *region) const {
252 return ((region == nullptr) || (region == region->next_ptr) || (region->next_ptr == nullptr));
253}
254
255 bool RegionAllocator::is_block_region_suitable_for_request(void *user_context, const BlockRegion *region, const MemoryRequest &request) const {
256 if (!is_available(region)) {
257#ifdef DEBUG_RUNTIME_INTERNAL
258 debug(user_context) << " skipping block region ... not available! ("
259 << " block_region=" << (void *)region
260 << " region_size=" << (uint32_t)(region->memory.size)
261 << ")";
262#endif
263 return false;
264 }
265
266 MemoryRequest region_request = request;
267 int error_code = conform(user_context, &region_request);
268 if (error_code) {
269#ifdef DEBUG_RUNTIME_INTERNAL
270 debug(user_context) << "RegionAllocator: Failed to conform region request! Unable to reserve memory ...\n";
271#endif
272 return false;
273 }
274
275 // skip incompatible block regions for this request
276 if (!is_compatible_block_region(region, region_request.properties)) {
277#ifdef DEBUG_RUNTIME_INTERNAL
278 debug(user_context) << " skipping block region ... incompatible properties! ("
279 << " block_region=" << (void *)region
280 << " region_size=" << (uint32_t)(region->memory.size)
281 << ")";
282#endif
283 return false;
284 }
285
286 // is the adjusted size larger than the current region?
287 if (region_request.size > region->memory.size) {
288#ifdef DEBUG_RUNTIME_INTERNAL
289 debug(user_context) << " skipping block region ... not enough space for adjusted size! ("
290 << " block_region=" << (void *)region
291 << " request_size=" << (uint32_t)(request.size)
292 << " actual_size=" << (uint32_t)(region_request.size)
293 << " region_size=" << (uint32_t)(region->memory.size)
294 << ")";
295#endif
296 return false;
297 }
298
299 // will the adjusted size fit within the remaining unallocated space?
300 if ((region_request.size + block->reserved) <= block->memory.size) {
301#ifdef DEBUG_RUNTIME_INTERNAL
302 debug(user_context) << " found suitable block region! ("
303 << " block_region=" << (void *)region
304 << " request_size=" << (uint32_t)(request.size)
305 << " actual_size=" << (uint32_t)(region_request.size)
306 << " region_size=" << (uint32_t)(region->memory.size)
307 << ")";
308#endif
309 return true; // you betcha
310 }
311
312 return false;
313}
314
315BlockRegion *RegionAllocator::find_block_region(void *user_context, const MemoryRequest &request) {
316#ifdef DEBUG_RUNTIME_INTERNAL
317 debug(user_context) << "RegionAllocator: find block region ( "
318 << "user_context=" << (void *)(user_context) << " "
319 << "requested_size=" << (uint32_t)request.size << " "
320 << "requested_is_dedicated=" << (request.dedicated ? "true" : "false") << " "
321 << "requested_usage=" << halide_memory_usage_name(request.properties.usage) << " "
322 << "requested_caching=" << halide_memory_caching_name(request.properties.caching) << " "
323 << "requested_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
324#endif
325 BlockRegion *block_region = block->regions;
326 while (block_region != nullptr) {
327 if (is_block_region_suitable_for_request(user_context, block_region, request)) {
328#ifdef DEBUG_RUNTIME_INTERNAL
329 debug(user_context) << "RegionAllocator: found suitable region ( "
330 << "user_context=" << (void *)(user_context) << " "
331 << "block_resource=" << (void *)block << " "
332 << "block_size=" << (uint32_t)block->memory.size << " "
333 << "block_reserved=" << (uint32_t)block->reserved << " "
334 << "requested_size=" << (uint32_t)request.size << " "
335 << "requested_is_dedicated=" << (request.dedicated ? "true" : "false") << " "
336 << "requested_usage=" << halide_memory_usage_name(request.properties.usage) << " "
337 << "requested_caching=" << halide_memory_caching_name(request.properties.caching) << " "
338 << "requested_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
339#endif
340 return block_region;
341 }
342
343 if (is_last_block_region(user_context, block_region)) {
344 block_region = nullptr; // end of list ... nothing found
345 break;
346 }
347 block_region = block_region->next_ptr;
348 }
349
350 if (block_region == nullptr) {
351#ifdef DEBUG_RUNTIME_INTERNAL
352 debug(user_context) << "RegionAllocator: couldn't find suitable region! ("
353 << "user_context=" << (void *)(user_context) << " "
354 << "requested_size=" << (uint32_t)request.size << " "
355 << "requested_is_dedicated=" << (request.dedicated ? "true" : "false") << " "
356 << "requested_usage=" << halide_memory_usage_name(request.properties.usage) << " "
357 << "requested_caching=" << halide_memory_caching_name(request.properties.caching) << " "
358 << "requested_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
359#endif
360 }
361
362 return block_region;
363}
364
365 bool RegionAllocator::is_available(const BlockRegion *block_region) const {
366 if (block_region == nullptr) {
367 return false;
368 }
369 if (block_region->usage_count > 0) {
370 return false;
371 }
372 if (block_region->status != AllocationStatus::Available) {
373 return false;
374 }
375 return true;
376}
377
378 bool RegionAllocator::can_coalesce(const BlockRegion *block_region) const {
379 if (!is_available(block_region)) {
380 return false;
381 }
382 if (is_available(block_region->prev_ptr)) {
383 return true;
384 }
385 if (is_available(block_region->next_ptr)) {
386 return true;
387 }
388 return false;
389}
390
391BlockRegion *RegionAllocator::coalesce_block_regions(void *user_context, BlockRegion *block_region) {
392
393 if ((block_region->usage_count == 0) && (block_region->memory.handle != nullptr)) {
394#ifdef DEBUG_RUNTIME_INTERNAL
395 debug(user_context) << "RegionAllocator: Freeing unused region to coalesce ("
396 << "block_ptr=" << (void *)block_region->block_ptr << " "
397 << "block_region=" << (void *)block_region << " "
398 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
399 << "block_reserved=" << (uint32_t)block->reserved << " "
400 << ")";
401#endif
402 halide_abort_if_false(user_context, allocators.region.deallocate != nullptr);
403 MemoryRegion *memory_region = &(block_region->memory);
404 allocators.region.deallocate(user_context, memory_region);
405 block_region->memory.handle = nullptr;
406 }
407
408 BlockRegion *prev_region = block_region->prev_ptr;
409 if (is_available(prev_region) && (prev_region != block_region)) {
410
411#ifdef DEBUG_RUNTIME_INTERNAL
412 debug(user_context) << "RegionAllocator: Coalescing "
413 << "previous region (offset=" << (int32_t)prev_region->memory.offset << " size=" << (int32_t)(prev_region->memory.size) << " bytes) "
414 << "into current region (offset=" << (int32_t)block_region->memory.offset << " size=" << (int32_t)(block_region->memory.size) << " bytes)!";
415#endif
416
417 prev_region->next_ptr = block_region->next_ptr;
418 if (block_region->next_ptr) {
419 block_region->next_ptr->prev_ptr = prev_region;
420 }
421 prev_region->memory.size += block_region->memory.size;
422 destroy_block_region(user_context, block_region);
423 block_region = prev_region;
424 }
425
426 BlockRegion *next_region = block_region->next_ptr;
427 if (is_available(next_region) && (next_region != block_region)) {
428
429#ifdef DEBUG_RUNTIME_INTERNAL
430 debug(user_context) << "RegionAllocator: Coalescing "
431 << "next region (offset=" << (int32_t)next_region->memory.offset << " size=" << (int32_t)(next_region->memory.size) << " bytes) "
432 << "into current region (offset=" << (int32_t)block_region->memory.offset << " size=" << (int32_t)(block_region->memory.size) << " bytes)";
433#endif
434
435 if (next_region->next_ptr) {
436 next_region->next_ptr->prev_ptr = block_region;
437 }
438 block_region->next_ptr = next_region->next_ptr;
439 block_region->memory.size += next_region->memory.size;
440 destroy_block_region(user_context, next_region);
441 }
442
443 return block_region;
444}
445
446 bool RegionAllocator::can_split(const BlockRegion *block_region, const MemoryRequest &split_request) const {
447 return (block_region && (block_region->memory.size > split_request.size) && (block_region->usage_count == 0));
448}
449
450BlockRegion *RegionAllocator::split_block_region(void *user_context, BlockRegion *block_region, const MemoryRequest &request) {
451
452 if ((block_region->usage_count == 0) && (block_region->memory.handle != nullptr)) {
453#ifdef DEBUG_RUNTIME_INTERNAL
454 debug(user_context) << "RegionAllocator: Split deallocate region ("
455 << "block_ptr=" << (void *)block_region->block_ptr << " "
456 << "block_region=" << (void *)block_region << " "
457 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
458 << "block_reserved=" << (uint32_t)block_region->block_ptr->reserved << " "
459 << ")";
460#endif
461 halide_abort_if_false(user_context, allocators.region.deallocate != nullptr);
462 MemoryRegion *memory_region = &(block_region->memory);
463 allocators.region.deallocate(user_context, memory_region);
464 block_region->memory.handle = nullptr;
465 }
466
467 MemoryRequest split_request = request;
468 split_request.size = block_region->memory.size - request.size;
469 split_request.offset = block_region->memory.offset + request.size;
470
471#ifdef DEBUG_RUNTIME_INTERNAL
472 debug(user_context) << "RegionAllocator: Splitting "
473 << "current region (offset=" << (int32_t)block_region->memory.offset << " size=" << (int32_t)(block_region->memory.size) << " bytes) "
474 << "to create empty region (offset=" << (int32_t)split_request.offset << " size=" << (int32_t)(split_request.size) << " bytes)";
475#endif
476 BlockRegion *next_region = block_region->next_ptr;
477 BlockRegion *empty_region = create_block_region(user_context, split_request);
478 halide_abort_if_false(user_context, empty_region != nullptr);
479
480 empty_region->next_ptr = next_region;
481 if (next_region) {
482 next_region->prev_ptr = empty_region;
483 }
484 empty_region->prev_ptr = block_region;
485 block_region->next_ptr = empty_region;
486 block_region->memory.size -= empty_region->memory.size;
487 return empty_region;
488}
489
490BlockRegion *RegionAllocator::create_block_region(void *user_context, const MemoryRequest &request) {
491#ifdef DEBUG_RUNTIME_INTERNAL
492 debug(user_context) << "RegionAllocator: Creating block region request ("
493 << "user_context=" << (void *)(user_context) << " "
494 << "offset=" << (uint32_t)request.offset << " "
495 << "size=" << (uint32_t)request.size << " "
496 << "alignment=" << (uint32_t)request.properties.alignment << " "
497 << "dedicated=" << (request.dedicated ? "true" : "false") << " "
498 << "usage=" << halide_memory_usage_name(request.properties.usage) << " "
499 << "caching=" << halide_memory_caching_name(request.properties.caching) << " "
500 << "visibility=" << halide_memory_visibility_name(request.properties.visibility) << ") ...";
501#endif
502
503 MemoryRequest region_request = request;
504 int error_code = conform(user_context, &region_request);
505 if (error_code) {
506#ifdef DEBUG_RUNTIME_INTERNAL
507 debug(user_context) << "RegionAllocator: Failed to conform request for new block region!\n";
508#endif
509 return nullptr;
510 }
511
512 if (region_request.size == 0) {
513#ifdef DEBUG_RUNTIME_INTERNAL
514 debug(user_context) << "RegionAllocator: Failed to allocate new block region ... region size was zero!\n";
515#endif
516 return nullptr;
517 }
518
519 BlockRegion *block_region = static_cast<BlockRegion *>(arena->reserve(user_context, true));
520 if (block_region == nullptr) {
521#ifdef DEBUG_RUNTIME_INTERNAL
522 debug(user_context) << "RegionAllocator: Failed to allocate new block region!\n";
523#endif
524 return nullptr;
525 }
526
527 block_region->memory.handle = nullptr;
528 block_region->memory.offset = region_request.offset;
529 block_region->memory.size = region_request.size;
530 block_region->memory.properties = region_request.properties;
531 block_region->memory.dedicated = region_request.dedicated;
532 block_region->status = AllocationStatus::Available;
533 block_region->block_ptr = block;
534 block_region->usage_count = 0;
535
536#ifdef DEBUG_RUNTIME_INTERNAL
537 debug(user_context) << "RegionAllocator: Created block region allocation ("
538 << "user_context=" << (void *)(user_context) << " "
539 << "block_ptr=" << (void *)block_region->block_ptr << " "
540 << "block_region=" << (void *)block_region << " "
541 << "memory_offset=" << (uint32_t)(block_region->memory.offset) << " "
542 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
543 << ")";
544#endif
545
546 return block_region;
547}
548
549 int RegionAllocator::release_block_region(void *user_context, BlockRegion *block_region) {
550#ifdef DEBUG_RUNTIME_INTERNAL
551 debug(user_context) << "RegionAllocator: Releasing block region ("
552 << "user_context=" << (void *)(user_context) << " "
553 << "block_ptr=" << ((block_region) ? ((void *)block_region->block_ptr) : nullptr) << " "
554 << "block_region=" << (void *)block_region << " "
555 << "usage_count=" << ((block_region) ? (uint32_t)(block_region->usage_count) : 0) << " "
556 << "memory_offset=" << ((block_region) ? (uint32_t)(block_region->memory.offset) : 0) << " "
557 << "memory_size=" << ((block_region) ? (uint32_t)(block_region->memory.size) : 0) << " "
558 << "block_reserved=" << (uint32_t)(block->reserved) << ") ... ";
559#endif
560 if (block_region == nullptr) {
561 return 0;
562 }
563
564 if (block_region->usage_count > 0) {
565 return 0;
566 }
567
568 if (block_region->status != AllocationStatus::Available) {
569
570#ifdef DEBUG_RUNTIME_INTERNAL
571 debug(user_context) << " releasing region ("
572 << "block_ptr=" << (void *)block_region->block_ptr << " "
573 << "block_region=" << (void *)block_region << " "
574 << "memory_offset=" << (uint32_t)(block_region->memory.offset) << " "
575 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
576 << "block_reserved=" << (uint32_t)(block->reserved - block_region->memory.size) << " "
577 << ")";
578#endif
579
580 block->reserved -= block_region->memory.size;
581 }
582 block_region->status = AllocationStatus::Available;
583 return 0;
584}
585
586 int RegionAllocator::destroy_block_region(void *user_context, BlockRegion *block_region) {
587#ifdef DEBUG_RUNTIME_INTERNAL
588 debug(user_context) << "RegionAllocator: Destroying block region ("
589 << "user_context=" << (void *)(user_context) << " "
590 << "block_region=" << (void *)(block_region) << ") ...";
591#endif
592
593 block_region->usage_count = 0;
594 release_block_region(user_context, block_region);
595 free_block_region(user_context, block_region);
596 arena->reclaim(user_context, block_region);
597 return 0;
598}
599
600 int RegionAllocator::alloc_block_region(void *user_context, BlockRegion *block_region) {
601#ifdef DEBUG_RUNTIME_INTERNAL
602 debug(user_context) << "RegionAllocator: Allocating region (user_context=" << (void *)(user_context)
603 << " size=" << (int32_t)(block_region->memory.size)
604 << " offset=" << (int32_t)block_region->memory.offset << ")";
605#endif
606 halide_abort_if_false(user_context, allocators.region.allocate != nullptr);
607 halide_abort_if_false(user_context, block_region->status == AllocationStatus::Available);
608 int error_code = 0;
609 MemoryRegion *memory_region = &(block_region->memory);
610 if (memory_region->handle == nullptr) {
611 error_code = allocators.region.allocate(user_context, memory_region);
612 memory_region->is_owner = true;
613
614#ifdef DEBUG_RUNTIME_INTERNAL
615 debug(user_context) << " allocating region ("
616 << "block_ptr=" << (void *)block_region->block_ptr << " "
617 << "block_region=" << (void *)block_region << " "
618 << "memory_offset=" << (uint32_t)(block_region->memory.offset) << " "
619 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
620 << "block_reserved=" << (uint32_t)block->reserved << " "
621 << ")";
622#endif
623
624 } else {
625
626#ifdef DEBUG_RUNTIME_INTERNAL
627 debug(user_context) << " re-using region ("
628 << "block_ptr=" << (void *)block_region->block_ptr << " "
629 << "block_region=" << (void *)block_region << " "
630 << "memory_offset=" << (uint32_t)(block_region->memory.offset) << " "
631 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
632 << "block_reserved=" << (uint32_t)block->reserved << " "
633 << ")";
634#endif
635 }
636 if (error_code == 0) {
637 block_region->status = block_region->memory.dedicated ? AllocationStatus::Dedicated : AllocationStatus::InUse;
638 block->reserved += block_region->memory.size;
639 }
640 return error_code;
641}
642
643 int RegionAllocator::free_block_region(void *user_context, BlockRegion *block_region) {
644#ifdef DEBUG_RUNTIME_INTERNAL
645 debug(user_context) << "RegionAllocator: Freeing block region ("
646 << "user_context=" << (void *)(user_context) << " "
647 << "block_ptr=" << (void *)block_region->block_ptr << " "
648 << "block_region=" << (void *)(block_region) << " "
649 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
650 << "status=" << (uint32_t)block_region->status << " "
651 << "usage_count=" << (uint32_t)block_region->usage_count << " "
652 << "block_reserved=" << (uint32_t)block->reserved << ")";
653#endif
654 int error_code = 0;
655 if ((block_region->usage_count == 0) && (block_region->memory.handle != nullptr)) {
656#ifdef DEBUG_RUNTIME_INTERNAL
657 debug(user_context) << " deallocating region ("
658 << "block_ptr=" << (void *)block_region->block_ptr << " "
659 << "block_region=" << (void *)block_region << " "
660 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
661 << "block_reserved=" << (uint32_t)block->reserved << " "
662 << ")";
663#endif
664 // NOTE: Deallocate but leave memory size as is, so that coalesce can compute region merging sizes
665 halide_abort_if_false(user_context, allocators.region.deallocate != nullptr);
666 MemoryRegion *memory_region = &(block_region->memory);
667 error_code = allocators.region.deallocate(user_context, memory_region);
668 block_region->memory.handle = nullptr;
669 }
670 block_region->usage_count = 0;
671 block_region->status = AllocationStatus::Available;
672 return error_code;
673}
674
675 int RegionAllocator::release(void *user_context) {
676#ifdef DEBUG_RUNTIME_INTERNAL
677 debug(user_context) << "RegionAllocator: Releasing all regions ("
678 << "user_context=" << (void *)(user_context) << ") ...";
679#endif
680
681 BlockRegion *block_region = block->regions;
682 while (block_region != nullptr) {
683 release_block_region(user_context, block_region);
684 if (is_last_block_region(user_context, block_region)) {
685 break;
686 }
687 block_region = block_region->next_ptr;
688 }
689 return 0;
690}
691
692 bool RegionAllocator::collect(void *user_context) {
693#ifdef DEBUG_RUNTIME_INTERNAL
694 debug(user_context) << "RegionAllocator: Collecting free block regions ("
695 << "user_context=" << (void *)(user_context) << ") ...";
696
697 uint32_t collected_count = 0;
698 uint32_t remaining_count = 0;
699 uint64_t available_bytes = 0;
700 uint64_t scanned_bytes = 0;
701 uint64_t reserved = block->reserved;
702 debug(user_context) << " collecting unused regions ("
703 << "block_ptr=" << (void *)block << " "
704 << "block_reserved=" << (uint32_t)block->reserved << " "
705 << ")";
706#endif
707
708 bool has_collected = false;
709 BlockRegion *block_region = block->regions;
710 while (block_region != nullptr) {
711#ifdef DEBUG_RUNTIME_INTERNAL
712 scanned_bytes += block_region->memory.size;
713 debug(user_context) << " checking region ("
714 << "block_ptr=" << (void *)block_region->block_ptr << " "
715 << "block_region=" << (void *)block_region << " "
716 << "usage_count=" << (uint32_t)(block_region->usage_count) << " "
717 << "status=" << (uint32_t)(block_region->status) << " "
718 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
719 << "block_reserved=" << (uint32_t)block->reserved << " "
720 << ")";
721#endif
722
723 if (can_coalesce(block_region)) {
724#ifdef DEBUG_RUNTIME_INTERNAL
725 collected_count++;
726 debug(user_context) << " collecting region ("
727 << "block_ptr=" << (void *)block_region->block_ptr << " "
728 << "block_region=" << (void *)block_region << " "
729 << "memory_size=" << (uint32_t)(block_region->memory.size) << " "
730 << "block_reserved=" << (uint32_t)block->reserved << " "
731 << ")";
732#endif
733 block_region = coalesce_block_regions(user_context, block_region);
734 has_collected = true;
735 } else {
736#ifdef DEBUG_RUNTIME_INTERNAL
737 remaining_count++;
738#endif
739 }
740#ifdef DEBUG_RUNTIME_INTERNAL
741 available_bytes += is_available(block_region) ? block_region->memory.size : 0;
742#endif
743 if (is_last_block_region(user_context, block_region)) {
744 break;
745 }
746 block_region = block_region->next_ptr;
747 }
748#ifdef DEBUG_RUNTIME_INTERNAL
749 debug(user_context) << " scanned active regions ("
750 << "block_ptr=" << (void *)block << " "
751 << "total_count=" << (uint32_t)(collected_count + remaining_count) << " "
752 << "block_reserved=" << (uint32_t)(block->reserved) << " "
753 << "scanned_bytes=" << (uint32_t)(scanned_bytes) << " "
754 << "available_bytes=" << (uint32_t)(available_bytes) << " "
755 << ")";
756#endif
757
758 if (has_collected) {
759#ifdef DEBUG_RUNTIME_INTERNAL
760 debug(user_context) << " collected unused regions ("
761 << "block_ptr=" << (void *)block << " "
762 << "collected_count=" << (uint32_t)collected_count << " "
763 << "remaining_count=" << (uint32_t)remaining_count << " "
764 << "reclaimed=" << (uint32_t)(reserved - block->reserved) << " "
765 << ")";
766#endif
767 }
768 return has_collected;
769}
770
771 int RegionAllocator::destroy(void *user_context) {
772#ifdef DEBUG_RUNTIME_INTERNAL
773 debug(user_context) << "RegionAllocator: Destroying all block regions ("
774 << "user_context=" << (void *)(user_context) << ") ...";
775#endif
776 if (block->regions != nullptr) {
777 for (BlockRegion *block_region = block->regions; block_region != nullptr;) {
778
779 if (is_last_block_region(user_context, block_region)) {
780 destroy_block_region(user_context, block_region);
781 block_region = nullptr;
782 } else {
783 BlockRegion *prev_region = block_region;
784 block_region = block_region->next_ptr;
785 destroy_block_region(user_context, prev_region);
786 }
787 }
788 }
789 block->reserved = 0;
790 block->regions = nullptr;
791 block->allocator = nullptr;
792 if (arena != nullptr) {
793 MemoryArena::destroy(user_context, arena);
794 }
795 arena = nullptr;
796 return 0;
797}
798
799 bool RegionAllocator::is_compatible_block_region(const BlockRegion *block_region, const MemoryProperties &properties) const {
800 if (properties.caching != MemoryCaching::DefaultCaching) {
801 if (properties.caching != block_region->memory.properties.caching) {
802 return false;
803 }
804 }
805
806 if (properties.visibility != MemoryVisibility::DefaultVisibility) {
807 if (properties.visibility != block_region->memory.properties.visibility) {
808 return false;
809 }
810 }
811
812 if (properties.usage != MemoryUsage::DefaultUsage) {
813 if (properties.usage != block_region->memory.properties.usage) {
814 return false;
815 }
816 }
817
818 return true;
819}
820
821 size_t RegionAllocator::region_count(void *user_context) const {
822 if (block == nullptr) {
823 return 0;
824 }
825 size_t count = 0;
826 for (BlockRegion const *region = block->regions; !is_last_block_region(user_context, region); region = region->next_ptr) {
827 ++count;
828 }
829 return count;
830}
831
832 BlockResource *RegionAllocator::block_resource() const {
833 return block;
834}
835
836// --
837
838} // namespace Internal
839} // namespace Runtime
840} // namespace Halide
841
842#endif // HALIDE_RUNTIME_REGION_ALLOCATOR_H
debug
#define debug(n)
For optional debugging during codegen, use the debug macro as follows:
Definition: Debug.h:52
Halide::Runtime::Internal::MemoryArena
Definition: memory_arena.h:17
Halide::Runtime::Internal::MemoryArena::reserve
void * reserve(void *user_context, bool initialize=false)
Definition: memory_arena.h:155
Halide::Runtime::Internal::MemoryArena::create
static MemoryArena * create(void *user_context, const Config &config, const SystemMemoryAllocatorFns &allocator=default_allocator())
Definition: memory_arena.h:101
Halide::Runtime::Internal::MemoryArena::default_capacity
static constexpr uint32_t default_capacity
Definition: memory_arena.h:24
Halide::Runtime::Internal::MemoryArena::destroy
static void destroy(void *user_context, MemoryArena *arena)
Definition: memory_arena.h:115
Halide::Runtime::Internal::MemoryArena::reclaim
void reclaim(void *user_context, void *ptr)
Definition: memory_arena.h:182
Halide::Runtime::Internal::RegionAllocator
Allocator class interface for sub-allocating a contiguous memory block into smaller regions of memory...
Definition: region_allocator.h:24
Halide::Runtime::Internal::RegionAllocator::reserve
MemoryRegion * reserve(void *user_context, const MemoryRequest &request)
Definition: region_allocator.h:166
Halide::Runtime::Internal::RegionAllocator::operator=
RegionAllocator & operator=(const RegionAllocator &)=delete
Halide::Runtime::Internal::RegionAllocator::RegionAllocator
RegionAllocator(const RegionAllocator &)=delete
Halide::Runtime::Internal::RegionAllocator::destroy
static int destroy(void *user_context, RegionAllocator *region_allocator)
Definition: region_allocator.h:127
Halide::Runtime::Internal::RegionAllocator::conform
int conform(void *user_context, MemoryRequest *request) const
Definition: region_allocator.h:152
Halide::Runtime::Internal::RegionAllocator::RegionAllocator
RegionAllocator()=delete
Halide::Runtime::Internal::RegionAllocator::find_allocator
static RegionAllocator * find_allocator(void *user_context, MemoryRegion *memory_region)
Definition: region_allocator.h:240
Halide::Runtime::Internal::RegionAllocator::retain
int retain(void *user_context, MemoryRegion *memory_region)
Definition: region_allocator.h:232
Halide::Runtime::Internal::RegionAllocator::collect
bool collect(void *user_context)
Definition: region_allocator.h:692
Halide::Runtime::Internal::RegionAllocator::~RegionAllocator
~RegionAllocator()=delete
Halide::Runtime::Internal::RegionAllocator::reclaim
int reclaim(void *user_context, MemoryRegion *memory_region)
Definition: region_allocator.h:220
Halide::Runtime::Internal::RegionAllocator::release
int release(void *user_context, MemoryRegion *memory_region)
Definition: region_allocator.h:210
Halide::Runtime::Internal::RegionAllocator::block_resource
BlockResource * block_resource() const
Definition: region_allocator.h:832
Halide::Runtime::Internal::RegionAllocator::create
static RegionAllocator * create(void *user_context, BlockResource *block, const MemoryAllocators &ma)
Definition: region_allocator.h:114
memory_arena.h
memory_resources.h
halide_memory_caching_name
WEAK const char * halide_memory_caching_name(MemoryCaching value)
Definition: memory_resources.h:293
halide_memory_usage_name
WEAK const char * halide_memory_usage_name(MemoryUsage value)
Definition: memory_resources.h:260
halide_memory_visibility_name
WEAK const char * halide_memory_visibility_name(MemoryVisibility value)
Definition: memory_resources.h:233
Halide::Runtime::Internal::conform_alignment
ALWAYS_INLINE size_t conform_alignment(size_t requested, size_t required)
Definition: memory_resources.h:135
Halide::Runtime::Internal::MemoryUsage::DefaultUsage
@ DefaultUsage
Halide::Runtime::Internal::AllocationStatus::Available
@ Available
Halide::Runtime::Internal::AllocationStatus::Dedicated
@ Dedicated
Halide::Runtime::Internal::AllocationStatus::InUse
@ InUse
Halide::Runtime::Internal::conform_size
ALWAYS_INLINE size_t conform_size(size_t offset, size_t size, size_t alignment, size_t nearest_multiple)
Definition: memory_resources.h:151
Halide::Runtime::Internal::MemoryVisibility::DefaultVisibility
@ DefaultVisibility
Halide::Runtime::Internal::MemoryCaching::DefaultCaching
@ DefaultCaching
Halide::Runtime::Internal::aligned_offset
ALWAYS_INLINE size_t aligned_offset(size_t offset, size_t alignment)
Definition: memory_resources.h:128
Halide
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
Definition: AbstractGenerator.h:19
Halide::LinkageType::Internal
@ Internal
Not visible externally, similar to 'static' linkage in C.
uint64_t
unsigned __INT64_TYPE__ uint64_t
Definition: runtime_internal.h:23
int32_t
signed __INT32_TYPE__ int32_t
Definition: runtime_internal.h:24
uint32_t
unsigned __INT32_TYPE__ uint32_t
Definition: runtime_internal.h:25
halide_abort_if_false
#define halide_abort_if_false(user_context, cond)
Definition: runtime_internal.h:271
Halide::Runtime::Internal::BlockRegion
Definition: memory_resources.h:112
Halide::Runtime::Internal::BlockRegion::next_ptr
BlockRegion * next_ptr
Definition: memory_resources.h:116
Halide::Runtime::Internal::BlockRegion::usage_count
uint32_t usage_count
Definition: memory_resources.h:114
Halide::Runtime::Internal::BlockRegion::status
AllocationStatus status
Definition: memory_resources.h:115
Halide::Runtime::Internal::BlockRegion::memory
MemoryRegion memory
Definition: memory_resources.h:113
Halide::Runtime::Internal::BlockRegion::block_ptr
BlockResource * block_ptr
Definition: memory_resources.h:118
Halide::Runtime::Internal::BlockResource
Definition: memory_resources.h:103
Halide::Runtime::Internal::BlockResource::reserved
size_t reserved
Definition: memory_resources.h:107
Halide::Runtime::Internal::BlockResource::allocator
RegionAllocator * allocator
Definition: memory_resources.h:105
Halide::Runtime::Internal::BlockResource::memory
MemoryBlock memory
Definition: memory_resources.h:104
Halide::Runtime::Internal::BlockResource::regions
BlockRegion * regions
Definition: memory_resources.h:106
Halide::Runtime::Internal::MemoryBlock::dedicated
bool dedicated
Definition: memory_resources.h:68
Halide::Runtime::Internal::MemoryBlock::properties
MemoryProperties properties
Definition: memory_resources.h:69
Halide::Runtime::Internal::MemoryBlock::size
size_t size
Definition: memory_resources.h:67
Halide::Runtime::Internal::MemoryProperties
Definition: memory_resources.h:56
Halide::Runtime::Internal::MemoryProperties::caching
MemoryCaching caching
Definition: memory_resources.h:59
Halide::Runtime::Internal::MemoryProperties::alignment
size_t alignment
Definition: memory_resources.h:60
Halide::Runtime::Internal::MemoryProperties::nearest_multiple
size_t nearest_multiple
Definition: memory_resources.h:61
Halide::Runtime::Internal::MemoryProperties::visibility
MemoryVisibility visibility
Definition: memory_resources.h:57
Halide::Runtime::Internal::MemoryProperties::usage
MemoryUsage usage
Definition: memory_resources.h:58
Halide::Runtime::Internal::MemoryRegionAllocatorFns
Definition: memory_resources.h:217
Halide::Runtime::Internal::MemoryRegionAllocatorFns::deallocate
DeallocateRegionFn deallocate
Definition: memory_resources.h:219
Halide::Runtime::Internal::MemoryRegionAllocatorFns::allocate
AllocateRegionFn allocate
Definition: memory_resources.h:218
Halide::Runtime::Internal::MemoryRegionAllocatorFns::conform
ConformBlockRegionFn conform
Definition: memory_resources.h:220
Halide::Runtime::Internal::MemoryRegion
Definition: memory_resources.h:79
Halide::Runtime::Internal::MemoryRegion::size
size_t size
Definition: memory_resources.h:82
Halide::Runtime::Internal::MemoryRegion::properties
MemoryProperties properties
Definition: memory_resources.h:86
Halide::Runtime::Internal::MemoryRequest
Definition: memory_resources.h:90
Halide::Runtime::Internal::MemoryRequest::size
size_t size
Definition: memory_resources.h:92
Halide::Runtime::Internal::MemoryRequest::offset
size_t offset
Definition: memory_resources.h:91
Halide::Runtime::Internal::MemoryRequest::alignment
size_t alignment
Definition: memory_resources.h:93
Halide::Runtime::Internal::RegionAllocator::MemoryAllocators
Definition: region_allocator.h:35
Halide::Runtime::Internal::RegionAllocator::MemoryAllocators::region
MemoryRegionAllocatorFns region
Definition: region_allocator.h:37
Halide::Runtime::Internal::RegionAllocator::MemoryAllocators::system
SystemMemoryAllocatorFns system
Definition: region_allocator.h:36
Halide::Runtime::Internal::SystemMemoryAllocatorFns
Definition: memory_resources.h:193
Halide::Runtime::Internal::SystemMemoryAllocatorFns::deallocate
DeallocateSystemFn deallocate
Definition: memory_resources.h:195
Halide::Runtime::Internal::SystemMemoryAllocatorFns::allocate
AllocateSystemFn allocate
Definition: memory_resources.h:194
error_code
int error_code
Definition: vulkan_resources.h:182
user_context
void * user_context
Definition: vulkan_resources.h:178