Halide 22.0.0 Halide compiler and libraries

Buffer.h
Go to the documentation of this file.
1#ifndef HALIDE_BUFFER_H
2#define HALIDE_BUFFER_H
3
4#include "DeviceInterface.h"
5#include "Expr.h"
6#include "IntrusivePtr.h"
7#include "runtime/HalideBuffer.h"
8
9 namespace Halide {
10
11 constexpr int AnyDims = Halide::Runtime::AnyDims; // -1
12
13 template<typename T = void, int Dims = AnyDims>
14 class Buffer;
15
16 struct JITUserContext;
17
18 namespace Internal {
19
20 struct BufferContents {
21 mutable RefCount ref_count;
22 std::string name;
23 Runtime::Buffer<> buf;
24};
25
26 Expr buffer_accessor(const Buffer<> &buf, const std::vector<Expr> &args);
27
28 template<typename... Args>
29 struct all_ints_and_optional_name : std::false_type {};
30
31 template<typename First, typename... Rest>
32 struct all_ints_and_optional_name<First, Rest...> : meta_and<std::is_convertible<First, int>,
33 all_ints_and_optional_name<Rest...>> {};
34
35 template<typename T>
36 struct all_ints_and_optional_name<T> : meta_or<std::is_convertible<T, std::string>,
37 std::is_convertible<T, int>> {};
38
39 template<>
40 struct all_ints_and_optional_name<> : std::true_type {};
41
42 template<typename T,
43 typename = std::enable_if_t<!std::is_convertible_v<T, std::string>>>
44std::string get_name_from_end_of_parameter_pack(T &&) {
45 return "";
46}
47
48 inline std::string get_name_from_end_of_parameter_pack(const std::string &n) {
49 return n;
50}
51
52 inline std::string get_name_from_end_of_parameter_pack() {
53 return "";
54}
55
56 template<typename First,
57 typename Second,
58 typename... Args>
59std::string get_name_from_end_of_parameter_pack(First first, Second second, Args &&...rest) {
60 return get_name_from_end_of_parameter_pack(second, std::forward<Args>(rest)...);
61}
62
63 inline void get_shape_from_start_of_parameter_pack_helper(std::vector<int> &, const std::string &) {
64}
65
66 inline void get_shape_from_start_of_parameter_pack_helper(std::vector<int> &) {
67}
68
69 template<typename... Args>
70 void get_shape_from_start_of_parameter_pack_helper(std::vector<int> &result, int x, Args &&...rest) {
71 result.push_back(x);
72 get_shape_from_start_of_parameter_pack_helper(result, std::forward<Args>(rest)...);
73}
74
75 template<typename... Args>
76std::vector<int> get_shape_from_start_of_parameter_pack(Args &&...args) {
77 std::vector<int> result;
78 get_shape_from_start_of_parameter_pack_helper(result, std::forward<Args>(args)...);
79 return result;
80}
81
82 template<typename T, typename T2>
83 using add_const_if_T_is_const = std::conditional_t<std::is_const_v<T>, const T2, T2>;
84
85// Helpers to produce the name of a Buffer element type (a Halide
86// scalar type, or void, possibly with const). Useful for an error
87// messages.
88 template<typename T>
89 void buffer_type_name_non_const(std::ostream &s) {
90 s << type_to_c_type(type_of<T>(), false);
91}
92
93 template<>
94 inline void buffer_type_name_non_const<void>(std::ostream &s) {
95 s << "void";
96}
97
98 template<typename T>
99std::string buffer_type_name() {
100 std::ostringstream oss;
101 if (std::is_const_v<T>) {
102 oss << "const ";
103 }
104 buffer_type_name_non_const<std::remove_const_t<T>>(oss);
105 return oss.str();
106}
107
108} // namespace Internal
109
110/** A Halide::Buffer is a named shared reference to a
111 * Halide::Runtime::Buffer.
112 *
113 * A Buffer<T1, D> can refer to a Buffer<T2, D> if T1 is const whenever T2
114 * is const, and either T1 = T2 or T1 is void. A Buffer<void, D> can
115 * refer to any Buffer of any non-const type, and the default
116 * template parameter is T = void.
117 *
118 * A Buffer<T, D1> can refer to a Buffer<T, D2> if D1 == D2,
119 * or if D1 is AnyDims (meaning "dimensionality is checked at runtime, not compiletime").
120 */
121 template<typename T, int Dims>
122 class Buffer {
123 Internal::IntrusivePtr<Internal::BufferContents> contents;
124
125 template<typename T2, int D2>
126 friend class Buffer;
127
128 template<typename T2, int D2>
129 static void assert_can_convert_from(const Buffer<T2, D2> &other) {
130 if (!other.defined()) {
131 // Avoid UB of deferencing offset of a null contents ptr
132 static_assert(!std::is_const_v<T2> || std::is_const_v<T>,
133 "Can't convert from a Buffer<const T> to a Buffer<T>");
134 static_assert(std::is_same_v<std::remove_const_t<T>, std::remove_const_t<T2>> ||
135 std::is_void_v<T> ||
136 std::is_void_v<T2>,
137 "type mismatch constructing Buffer");
138 static_assert(Dims == AnyDims || D2 == AnyDims || Dims == D2,
139 "Can't convert from a Buffer with static dimensionality to a Buffer with different static dimensionality");
140 } else {
141 // Don't delegate to
142 // Runtime::Buffer<T>::assert_can_convert_from. It might
143 // not assert if NDEBUG is defined. user_assert is
144 // friendlier anyway because it reports line numbers when
145 // debugging symbols are found, it throws an exception
146 // when exceptions are enabled, and we can print the
147 // actual types in question.
148 using BufType = Runtime::Buffer<T, Dims>; // alias because commas in user_assert() macro confuses compiler
149 user_assert(BufType::can_convert_from(*(other.get())))
150 << "Type mismatch constructing Buffer. Can't construct Buffer<"
151 << Internal::buffer_type_name<T>() << ", " << Dims << "> from Buffer<"
152 << type_to_c_type(other.type(), false) << ", " << D2 << ">, dimensions() = " << other.dimensions() << "\n";
153 }
154 }
155
156 public:
157 static constexpr int AnyDims = Halide::AnyDims;
158 static_assert(Dims == AnyDims || Dims >= 0);
159
160 typedef T ElemType;
161
162 // This class isn't final (and is subclassed from the Python binding
163 // code, at least) so it needs a virtual dtor.
164 virtual ~Buffer() = default;
165
166 /** Make a null Buffer, which points to no Runtime::Buffer */
167 Buffer() = default;
168
169 /** Trivial copy constructor. */
170 Buffer(const Buffer &that) = default;
171
172 /** Trivial copy assignment operator. */
173 Buffer &operator=(const Buffer &that) = default;
174
175 /** Trivial move assignment operator. */
176 Buffer &operator=(Buffer &&) noexcept = default;
177
178 /** Make a Buffer from a Buffer of a different type */
179 template<typename T2, int D2>
180 Buffer(const Buffer<T2, D2> &other)
181 : contents(other.contents) {
182 assert_can_convert_from(other);
183 }
184
185 /** Move construct from a Buffer of a different type */
186 template<typename T2, int D2>
187 Buffer(Buffer<T2, D2> &&other) noexcept {
188 assert_can_convert_from(other);
189 contents = std::move(other.contents);
190 }
191
192 /** Construct a Buffer that captures and owns an rvalue Runtime::Buffer */
193 template<int D2>
194 Buffer(Runtime::Buffer<T, D2> &&buf, const std::string &name = "")
195 : contents(new Internal::BufferContents) {
196 contents->buf = std::move(buf);
197 if (name.empty()) {
198 contents->name = Internal::unique_name('b');
199 } else {
200 contents->name = name;
201 }
202 }
203
204 /** Constructors that match Runtime::Buffer with two differences:
205 * 1) They take a Type instead of a halide_type_t
206 * 2) There is an optional last string argument that gives the buffer a specific name
207 */
208 // @{
209 template<typename... Args,
210 typename = std::enable_if_t<Internal::all_ints_and_optional_name<Args...>::value>>
211 explicit Buffer(Type t,
212 int first, Args... rest)
213 : Buffer(Runtime::Buffer<T, Dims>(t, Internal::get_shape_from_start_of_parameter_pack(first, rest...)),
214 Internal::get_name_from_end_of_parameter_pack(rest...)) {
215 }
216
217 explicit Buffer(const halide_buffer_t &buf,
218 const std::string &name = "")
219 : Buffer(Runtime::Buffer<T, Dims>(buf), name) {
220 }
221
222 template<typename... Args,
223 typename = std::enable_if_t<Internal::all_ints_and_optional_name<Args...>::value>>
224 explicit Buffer(int first, const Args &...rest)
225 : Buffer(Runtime::Buffer<T, Dims>(Internal::get_shape_from_start_of_parameter_pack(first, rest...)),
226 Internal::get_name_from_end_of_parameter_pack(rest...)) {
227 }
228
229 explicit Buffer(Type t,
230 const std::vector<int> &sizes,
231 const std::string &name = "")
232 : Buffer(Runtime::Buffer<T, Dims>(t, sizes), name) {
233 }
234
235 explicit Buffer(Type t,
236 const std::vector<int> &sizes,
237 const std::vector<int> &storage_order,
238 const std::string &name = "")
239 : Buffer(Runtime::Buffer<T, Dims>(t, sizes, storage_order), name) {
240 }
241
242 explicit Buffer(const std::vector<int> &sizes,
243 const std::string &name = "")
244 : Buffer(Runtime::Buffer<T, Dims>(sizes), name) {
245 }
246
247 explicit Buffer(const std::vector<int> &sizes,
248 const std::vector<int> &storage_order,
249 const std::string &name = "")
250 : Buffer(Runtime::Buffer<T, Dims>(sizes, storage_order), name) {
251 }
252
253 template<typename Array, size_t N>
254 explicit Buffer(Array (&vals)[N],
255 const std::string &name = "")
256 : Buffer(Runtime::Buffer<T, Dims>(vals), name) {
257 }
258
259 template<typename... Args,
260 typename = std::enable_if_t<Internal::all_ints_and_optional_name<Args...>::value>>
261 explicit Buffer(Type t,
262 Internal::add_const_if_T_is_const<T, void> *data,
263 int first, Args &&...rest)
264 : Buffer(Runtime::Buffer<T, Dims>(t, data, Internal::get_shape_from_start_of_parameter_pack(first, rest...)),
265 Internal::get_name_from_end_of_parameter_pack(rest...)) {
266 }
267
268 template<typename... Args,
269 typename = std::enable_if_t<Internal::all_ints_and_optional_name<Args...>::value>>
270 explicit Buffer(Type t,
271 Internal::add_const_if_T_is_const<T, void> *data,
272 const std::vector<int> &sizes,
273 const std::string &name = "")
274 : Buffer(Runtime::Buffer<T, Dims>(t, data, sizes, name)) {
275 }
276
277 template<typename... Args,
278 typename = std::enable_if_t<Internal::all_ints_and_optional_name<Args...>::value>>
279 explicit Buffer(T *data,
280 int first, Args &&...rest)
281 : Buffer(Runtime::Buffer<T, Dims>(data, Internal::get_shape_from_start_of_parameter_pack(first, rest...)),
282 Internal::get_name_from_end_of_parameter_pack(rest...)) {
283 }
284
285 explicit Buffer(T *data,
286 const std::vector<int> &sizes,
287 const std::string &name = "")
288 : Buffer(Runtime::Buffer<T, Dims>(data, sizes), name) {
289 }
290
291 explicit Buffer(Type t,
292 Internal::add_const_if_T_is_const<T, void> *data,
293 const std::vector<int> &sizes,
294 const std::string &name = "")
295 : Buffer(Runtime::Buffer<T, Dims>(t, data, sizes), name) {
296 }
297
298 explicit Buffer(Type t,
299 Internal::add_const_if_T_is_const<T, void> *data,
300 int d,
301 const halide_dimension_t *shape,
302 const std::string &name = "")
303 : Buffer(Runtime::Buffer<T, Dims>(t, data, d, shape), name) {
304 }
305
306 explicit Buffer(T *data,
307 int d,
308 const halide_dimension_t *shape,
309 const std::string &name = "")
310 : Buffer(Runtime::Buffer<T, Dims>(data, d, shape), name) {
311 }
312
313 static Buffer<T, Dims> make_scalar(const std::string &name = "") {
314 return Buffer<T, Dims>(Runtime::Buffer<T, Dims>::make_scalar(), name);
315 }
316
317 static Buffer<> make_scalar(Type t, const std::string &name = "") {
318 return Buffer<>(Runtime::Buffer<>::make_scalar(t), name);
319 }
320
321 static Buffer<T, Dims> make_scalar(T *data, const std::string &name = "") {
322 return Buffer<T, Dims>(Runtime::Buffer<T, Dims>::make_scalar(data), name);
323 }
324
325 static Buffer<T, Dims> make_interleaved(int width, int height, int channels, const std::string &name = "") {
326 return Buffer<T, Dims>(Runtime::Buffer<T, Dims>::make_interleaved(width, height, channels), name);
327 }
328
329 static Buffer<> make_interleaved(Type t, int width, int height, int channels, const std::string &name = "") {
330 return Buffer<>(Runtime::Buffer<>::make_interleaved(t, width, height, channels), name);
331 }
332
333 static Buffer<T, Dims> make_interleaved(T *data, int width, int height, int channels, const std::string &name = "") {
334 return Buffer<T, Dims>(Runtime::Buffer<T, Dims>::make_interleaved(data, width, height, channels), name);
335 }
336
337 static Buffer<Internal::add_const_if_T_is_const<T, void>>
338 make_interleaved(Type t, T *data, int width, int height, int channels, const std::string &name = "") {
339 using T2 = Internal::add_const_if_T_is_const<T, void>;
340 return Buffer<T2, Dims>(Runtime::Buffer<T2, Dims>::make_interleaved(t, data, width, height, channels), name);
341 }
342
343 template<typename T2, int D2>
344 static Buffer<T, Dims> make_with_shape_of(Buffer<T2, D2> src,
345 void *(*allocate_fn)(size_t) = nullptr,
346 void (*deallocate_fn)(void *) = nullptr,
347 const std::string &name = "") {
348 return Buffer<T, Dims>(Runtime::Buffer<T, Dims>::make_with_shape_of(*src.get(), allocate_fn, deallocate_fn), name);
349 }
350
351 template<typename T2, int D2>
352 static Buffer<T, Dims> make_with_shape_of(const Runtime::Buffer<T2, D2> &src,
353 void *(*allocate_fn)(size_t) = nullptr,
354 void (*deallocate_fn)(void *) = nullptr,
355 const std::string &name = "") {
356 return Buffer<T, Dims>(Runtime::Buffer<T, Dims>::make_with_shape_of(src, allocate_fn, deallocate_fn), name);
357 }
358 // @}
359
360 /** Buffers are optionally named. */
361 // @{
362 void set_name(const std::string &n) {
363 contents->name = n;
364 }
365
366 const std::string &name() const {
367 return contents->name;
368 }
369 // @}
370
371 /** Check if two Buffer objects point to the same underlying Buffer */
372 template<typename T2, int D2>
373 bool same_as(const Buffer<T2, D2> &other) const {
374 return (const void *)(contents.get()) == (const void *)(other.contents.get());
375 }
376
377 /** Check if this Buffer refers to an existing
378 * Buffer. Default-constructed Buffer objects do not refer to any
379 * existing Buffer. */
380 bool defined() const {
381 return contents.defined();
382 }
383
384 /** Get a pointer to the underlying Runtime::Buffer */
385 // @{
386 Runtime::Buffer<T, Dims> *get() {
387 // It's already type-checked, so no need to use as<T>.
388 return (Runtime::Buffer<T, Dims> *)(&contents->buf);
389 }
390 const Runtime::Buffer<T, Dims> *get() const {
391 return (const Runtime::Buffer<T, Dims> *)(&contents->buf);
392 }
393 // @}
394
395 // We forward numerous methods from the underlying Buffer
396#define HALIDE_BUFFER_FORWARD_CONST(method) \
397 template<typename... Args> \
398 auto method(Args &&...args) const -> decltype(std::declval<const Runtime::Buffer<T, Dims>>().method(std::forward<Args>(args)...)) { \
399 user_assert(defined()) << "Undefined buffer calling const method " #method "\n"; \
400 return get()->method(std::forward<Args>(args)...); \
401 }
402
403#define HALIDE_BUFFER_FORWARD(method) \
404 template<typename... Args> \
405 auto method(Args &&...args) -> decltype(std::declval<Runtime::Buffer<T, Dims>>().method(std::forward<Args>(args)...)) { \
406 user_assert(defined()) << "Undefined buffer calling method " #method "\n"; \
407 return get()->method(std::forward<Args>(args)...); \
408 }
409
410// This is a weird-looking but effective workaround for a deficiency in "perfect forwarding":
411// namely, it can't really handle initializer-lists. The idea here is that we declare
412// the expected type to be passed on, and that allows the compiler to handle it.
413// The weirdness comes in with the variadic macro: the problem is that the type
414// we want to forward might be something like `std::vector<std::pair<int, int>>`,
415// which contains a comma, which throws a big wrench in C++ macro system.
416// However... since all we really need to do is capture the remainder of the macro,
417// and forward it as is, we can just use ... to allow an arbitrary number of commas,
418// then use __VA_ARGS__ to forward the mess as-is, and while it looks horrible, it
419// works.
420#define HALIDE_BUFFER_FORWARD_INITIALIZER_LIST(method, ...) \
421 inline auto method(const __VA_ARGS__ &a) -> decltype(std::declval<Runtime::Buffer<T, Dims>>().method(a)) { \
422 user_assert(defined()) << "Undefined buffer calling method " #method "\n"; \
423 return get()->method(a); \
424 }
425
426 /** Does the same thing as the equivalent Halide::Runtime::Buffer method */
427 // @{
428 HALIDE_BUFFER_FORWARD(raw_buffer)
429 HALIDE_BUFFER_FORWARD_CONST(raw_buffer)
430 HALIDE_BUFFER_FORWARD_CONST(dimensions)
431 HALIDE_BUFFER_FORWARD_CONST(dim)
432 HALIDE_BUFFER_FORWARD_CONST(width)
433 HALIDE_BUFFER_FORWARD_CONST(height)
434 HALIDE_BUFFER_FORWARD_CONST(channels)
435 HALIDE_BUFFER_FORWARD_CONST(min)
436 HALIDE_BUFFER_FORWARD_CONST(extent)
437 HALIDE_BUFFER_FORWARD_CONST(stride)
438 HALIDE_BUFFER_FORWARD_CONST(left)
439 HALIDE_BUFFER_FORWARD_CONST(right)
440 HALIDE_BUFFER_FORWARD_CONST(top)
441 HALIDE_BUFFER_FORWARD_CONST(bottom)
442 HALIDE_BUFFER_FORWARD_CONST(number_of_elements)
443 HALIDE_BUFFER_FORWARD_CONST(size_in_bytes)
444 HALIDE_BUFFER_FORWARD_CONST(begin)
445 HALIDE_BUFFER_FORWARD_CONST(end)
446 HALIDE_BUFFER_FORWARD(data)
447 HALIDE_BUFFER_FORWARD_CONST(data)
448 HALIDE_BUFFER_FORWARD_CONST(contains)
449 HALIDE_BUFFER_FORWARD(crop)
450 HALIDE_BUFFER_FORWARD_INITIALIZER_LIST(crop, std::vector<std::pair<int, int>>)
451 HALIDE_BUFFER_FORWARD_CONST(cropped)
452 HALIDE_BUFFER_FORWARD(slice)
453 HALIDE_BUFFER_FORWARD_CONST(sliced)
454 HALIDE_BUFFER_FORWARD(embed)
455 HALIDE_BUFFER_FORWARD_CONST(embedded)
456 HALIDE_BUFFER_FORWARD(set_min)
457 HALIDE_BUFFER_FORWARD(translate)
458 HALIDE_BUFFER_FORWARD_INITIALIZER_LIST(translate, std::vector<int>)
459 HALIDE_BUFFER_FORWARD_CONST(translated)
460 HALIDE_BUFFER_FORWARD(transpose)
461 HALIDE_BUFFER_FORWARD_CONST(transposed)
462 HALIDE_BUFFER_FORWARD(add_dimension)
463 HALIDE_BUFFER_FORWARD(copy_to_host)
464 HALIDE_BUFFER_FORWARD(copy_to_device)
465 HALIDE_BUFFER_FORWARD_CONST(has_device_allocation)
466 HALIDE_BUFFER_FORWARD_CONST(host_dirty)
467 HALIDE_BUFFER_FORWARD_CONST(device_dirty)
468 HALIDE_BUFFER_FORWARD(set_host_dirty)
469 HALIDE_BUFFER_FORWARD(set_device_dirty)
470 HALIDE_BUFFER_FORWARD(device_sync)
471 HALIDE_BUFFER_FORWARD(device_malloc)
472 HALIDE_BUFFER_FORWARD(device_wrap_native)
473 HALIDE_BUFFER_FORWARD(device_detach_native)
474 HALIDE_BUFFER_FORWARD(allocate)
475 HALIDE_BUFFER_FORWARD(deallocate)
476 HALIDE_BUFFER_FORWARD(device_deallocate)
477 HALIDE_BUFFER_FORWARD(device_free)
478 HALIDE_BUFFER_FORWARD_CONST(all_equal)
479
480#undef HALIDE_BUFFER_FORWARD
481#undef HALIDE_BUFFER_FORWARD_CONST
482
483 template<typename Fn, typename... Args>
484 Buffer<T, Dims> &for_each_value(Fn &&f, Args... other_buffers) {
485 get()->for_each_value(std::forward<Fn>(f), (*std::forward<Args>(other_buffers).get())...);
486 return *this;
487 }
488
489 template<typename Fn, typename... Args>
490 const Buffer<T, Dims> &for_each_value(Fn &&f, Args... other_buffers) const {
491 get()->for_each_value(std::forward<Fn>(f), (*std::forward<Args>(other_buffers).get())...);
492 return *this;
493 }
494
495 template<typename Fn>
496 Buffer<T, Dims> &for_each_element(Fn &&f) {
497 get()->for_each_element(std::forward<Fn>(f));
498 return *this;
499 }
500
501 template<typename Fn>
502 const Buffer<T, Dims> &for_each_element(Fn &&f) const {
503 get()->for_each_element(std::forward<Fn>(f));
504 return *this;
505 }
506
507 template<typename FnOrValue>
508 Buffer<T, Dims> &fill(FnOrValue &&f) {
509 get()->fill(std::forward<FnOrValue>(f));
510 return *this;
511 }
512
513 static constexpr bool has_static_halide_type = Runtime::Buffer<T, Dims>::has_static_halide_type;
514
515 static constexpr halide_type_t static_halide_type() {
516 return Runtime::Buffer<T, Dims>::static_halide_type();
517 }
518
519 static constexpr bool has_static_dimensions = Runtime::Buffer<T, Dims>::has_static_dimensions;
520
521 static constexpr int static_dimensions() {
522 return Runtime::Buffer<T, Dims>::static_dimensions();
523 }
524
525 template<typename T2, int D2>
526 static bool can_convert_from(const Buffer<T2, D2> &other) {
527 return Halide::Runtime::Buffer<T, Dims>::can_convert_from(*other.get());
528 }
529
530 // Note that since Runtime::Buffer stores halide_type_t rather than Halide::Type,
531 // there is no handle-specific type information, so all handle types are
532 // considered equivalent to void* here. (This only matters if you are making
533 // a Buffer-of-handles, which is not really a real use case...)
534 Type type() const {
535 return contents->buf.type();
536 }
537
538 template<typename T2, int D2 = Dims>
539 Buffer<T2, D2> as() const {
540 return Buffer<T2, D2>(*this);
541 }
542
543 Buffer<T, Dims> copy() const {
544 return Buffer<T, Dims>(std::move(contents->buf.as<T, Dims>().copy()));
545 }
546
547 template<typename T2, int D2>
548 void copy_from(const Buffer<T2, D2> &other) {
549 contents->buf.copy_from(*other.get());
550 }
551
552 template<typename... Args>
553 auto operator()(int first, Args &&...args) -> decltype(std::declval<Runtime::Buffer<T, Dims>>()(first, std::forward<Args>(args)...)) {
554 return (*get())(first, std::forward<Args>(args)...);
555 }
556
557 template<typename... Args>
558 auto operator()(int first, Args &&...args) const -> decltype(std::declval<const Runtime::Buffer<T, Dims>>()(first, std::forward<Args>(args)...)) {
559 return (*get())(first, std::forward<Args>(args)...);
560 }
561
562 auto operator()(const int *pos) -> decltype(std::declval<Runtime::Buffer<T, Dims>>()(pos)) {
563 return (*get())(pos);
564 }
565
566 auto operator()(const int *pos) const -> decltype(std::declval<const Runtime::Buffer<T, Dims>>()(pos)) {
567 return (*get())(pos);
568 }
569
570 auto operator()() -> decltype(std::declval<Runtime::Buffer<T, Dims>>()()) {
571 return (*get())();
572 }
573
574 auto operator()() const -> decltype(std::declval<const Runtime::Buffer<T, Dims>>()()) {
575 return (*get())();
576 }
577 // @}
578
579 /** Make an Expr that loads from this concrete buffer at a computed
580 * coordinate. Returned Expr is const so that it's not possible to
581 * accidentally treat a buffer like a Func and try to assign an Expr to a
582 * given symbolic coordinate. */
583 // @{
584 template<typename... Args>
585 const Expr operator()(const Expr &first, const Args &...rest) const { // NOLINT(readability-const-return-type)
586 std::vector<Expr> args = {first, rest...};
587 return (*this)(args);
588 }
589
590 template<typename... Args>
591 const Expr operator()(const std::vector<Expr> &args) const { // NOLINT(readability-const-return-type)
592 return buffer_accessor(Buffer<>(*this), args);
593 }
594 // @}
595
596 /** Copy to the GPU, using the device API that is the default for the given Target. */
597 int copy_to_device(const Target &t = get_jit_target_from_environment(), JITUserContext *context = nullptr) {
598 return copy_to_device(DeviceAPI::Default_GPU, t, context);
599 }
600
601 /** Copy to the GPU, using the given device API */
602 int copy_to_device(const DeviceAPI &d, const Target &t = get_jit_target_from_environment(), JITUserContext *context = nullptr) {
603 return contents->buf.copy_to_device(get_device_interface_for_device_api(d, t, "Buffer::copy_to_device"), context);
604 }
605
606 /** Allocate on the GPU, using the device API that is the default for the given Target. */
607 int device_malloc(const Target &t = get_jit_target_from_environment(), JITUserContext *context = nullptr) {
608 return device_malloc(DeviceAPI::Default_GPU, t, context);
609 }
610
611 /** Allocate storage on the GPU, using the given device API */
612 int device_malloc(const DeviceAPI &d, const Target &t = get_jit_target_from_environment(), JITUserContext *context = nullptr) {
613 return contents->buf.device_malloc(get_device_interface_for_device_api(d, t, "Buffer::device_malloc"), context);
614 }
615
616 /** Wrap a native handle, using the given device API.
617 * It is a bad idea to pass DeviceAPI::Default_GPU to this routine
618 * as the handle argument must match the API that the default
619 * resolves to and it is clearer and more reliable to pass the
620 * resolved DeviceAPI explicitly. */
621 int device_wrap_native(const DeviceAPI &d, uint64_t handle, const Target &t = get_jit_target_from_environment(), JITUserContext *context = nullptr) {
622 return contents->buf.device_wrap_native(get_device_interface_for_device_api(d, t, "Buffer::device_wrap_native"), handle, context);
623 }
624};
625
626} // namespace Halide
627
628#endif
HALIDE_BUFFER_FORWARD
#define HALIDE_BUFFER_FORWARD(method)
Definition: Buffer.h:403
HALIDE_BUFFER_FORWARD_INITIALIZER_LIST
#define HALIDE_BUFFER_FORWARD_INITIALIZER_LIST(method,...)
Definition: Buffer.h:420
HALIDE_BUFFER_FORWARD_CONST
#define HALIDE_BUFFER_FORWARD_CONST(method)
Definition: Buffer.h:396
DeviceInterface.h
Methods for managing device allocations when jitting.
user_assert
#define user_assert(c)
Definition: Error.h:233
Expr.h
Base classes for Halide expressions (Halide::Expr) and statements (Halide::Internal::Stmt)
HalideBuffer.h
Defines a Buffer type that wraps from halide_buffer_t and adds functionality, and methods for more co...
IntrusivePtr.h
Support classes for reference-counting via intrusive shared pointers.
Halide::Buffer
A Halide::Buffer is a named shared reference to a Halide::Runtime::Buffer.
Definition: Buffer.h:122
Halide::Buffer::operator()
auto operator()() -> decltype(std::declval< Runtime::Buffer< T, Dims > >()())
Definition: Buffer.h:570
Halide::Buffer::operator=
Buffer & operator=(const Buffer &that)=default
Trivial copy assignment operator.
Halide::Buffer::make_with_shape_of
static Buffer< T, Dims > make_with_shape_of(const Runtime::Buffer< T2, D2 > &src, void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr, const std::string &name="")
Definition: Buffer.h:352
Halide::Buffer::static_dimensions
static constexpr int static_dimensions()
Definition: Buffer.h:521
Halide::Buffer::type
Type type() const
Definition: Buffer.h:534
Halide::Buffer::Buffer
Buffer(Type t, const std::vector< int > &sizes, const std::string &name="")
Definition: Buffer.h:229
Halide::Buffer::Buffer
Buffer(Runtime::Buffer< T, D2 > &&buf, const std::string &name="")
Construct a Buffer that captures and owns an rvalue Runtime::Buffer.
Definition: Buffer.h:194
Halide::Buffer::for_each_value
Buffer< T, Dims > & for_each_value(Fn &&f, Args... other_buffers)
Does the same thing as the equivalent Halide::Runtime::Buffer method.
Definition: Buffer.h:484
Halide::Buffer::set_name
void set_name(const std::string &n)
Buffers are optionally named.
Definition: Buffer.h:362
Halide::Buffer::has_static_dimensions
static constexpr bool has_static_dimensions
Definition: Buffer.h:519
Halide::Buffer::for_each_element
const Buffer< T, Dims > & for_each_element(Fn &&f) const
Definition: Buffer.h:502
Halide::Buffer::device_malloc
int device_malloc(const DeviceAPI &d, const Target &t=get_jit_target_from_environment(), JITUserContext *context=nullptr)
Allocate storage on the GPU, using the given device API.
Definition: Buffer.h:612
Halide::Buffer::~Buffer
virtual ~Buffer()=default
Halide::Buffer::Buffer
Buffer(const Buffer &that)=default
Trivial copy constructor.
Halide::Buffer::Buffer
Buffer(Type t, int first, Args... rest)
Constructors that match Runtime::Buffer with two differences: 1) They take a Type instead of a halide...
Definition: Buffer.h:211
Halide::Buffer::for_each_value
const Buffer< T, Dims > & for_each_value(Fn &&f, Args... other_buffers) const
Definition: Buffer.h:490
Halide::Buffer::make_interleaved
static Buffer< T, Dims > make_interleaved(int width, int height, int channels, const std::string &name="")
Definition: Buffer.h:325
Halide::Buffer::defined
bool defined() const
Check if this Buffer refers to an existing Buffer.
Definition: Buffer.h:380
Halide::Buffer::operator()
const Expr operator()(const std::vector< Expr > &args) const
Definition: Buffer.h:591
Halide::Buffer::same_as
bool same_as(const Buffer< T2, D2 > &other) const
Check if two Buffer objects point to the same underlying Buffer.
Definition: Buffer.h:373
Halide::Buffer::operator()
auto operator()(const int *pos) -> decltype(std::declval< Runtime::Buffer< T, Dims > >()(pos))
Definition: Buffer.h:562
Halide::Buffer::for_each_element
Buffer< T, Dims > & for_each_element(Fn &&f)
Definition: Buffer.h:496
Halide::Buffer::make_interleaved
static Buffer make_interleaved(Type t, int width, int height, int channels, const std::string &name="")
Definition: Buffer.h:329
Halide::Buffer::make_scalar
static Buffer make_scalar(Type t, const std::string &name="")
Definition: Buffer.h:317
Halide::Buffer::make_scalar
static Buffer< T, Dims > make_scalar(T *data, const std::string &name="")
Definition: Buffer.h:321
Halide::Buffer::Buffer
Buffer(Type t, Internal::add_const_if_T_is_const< T, void > *data, int d, const halide_dimension_t *shape, const std::string &name="")
Definition: Buffer.h:298
Halide::Buffer::operator()
auto operator()(int first, Args &&...args) -> decltype(std::declval< Runtime::Buffer< T, Dims > >()(first, std::forward< Args >(args)...))
Definition: Buffer.h:553
Halide::Buffer::AnyDims
static constexpr int AnyDims
Definition: Buffer.h:157
Halide::Buffer::copy_to_device
int copy_to_device(const DeviceAPI &d, const Target &t=get_jit_target_from_environment(), JITUserContext *context=nullptr)
Copy to the GPU, using the given device API.
Definition: Buffer.h:602
Halide::Buffer::static_halide_type
static constexpr halide_type_t static_halide_type()
Definition: Buffer.h:515
Halide::Buffer::operator()
auto operator()(int first, Args &&...args) const -> decltype(std::declval< const Runtime::Buffer< T, Dims > >()(first, std::forward< Args >(args)...))
Definition: Buffer.h:558
Halide::Buffer::Buffer
Buffer(Array(&vals)[N], const std::string &name="")
Definition: Buffer.h:254
Halide::Buffer::copy
Buffer< T, Dims > copy() const
Definition: Buffer.h:543
Halide::Buffer::Buffer
Buffer(const std::vector< int > &sizes, const std::vector< int > &storage_order, const std::string &name="")
Definition: Buffer.h:247
Halide::Buffer::fill
Buffer< T, Dims > & fill(FnOrValue &&f)
Definition: Buffer.h:508
Halide::Buffer::name
const std::string & name() const
Definition: Buffer.h:366
Halide::Buffer::make_with_shape_of
static Buffer< T, Dims > make_with_shape_of(Buffer< T2, D2 > src, void *(*allocate_fn)(size_t)=nullptr, void(*deallocate_fn)(void *)=nullptr, const std::string &name="")
Definition: Buffer.h:344
Halide::Buffer::get
Runtime::Buffer< T, Dims > * get()
Get a pointer to the underlying Runtime::Buffer.
Definition: Buffer.h:386
Halide::Buffer::device_wrap_native
int device_wrap_native(const DeviceAPI &d, uint64_t handle, const Target &t=get_jit_target_from_environment(), JITUserContext *context=nullptr)
Wrap a native handle, using the given device API.
Definition: Buffer.h:621
Halide::Buffer::Buffer
Buffer()=default
Make a null Buffer, which points to no Runtime::Buffer.
Halide::Buffer::device_malloc
int device_malloc(const Target &t=get_jit_target_from_environment(), JITUserContext *context=nullptr)
Allocate on the GPU, using the device API that is the default for the given Target.
Definition: Buffer.h:607
Halide::Buffer::Buffer
Buffer(Type t, Internal::add_const_if_T_is_const< T, void > *data, const std::vector< int > &sizes, const std::string &name="")
Definition: Buffer.h:291
Halide::Buffer::has_static_halide_type
static constexpr bool has_static_halide_type
Definition: Buffer.h:513
Halide::Buffer::Buffer
Buffer(T *data, const std::vector< int > &sizes, const std::string &name="")
Definition: Buffer.h:285
Halide::Buffer::Buffer
Buffer(T *data, int first, Args &&...rest)
Definition: Buffer.h:279
Halide::Buffer::make_scalar
static Buffer< T, Dims > make_scalar(const std::string &name="")
Definition: Buffer.h:313
Halide::Buffer::Buffer
Buffer(Type t, Internal::add_const_if_T_is_const< T, void > *data, int first, Args &&...rest)
Definition: Buffer.h:261
Halide::Buffer::Buffer
Buffer(Buffer< T2, D2 > &&other) noexcept
Move construct from a Buffer of a different type.
Definition: Buffer.h:187
Halide::Buffer::operator=
Buffer & operator=(Buffer &&) noexcept=default
Trivial move assignment operator.
Halide::Buffer::Buffer
Buffer(int first, const Args &...rest)
Definition: Buffer.h:224
Halide::Buffer::Buffer
Buffer(Type t, const std::vector< int > &sizes, const std::vector< int > &storage_order, const std::string &name="")
Definition: Buffer.h:235
Halide::Buffer::operator()
auto operator()(const int *pos) const -> decltype(std::declval< const Runtime::Buffer< T, Dims > >()(pos))
Definition: Buffer.h:566
Halide::Buffer::copy_to_device
int copy_to_device(const Target &t=get_jit_target_from_environment(), JITUserContext *context=nullptr)
Copy to the GPU, using the device API that is the default for the given Target.
Definition: Buffer.h:597
Halide::Buffer::ElemType
T ElemType
Definition: Buffer.h:158
Halide::Buffer::make_interleaved
static Buffer< Internal::add_const_if_T_is_const< T, void > > make_interleaved(Type t, T *data, int width, int height, int channels, const std::string &name="")
Definition: Buffer.h:338
Halide::Buffer::as
Buffer< T2, D2 > as() const
Definition: Buffer.h:539
Halide::Buffer::Buffer
Buffer(const std::vector< int > &sizes, const std::string &name="")
Definition: Buffer.h:242
Halide::Buffer::operator()
const Expr operator()(const Expr &first, const Args &...rest) const
Make an Expr that loads from this concrete buffer at a computed coordinate.
Definition: Buffer.h:585
Halide::Buffer::Buffer
Buffer(const halide_buffer_t &buf, const std::string &name="")
Definition: Buffer.h:217
Halide::Buffer::Buffer
Buffer(T *data, int d, const halide_dimension_t *shape, const std::string &name="")
Definition: Buffer.h:306
Halide::Buffer::Buffer
Buffer(Type t, Internal::add_const_if_T_is_const< T, void > *data, const std::vector< int > &sizes, const std::string &name="")
Definition: Buffer.h:270
Halide::Buffer::can_convert_from
static bool can_convert_from(const Buffer< T2, D2 > &other)
Definition: Buffer.h:526
Halide::Buffer::get
const Runtime::Buffer< T, Dims > * get() const
Definition: Buffer.h:390
Halide::Buffer::operator()
auto operator()() const -> decltype(std::declval< const Runtime::Buffer< T, Dims > >()())
Definition: Buffer.h:574
Halide::Buffer::copy_from
void copy_from(const Buffer< T2, D2 > &other)
Definition: Buffer.h:548
Halide::Buffer::make_interleaved
static Buffer< T, Dims > make_interleaved(T *data, int width, int height, int channels, const std::string &name="")
Definition: Buffer.h:333
Halide::Internal::RefCount
A class representing a reference count to be used with IntrusivePtr.
Definition: IntrusivePtr.h:19
Halide::Runtime::Buffer
A templated Buffer class that wraps halide_buffer_t and adds functionality.
Definition: HalideBuffer.h:224
Halide::Runtime::Buffer::can_convert_from
static bool can_convert_from(const Buffer< T2, D2, S2 > &other)
Determine if a Buffer<T, Dims, InClassDimStorage> can be constructed from some other Buffer type.
Definition: HalideBuffer.h:740
Halide::Runtime::Buffer::static_halide_type
static constexpr halide_type_t static_halide_type()
Get the Halide type of T.
Definition: HalideBuffer.h:267
Halide::Runtime::Buffer::static_dimensions
static constexpr int static_dimensions()
Callers should not use the result if has_static_dimensions is false.
Definition: HalideBuffer.h:280
Halide::Internal::IRMatcher::slice
HALIDE_ALWAYS_INLINE auto slice(Vec vec, Base base, Stride stride, Lanes lanes) noexcept -> SliceOp< decltype(pattern_arg(vec)), decltype(pattern_arg(base)), decltype(pattern_arg(stride)), decltype(pattern_arg(lanes))>
Definition: IRMatch.h:2247
Halide::Internal::get_shape_from_start_of_parameter_pack
std::vector< int > get_shape_from_start_of_parameter_pack(Args &&...args)
Definition: Buffer.h:76
Halide::Internal::begin
auto begin(reverse_adaptor< T > i)
Definition: Util.h:478
Halide::Internal::add_const_if_T_is_const
std::conditional_t< std::is_const_v< T >, const T2, T2 > add_const_if_T_is_const
Definition: Buffer.h:83
Halide::Internal::get_name_from_end_of_parameter_pack
std::string get_name_from_end_of_parameter_pack(First first, Second second, Args &&...rest)
Definition: Buffer.h:59
Halide::Internal::buffer_type_name_non_const< void >
void buffer_type_name_non_const< void >(std::ostream &s)
Definition: Buffer.h:94
Halide::Internal::buffer_type_name_non_const
void buffer_type_name_non_const(std::ostream &s)
Definition: Buffer.h:89
Halide::Internal::buffer_accessor
Expr buffer_accessor(const Buffer<> &buf, const std::vector< Expr > &args)
Halide::Internal::unique_name
std::string unique_name(char prefix)
Generate a unique name starting with the given prefix.
Halide::Internal::buffer_type_name
std::string buffer_type_name()
Definition: Buffer.h:99
Halide::Internal::get_shape_from_start_of_parameter_pack_helper
void get_shape_from_start_of_parameter_pack_helper(std::vector< int > &, const std::string &)
Definition: Buffer.h:63
Halide::Internal::end
auto end(reverse_adaptor< T > i)
Definition: Util.h:483
Halide::Internal::get_name_from_end_of_parameter_pack
std::string get_name_from_end_of_parameter_pack(T &&)
Definition: Buffer.h:44
Halide::Runtime::AnyDims
constexpr int AnyDims
Definition: HalideBuffer.h:195
Halide
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
Definition: AbstractGenerator.h:19
Halide::type_to_c_type
std::string type_to_c_type(Type type, bool include_space, bool c_plus_plus=true)
Halide type to a C++ type.
Halide::LinkageType::Internal
@ Internal
Not visible externally, similar to 'static' linkage in C.
Halide::get_device_interface_for_device_api
const halide_device_interface_t * get_device_interface_for_device_api(DeviceAPI d, const Target &t=get_jit_target_from_environment(), const char *error_site=nullptr)
Gets the appropriate halide_device_interface_t * for a DeviceAPI.
Halide::min
Expr min(const FuncRef &a, const FuncRef &b)
Explicit overloads of min and max for FuncRef.
Definition: Func.h:603
Halide::get_jit_target_from_environment
Target get_jit_target_from_environment()
Return the target that Halide will use for jit-compilation.
Halide::DeviceAPI
DeviceAPI
An enum describing a type of device API.
Definition: DeviceAPI.h:15
Halide::DeviceAPI::Default_GPU
@ Default_GPU
Halide::AnyDims
constexpr int AnyDims
Definition: Buffer.h:11
uint64_t
unsigned __INT64_TYPE__ uint64_t
Definition: runtime_internal.h:23
Halide::Expr
A fragment of Halide syntax.
Definition: Expr.h:258
Halide::Internal::BufferContents
Definition: Buffer.h:20
Halide::Internal::BufferContents::buf
Runtime::Buffer buf
Definition: Buffer.h:23
Halide::Internal::BufferContents::ref_count
RefCount ref_count
Definition: Buffer.h:21
Halide::Internal::BufferContents::name
std::string name
Definition: Buffer.h:22
Halide::Internal::IntrusivePtr
Intrusive shared pointers have a reference count (a RefCount object) stored in the class itself.
Definition: IntrusivePtr.h:71
Halide::Internal::IntrusivePtr::defined
HALIDE_ALWAYS_INLINE bool defined() const
Definition: IntrusivePtr.h:164
Halide::Internal::IntrusivePtr::get
T * get() const
Access the raw pointer in a variety of ways.
Definition: IntrusivePtr.h:102
Halide::Internal::all_ints_and_optional_name
Definition: Buffer.h:29
Halide::Internal::meta_and
Definition: Util.h:251
Halide::Internal::meta_or
Definition: Util.h:257
Halide::JITUserContext
A context to be passed to Pipeline::realize.
Definition: JITModule.h:136
Halide::Target
A struct representing a target machine and os to generate code for.
Definition: Target.h:19
Halide::Type
Types in the halide type system.
Definition: Type.h:283
halide_buffer_t
The raw representation of an image passed around by generated Halide code.
Definition: HalideRuntime.h:1569
halide_dimension_t
Definition: HalideRuntime.h:1516
halide_type_t
A runtime tag for a type in the halide type system.
Definition: HalideRuntime.h:503