Halide 22.0.0 Halide compiler and libraries

CodeGen_LLVM.h
Go to the documentation of this file.
1#ifndef HALIDE_CODEGEN_LLVM_H
2#define HALIDE_CODEGEN_LLVM_H
3
4/** \file
5 *
6 * Defines the base-class for all architecture-specific code
7 * generators that use llvm.
8 */
9
10 namespace llvm {
11 class Value;
12 class Module;
13 class Function;
14 class FunctionType;
15 class IRBuilderDefaultInserter;
16 class ConstantFolder;
17 template<typename, typename>
18 class IRBuilder;
19 class LLVMContext;
20 class Type;
21 class PointerType;
22 class StructType;
23 class Instruction;
24 class CallInst;
25 class ExecutionEngine;
26 class AllocaInst;
27 class Constant;
28 class Triple;
29 class MDNode;
30 class NamedMDNode;
31 class DataLayout;
32 class BasicBlock;
33 class GlobalVariable;
34} // namespace llvm
35
36#include <map>
37#include <memory>
38#include <optional>
39#include <string>
40#include <variant>
41#include <vector>
42
43#include "IRVisitor.h"
44#include "Module.h"
45#include "Scope.h"
46#include "Target.h"
47
48 namespace Halide {
49
50 struct ExternSignature;
51
52 namespace Internal {
53
54/** A code generator abstract base class. Actual code generators
55 * (e.g. CodeGen_X86) inherit from this. This class is responsible
56 * for taking a Halide Stmt and producing llvm bitcode, machine
57 * code in an object file, or machine code accessible through a
58 * function pointer.
59 */
60 class CodeGen_LLVM : public IRVisitor {
61 public:
62 /** Create an instance of CodeGen_LLVM suitable for the target. */
63 static std::unique_ptr<CodeGen_LLVM> new_for_target(const Target &target, llvm::LLVMContext &context);
64
65 /** Takes a halide Module and compiles it to an llvm Module. */
66 virtual std::unique_ptr<llvm::Module> compile(const Module &module);
67
68 /** The target we're generating code for */
69 const Target &get_target() const {
70 return target;
71 }
72
73 /** Tell the code generator which LLVM context to use. */
74 void set_context(llvm::LLVMContext &context);
75
76 /** Initialize internal llvm state for the enabled targets. */
77 static void initialize_llvm();
78
79 static std::unique_ptr<llvm::Module> compile_trampolines(
80 const Target &target,
81 llvm::LLVMContext &context,
82 const std::string &suffix,
83 const std::vector<std::pair<std::string, ExternSignature>> &externs);
84
85 size_t get_requested_alloca_total() const {
86 return requested_alloca_total;
87 }
88
89 protected:
90 CodeGen_LLVM(const Target &t);
91
92 /** Compile a specific halide declaration into the llvm Module. */
93 // @{
94 virtual void compile_func(const LoweredFunc &func, const std::string &simple_name, const std::string &extern_name);
95 virtual void compile_buffer(const Buffer<> &buffer);
96 // @}
97
98 /** Helper functions for compiling Halide functions to llvm
99 * functions. begin_func performs all the work necessary to begin
100 * generating code for a function with a given argument list with
101 * the IRBuilder. A call to begin_func should be a followed by a
102 * call to end_func with the same arguments, to generate the
103 * appropriate cleanup code. */
104 // @{
105 virtual void begin_func(LinkageType linkage, const std::string &simple_name,
106 const std::string &extern_name, const std::vector<LoweredArgument> &args);
107 virtual void end_func(const std::vector<LoweredArgument> &args);
108 // @}
109
110 /** What should be passed as -mcpu (warning: implies attrs!), -mattrs,
111 * and related for compilation. The architecture-specific code generator
112 * should define these.
113 *
114 * `mcpu_target()` - target this specific CPU, in the sense of the allowed
115 * ISA sets *and* the CPU-specific tuning/assembly instruction scheduling.
116 *
117 * `mcpu_tune()` - expect that we will be running on this specific CPU,
118 * so perform CPU-specific tuning/assembly instruction scheduling, *but*
119 * DON'T sacrifice the portability, support running on other CPUs, only
120 * make use of the ISAs that are enabled by `mcpu_target()`+`mattrs()`.
121 */
122 // @{
123 virtual std::string mcpu_target() const = 0;
124 virtual std::string mcpu_tune() const = 0;
125 virtual std::string mattrs() const = 0;
126 virtual std::string mabi() const;
127 virtual bool use_soft_float_abi() const = 0;
128 virtual bool use_pic() const;
129 // @}
130
131 /** Should indexing math be promoted to 64-bit on platforms with
132 * 64-bit pointers? */
133 virtual bool promote_indices() const {
134 return true;
135 }
136
137 /** What's the natural vector bit-width to use for loads, stores, etc. */
138 virtual int native_vector_bits() const = 0;
139
140 /** Used to decide whether to break a vector up into multiple smaller
141 * operations. This is the largest size the architecture supports. */
142 virtual int maximum_vector_bits() const {
143 return native_vector_bits();
144 }
145 /** For architectures that have vscale vectors, return the constant vscale to use.
146 * Default of 0 means do not use vscale vectors. Generally will depend on
147 * the target flags and vector_bits settings.
148 */
149 virtual int target_vscale() const {
150 return 0;
151 }
152
153 /** Return the type in which arithmetic should be done for the
154 * given storage type. */
155 virtual Type upgrade_type_for_arithmetic(const Type &) const;
156
157 /** Return the type that a given Halide type should be
158 * stored/loaded from memory as. */
159 virtual Type upgrade_type_for_storage(const Type &) const;
160
161 /** Return the type that a Halide type should be passed in and out
162 * of functions as. */
163 virtual Type upgrade_type_for_argument_passing(const Type &) const;
164
165 std::unique_ptr<llvm::Module> module;
166 llvm::Function *function = nullptr;
167 llvm::LLVMContext *context = nullptr;
168 std::unique_ptr<llvm::IRBuilder<llvm::ConstantFolder, llvm::IRBuilderDefaultInserter>> builder;
169 llvm::Value *value = nullptr;
170 llvm::MDNode *very_likely_branch = nullptr;
171 llvm::MDNode *fast_fp_math_md = nullptr;
172 llvm::MDNode *strict_fp_math_md = nullptr;
173 std::vector<LoweredArgument> current_function_args;
174
175 bool in_strict_float = false;
176 bool any_strict_float = false;
177
178 /** Change floating-point math op emission to use fast flags. */
179 void set_fast_fp_math();
180
181 /** Change floating-point math op emission to use strict flags. */
182 void set_strict_fp_math();
183
184 /** If any_strict_float is true, sets fast math flags for the lifetime of
185 * this object, then sets them to strict on destruction. If any_strict_float
186 * is false, does nothing. Any call to an IRBuilder method that starts with
187 * "CreateF" should probably be wrapped in one of these, but it's safe to
188 * miss one - we just miss out on some optimizations. In this way codegen is
189 * designed to fail safe. */
190 struct ScopedFastMath {
191 CodeGen_LLVM *codegen;
192 ScopedFastMath(CodeGen_LLVM *);
193 ~ScopedFastMath();
194 };
195
196 /** The target we're generating code for */
197 Halide::Target target;
198
199 /** Grab all the context specific internal state. */
200 virtual void init_context();
201 /** Initialize the CodeGen_LLVM internal state to compile a fresh
202 * module. This allows reuse of one CodeGen_LLVM object to compiled
203 * multiple related modules (e.g. multiple device kernels). */
204 virtual void init_module();
205
206 /** Run all of llvm's optimization passes on the module. */
207 void optimize_module();
208
209 /** Add an entry to the symbol table, hiding previous entries with
210 * the same name. Call this when new values come into scope. */
211 void sym_push(const std::string &name, llvm::Value *value);
212
213 /** Remove an entry for the symbol table, revealing any previous
214 * entries with the same name. Call this when values go out of
215 * scope. */
216 void sym_pop(const std::string &name);
217
218 /** Fetch an entry from the symbol table. If the symbol is not
219 * found, it either errors out (if the second arg is true), or
220 * returns nullptr. */
221 llvm::Value *sym_get(const std::string &name,
222 bool must_succeed = true) const;
223
224 /** Test if an item exists in the symbol table. */
225 bool sym_exists(const std::string &name) const;
226
227 /** Given a Halide ExternSignature, return the equivalent llvm::FunctionType. */
228 llvm::FunctionType *signature_to_type(const ExternSignature &signature);
229
230 /** Some useful llvm types */
231 // @{
232 llvm::Type *void_t = nullptr, *i1_t = nullptr, *i8_t = nullptr, *i16_t = nullptr, *i32_t = nullptr, *i64_t = nullptr, *f16_t = nullptr, *f32_t = nullptr, *f64_t = nullptr;
233 llvm::PointerType *ptr_t = nullptr;
234 llvm::StructType *halide_buffer_t_type = nullptr,
235 *type_t_type,
236 *dimension_t_type,
237 *metadata_t_type = nullptr,
238 *argument_t_type = nullptr,
239 *scalar_value_t_type = nullptr,
240 *device_interface_t_type = nullptr,
241 *pseudostack_slot_t_type = nullptr,
242 *semaphore_t_type;
243
244 // @}
245
246 /** Some wildcard variables used for peephole optimizations in
247 * subclasses */
248 // @{
249 Expr wild_u1x_, wild_i8x_, wild_u8x_, wild_i16x_, wild_u16x_;
250 Expr wild_i32x_, wild_u32x_, wild_i64x_, wild_u64x_;
251 Expr wild_f32x_, wild_f64x_;
252
253 // Wildcards for scalars.
254 Expr wild_u1_, wild_i8_, wild_u8_, wild_i16_, wild_u16_;
255 Expr wild_i32_, wild_u32_, wild_i64_, wild_u64_;
256 Expr wild_f32_, wild_f64_;
257 // @}
258
259 /** Emit code that evaluates an expression, and return the llvm
260 * representation of the result of the expression. */
261 llvm::Value *codegen(const Expr &);
262
263 /** Emit code that runs a statement. */
264 void codegen(const Stmt &);
265
266 /** Codegen a vector Expr by codegenning each lane and combining. */
267 void scalarize(const Expr &);
268
269 /** Some destructors should always be called. Others should only
270 * be called if the pipeline is exiting with an error code. */
271 enum DestructorType { Always,
272 OnError,
273 OnSuccess };
274
275 /* Call this at the location of object creation to register how an
276 * object should be destroyed. This does three things:
277 * 1) Emits code here that puts the object in a unique
278 * null-initialized stack slot
279 * 2) Adds an instruction to the destructor block that calls the
280 * destructor on that stack slot if it's not null.
281 * 3) Returns that stack slot, so you can neuter the destructor
282 * (by storing null to the stack slot) or destroy the object early
283 * (by calling trigger_destructor).
284 */
285 llvm::Value *register_destructor(llvm::Function *destructor_fn, llvm::Value *obj, DestructorType when);
286
287 /** Call a destructor early. Pass in the value returned by register destructor. */
288 void trigger_destructor(llvm::Function *destructor_fn, llvm::Value *stack_slot);
289
290 /** Retrieves the block containing the error handling
291 * code. Creates it if it doesn't already exist for this
292 * function. */
293 llvm::BasicBlock *get_destructor_block();
294
295 /** Codegen an assertion. If false, returns the error code (if not
296 * null), or evaluates and returns the message, which must be an
297 * Int(32) expression. */
298 // @{
299 void create_assertion(llvm::Value *condition, const Expr &message, llvm::Value *error_code = nullptr);
300 // @}
301
302 /** Codegen a block of asserts with pure conditions */
303 void codegen_asserts(const std::vector<const AssertStmt *> &asserts);
304
305 /** Return the the pipeline with the given error code. Will run
306 * the destructor block. */
307 void return_with_error_code(llvm::Value *error_code);
308
309 /** Put a string constant in the module as a global variable and return a pointer to it. */
310 llvm::Constant *create_string_constant(const std::string &str);
311
312 /** Put a binary blob in the module as a global variable and return a pointer to it. */
313 llvm::Constant *create_binary_blob(const std::vector<char> &data, const std::string &name, bool constant = true);
314
315 /** Widen an llvm scalar into an llvm vector with the given number of lanes. */
316 llvm::Value *create_broadcast(llvm::Value *, int lanes);
317
318 /** Generate a pointer into a named buffer at a given index, of a
319 * given type. The index counts according to the scalar type of
320 * the type passed in. */
321 // @{
322 llvm::Value *codegen_buffer_pointer(const std::string &buffer, Type type, llvm::Value *index);
323 llvm::Value *codegen_buffer_pointer(const std::string &buffer, Type type, Expr index);
324 llvm::Value *codegen_buffer_pointer(llvm::Value *base_address, Type type, Expr index);
325 llvm::Value *codegen_buffer_pointer(llvm::Value *base_address, Type type, llvm::Value *index);
326 // @}
327
328 /** Return type string for LLVM type using LLVM IR intrinsic type mangling.
329 * E.g. ".i32 or ".f32" for scalars, ".p0" for pointers,
330 * ".nxv4i32" for a scalable vector of four 32-bit integers,
331 * or ".v4f32" for a fixed vector of four 32-bit floats.
332 * The dot is included in the result.
333 */
334 std::string mangle_llvm_type(llvm::Type *type);
335
336 /** Turn a Halide Type into an llvm::Value representing a constant halide_type_t */
337 llvm::Value *make_halide_type_t(const Type &);
338
339 /** Mark a load or store with type-based-alias-analysis metadata
340 * so that llvm knows it can reorder loads and stores across
341 * different buffers */
342 void add_tbaa_metadata(llvm::Instruction *inst, std::string buffer, const Expr &index);
343
344 /** Get a unique name for the actual block of memory that an
345 * allocate node uses. Used so that alias analysis understands
346 * when multiple Allocate nodes shared the same memory. */
347 virtual std::string get_allocation_name(const std::string &n) {
348 return n;
349 }
350
351 /** Add the appropriate function attribute to tell LLVM that the function
352 * doesn't access memory. */
353 void function_does_not_access_memory(llvm::Function *fn);
354
355 using IRVisitor::visit;
356
357 /** Generate code for various IR nodes. These can be overridden by
358 * architecture-specific code to perform peephole
359 * optimizations. The result of each is stored in \ref value */
360 // @{
361 void visit(const IntImm *) override;
362 void visit(const UIntImm *) override;
363 void visit(const FloatImm *) override;
364 void visit(const StringImm *) override;
365 void visit(const Cast *) override;
366 void visit(const Reinterpret *) override;
367 void visit(const Variable *) override;
368 void visit(const Add *) override;
369 void visit(const Sub *) override;
370 void visit(const Mul *) override;
371 void visit(const Div *) override;
372 void visit(const Mod *) override;
373 void visit(const Min *) override;
374 void visit(const Max *) override;
375 void visit(const EQ *) override;
376 void visit(const NE *) override;
377 void visit(const LT *) override;
378 void visit(const LE *) override;
379 void visit(const GT *) override;
380 void visit(const GE *) override;
381 void visit(const And *) override;
382 void visit(const Or *) override;
383 void visit(const Not *) override;
384 void visit(const Select *) override;
385 void visit(const Load *) override;
386 void visit(const Ramp *) override;
387 void visit(const Broadcast *) override;
388 void visit(const Call *) override;
389 void visit(const Let *) override;
390 void visit(const LetStmt *) override;
391 void visit(const AssertStmt *) override;
392 void visit(const ProducerConsumer *) override;
393 void visit(const For *) override;
394 void visit(const Store *) override;
395 void visit(const Block *) override;
396 void visit(const IfThenElse *) override;
397 void visit(const Evaluate *) override;
398 void visit(const Shuffle *) override;
399 void visit(const VectorReduce *) override;
400 void visit(const Prefetch *) override;
401 void visit(const Atomic *) override;
402 // @}
403
404 /** Generate code for an allocate node. It has no default
405 * implementation - it must be handled in an architecture-specific
406 * way. */
407 void visit(const Allocate *) override = 0;
408
409 /** Generate code for a free node. It has no default
410 * implementation and must be handled in an architecture-specific
411 * way. */
412 void visit(const Free *) override = 0;
413
414 /** These IR nodes should have been removed during
415 * lowering. CodeGen_LLVM will error out if they are present */
416 // @{
417 void visit(const Provide *) override;
418 void visit(const Realize *) override;
419 // @}
420
421 /** Get the llvm type equivalent to the given halide type in the
422 * current context. */
423 virtual llvm::Type *llvm_type_of(const Type &) const;
424
425 /** Get the llvm type equivalent to a given halide type. If
426 * effective_vscale is nonzero and the type is a vector type with lanes
427 * a multiple of effective_vscale, a scalable vector type is generated
428 * with total lanes divided by effective_vscale. That is a scalable
429 * vector intended to be used with a fixed vscale of effective_vscale.
430 */
431 llvm::Type *llvm_type_of(llvm::LLVMContext *context, Halide::Type t,
432 int effective_vscale) const;
433
434 /** Perform an alloca at the function entrypoint. Will be cleaned
435 * on function exit. */
436 llvm::Value *create_alloca_at_entry(llvm::Type *type, int n,
437 bool zero_initialize = false,
438 const std::string &name = "");
439
440 /** A (very) conservative guess at the size of all alloca() storage requested
441 * (including alignment padding). It's currently meant only to be used as
442 * a very coarse way to ensure there is enough stack space when testing
443 * on the WebAssembly backend.
444 *
445 * It is *not* meant to be a useful proxy for "stack space needed", for a
446 * number of reasons:
447 * - allocas with non-overlapping lifetimes will share space
448 * - on some backends, LLVM may promote register-sized allocas into registers
449 * - while this accounts for alloca() calls we know about, it doesn't attempt
450 * to account for stack spills, function call overhead, etc.
451 */
452 size_t requested_alloca_total = 0;
453
454 /** The user_context argument. May be a constant null if the
455 * function is being compiled without a user context. */
456 llvm::Value *get_user_context() const;
457
458 /** Implementation of the intrinsic call to
459 * interleave_vectors. This implementation allows for interleaving
460 * an arbitrary number of vectors.*/
461 virtual llvm::Value *interleave_vectors(const std::vector<llvm::Value *> &);
462
463 /** Description of an intrinsic function overload. Overloads are resolved
464 * using both argument and return types. The scalar types of the arguments
465 * and return type must match exactly for an overload resolution to succeed. */
466 struct Intrinsic {
467 Type result_type;
468 std::vector<Type> arg_types;
469 llvm::Function *impl;
470
471 Intrinsic(Type result_type, std::vector<Type> arg_types, llvm::Function *impl)
472 : result_type(result_type), arg_types(std::move(arg_types)), impl(impl) {
473 }
474 };
475 /** Mapping of intrinsic functions to the various overloads implementing it. */
476 std::map<std::string, std::vector<Intrinsic>> intrinsics;
477
478 /** Get an LLVM intrinsic declaration. If it doesn't exist, it will be created. */
479 llvm::Function *get_llvm_intrin(const Type &ret_type, const std::string &name, const std::vector<Type> &arg_types, bool scalars_are_vectors = false);
480 llvm::Function *get_llvm_intrin(llvm::Type *ret_type, const std::string &name, const std::vector<llvm::Type *> &arg_types);
481 /** Declare an intrinsic function that participates in overload resolution. */
482 llvm::Function *declare_intrin_overload(const std::string &name, const Type &ret_type, const std::string &impl_name, std::vector<Type> arg_types, bool scalars_are_vectors = false);
483 void declare_intrin_overload(const std::string &name, const Type &ret_type, llvm::Function *impl, std::vector<Type> arg_types);
484 /** Call an overloaded intrinsic function. Returns nullptr if no suitable overload is found. */
485 llvm::Value *call_overloaded_intrin(const Type &result_type, const std::string &name, const std::vector<Expr> &args);
486
487 /** Generate a call to a vector intrinsic or runtime inlined
488 * function. The arguments are sliced up into vectors of the width
489 * given by 'intrin_lanes', the intrinsic is called on each
490 * piece, then the results (if any) are concatenated back together
491 * into the original type 't'. For the version that takes an
492 * llvm::Type *, the type may be void, so the vector width of the
493 * arguments must be specified explicitly as
494 * 'called_lanes'. */
495 // @{
496 llvm::Value *call_intrin(const Type &t, int intrin_lanes,
497 const std::string &name, std::vector<Expr>);
498 llvm::Value *call_intrin(const Type &t, int intrin_lanes,
499 llvm::Function *intrin, std::vector<Expr>);
500 llvm::Value *call_intrin(const llvm::Type *t, int intrin_lanes,
501 const std::string &name, std::vector<llvm::Value *>,
502 bool scalable_vector_result = false, bool is_reduction = false);
503 llvm::Value *call_intrin(const llvm::Type *t, int intrin_lanes,
504 llvm::Function *intrin, std::vector<llvm::Value *>,
505 bool is_reduction = false);
506 // @}
507
508 /** Take a slice of lanes out of an llvm vector. Pads with undefs
509 * if you ask for more lanes than the vector has. */
510 virtual llvm::Value *slice_vector(llvm::Value *vec, int start, int extent);
511
512 /** Concatenate a bunch of llvm vectors. Must be of the same type. */
513 virtual llvm::Value *concat_vectors(const std::vector<llvm::Value *> &);
514
515 /** Reverse elements in a vector */
516 llvm::Value *reverse_vector(llvm::Value *vec);
517
518 /** Create an LLVM shuffle vectors instruction. Takes a combination of
519 * fixed or scalable vectors as input, so long as the effective lengths match,
520 * but always returns a fixed vector. */
521 virtual llvm::Value *shuffle_vectors(llvm::Value *a, llvm::Value *b,
522 const std::vector<int> &indices);
523 /** Shorthand for shuffling a single vector. */
524 llvm::Value *shuffle_vectors(llvm::Value *v, const std::vector<int> &indices);
525
526 bool is_power_of_two(int x) const;
527
528 bool is_scalable_vector(llvm::Value *v) const;
529
530 /** Go looking for a vector version of a runtime function. Will
531 * return the best match. Matches in the following order:
532 *
533 * 1) The requested vector width.
534 *
535 * 2) The width which is the smallest power of two
536 * greater than or equal to the vector width.
537 *
538 * 3) All the factors of 2) greater than one, in decreasing order.
539 *
540 * 4) The smallest power of two not yet tried.
541 *
542 * So for a 5-wide vector, it tries: 5, 8, 4, 2, 16.
543 *
544 * If there's no match, returns (nullptr, 0).
545 */
546 std::pair<llvm::Function *, int> find_vector_runtime_function(const std::string &name, int lanes);
547
548 virtual bool supports_atomic_add(const Type &t) const;
549
550 /** Compile a horizontal reduction that starts with an explicit
551 * initial value. There are lots of complex ways to peephole
552 * optimize this pattern, especially with the proliferation of
553 * dot-product instructions, and they can usefully share logic
554 * across backends. */
555 virtual void codegen_vector_reduce(const VectorReduce *op, const Expr &init);
556
557 /** Are we inside an atomic node that uses mutex locks?
558 This is used for detecting deadlocks from nested atomics & illegal vectorization. */
559 bool inside_atomic_mutex_node = false;
560
561 /** Emit atomic store instructions? */
562 bool emit_atomic_stores = false;
563
564 /** Can we call this operation with float16 type?
565 This is used to avoid "emulated" equivalent code-gen in case target has FP16 feature **/
566 virtual bool supports_call_as_float16(const Call *op) const;
567
568 /** call_intrin does far too much to be useful and generally breaks things
569 * when one has carefully set things up for a specific architecture. This
570 * just does the bare minimum. call_intrin should be refactored and could
571 * call this, possibly with renaming of the methods. */
572 llvm::Value *simple_call_intrin(const std::string &intrin,
573 const std::vector<llvm::Value *> &args,
574 llvm::Type *result_type);
575
576 /** Ensure that a vector value is either fixed or vscale depending to match desired_type.
577 */
578 llvm::Value *normalize_fixed_scalable_vector_type(llvm::Type *desired_type, llvm::Value *result);
579
580 /** Convert between two LLVM vectors of potentially different scalable/fixed and size.
581 * Used to handle converting to/from fixed vectors that are smaller than the minimum
582 * size scalable vector. */
583 llvm::Value *convert_fixed_or_scalable_vector_type(llvm::Value *arg,
584 llvm::Type *desired_type);
585
586 /** Convert an LLVM fixed vector value to the corresponding vscale vector value. */
587 llvm::Value *fixed_to_scalable_vector_type(llvm::Value *fixed);
588
589 /** Convert an LLVM vscale vector value to the corresponding fixed vector value. */
590 llvm::Value *scalable_to_fixed_vector_type(llvm::Value *scalable);
591
592 /** Get number of vector elements, taking into account scalable vectors. Returns 1 for scalars. */
593 int get_vector_num_elements(const llvm::Type *t);
594
595 /** Interface to abstract vector code generation as LLVM is now
596 * providing multiple options to express even simple vector
597 * operations. Specifically traditional fixed length vectors, vscale
598 * based variable length vectors, and the vector predicate based approach
599 * where an explict length is passed with each instruction.
600 */
601 // @{
602 enum class VectorTypeConstraint {
603 None, /// Use default for current target.
604 Fixed, /// Force use of fixed size vectors.
605 VScale, /// For use of scalable vectors.
606 };
607 llvm::Type *get_vector_type(llvm::Type *, int n,
608 VectorTypeConstraint type_constraint = VectorTypeConstraint::None) const;
609 // @}
610
611 llvm::Constant *get_splat(int lanes, llvm::Constant *value,
612 VectorTypeConstraint type_constraint = VectorTypeConstraint::None) const;
613
614 /** Make sure a value type has the same scalable/fixed vector type as a guide. */
615 // @{
616 llvm::Value *match_vector_type_scalable(llvm::Value *value, VectorTypeConstraint constraint);
617 llvm::Value *match_vector_type_scalable(llvm::Value *value, llvm::Type *guide);
618 llvm::Value *match_vector_type_scalable(llvm::Value *value, llvm::Value *guide);
619 // @}
620
621 /** Support for generating LLVM vector predication intrinsics
622 * ("@llvm.vp.*" and "@llvm.experimental.vp.*")
623 */
624 // @{
625 /** Struct to hold descriptor for an argument to a vector
626 * predicated intrinsic. This includes the value, whether the
627 * type of the argument should be mangled into the intrisic name
628 * and if so, where, and the alignment for pointer arguments. */
629 struct VPArg {
630 llvm::Value *value;
631 // If provided, put argument's type into the intrinsic name via LLVM IR type mangling.
632 std::optional<size_t> mangle_index;
633 int alignment;
634 VPArg(llvm::Value *value, std::optional<size_t> mangle_index = std::nullopt, int32_t alignment = 0)
635 : value(value), mangle_index(mangle_index), alignment(alignment) {
636 }
637 };
638
639 /** Type indicating an intrinsic does not take a mask. */
640 struct NoMask {
641 };
642
643 /** Type indicating mask to use is all true -- all lanes enabled. */
644 struct AllEnabledMask {
645 };
646
647 /** Predication mask using the above two types for special cases
648 * and an llvm::Value for the general one. */
649 using MaskVariant = std::variant<NoMask, AllEnabledMask, llvm::Value *>;
650
651 /** Generate a vector predicated comparison intrinsic call if
652 * use_llvm_vp_intrinsics is true and result_type is a vector
653 * type. If generated, assigns result of vp intrinsic to value and
654 * returns true if it an instuction is generated, otherwise
655 * returns false. */
656 bool try_vector_predication_comparison(const std::string &name, const Type &result_type,
657 MaskVariant mask, llvm::Value *a, llvm::Value *b,
658 const char *cmp_op);
659
660 struct VPResultType {
661 llvm::Type *type;
662 std::optional<size_t> mangle_index;
663 VPResultType(llvm::Type *type, std::optional<size_t> mangle_index = std::nullopt)
664 : type(type), mangle_index(mangle_index) {
665 }
666 };
667
668 /** Generate an intrisic call if use_llvm_vp_intrinsics is true
669 * and length is greater than 1. If generated, assigns result
670 * of vp intrinsic to value and returns true if it an instuction
671 * is generated, otherwise returns false. */
672 bool try_vector_predication_intrinsic(const std::string &name, VPResultType result_type,
673 int32_t length, MaskVariant mask, std::vector<VPArg> args);
674
675 /** Controls use of vector predicated intrinsics for vector operations.
676 * Will be set by certain backends (e.g. RISC V) to control codegen. */
677 bool use_llvm_vp_intrinsics = false;
678 // @}
679
680 /** Generate a basic dense vector load, with an optional predicate and
681 * control over whether or not we should slice the load into native
682 * vectors. Used by CodeGen_ARM to help with vld2/3/4 emission. */
683 llvm::Value *codegen_dense_vector_load(const Load *load, llvm::Value *vpred = nullptr, bool slice_to_native = true);
684
685 /** Warning messages which we want to avoid displaying number of times */
686 enum class WarningKind {
687 EmulatedFloat16,
688 };
689 std::map<WarningKind, std::string> onetime_warnings;
690
691 private:
692 /** All the values in scope at the current code location during
693 * codegen. Use sym_push and sym_pop to access. */
694 Scope<llvm::Value *> symbol_table;
695
696 /** String constants already emitted to the module. Tracked to
697 * prevent emitting the same string many times. */
698 std::map<std::string, llvm::Constant *> string_constants;
699
700 /** A basic block to branch to on error that triggers all
701 * destructors. As destructors are registered, code gets added
702 * to this block. */
703 llvm::BasicBlock *destructor_block = nullptr;
704
705 /** Turn off all unsafe math flags in scopes while this is set. */
706 bool strict_float;
707
708 /** Use the LLVM large code model when this is set. */
709 bool llvm_large_code_model;
710
711 /** Cache the result of target_vscale from architecture specific implementation
712 * as this is used on every Halide to LLVM type conversion.
713 */
714 int effective_vscale = 0;
715
716 /** Assign a unique ID to each producer-consumer and for-loop node. The IDs
717 * are printed as comments in assembly and used to link visualizations with
718 * the generated assembly code within `StmtToViz`
719 */
720 int producer_consumer_id = 0;
721 int for_loop_id = 0;
722
723 /** Embed an instance of halide_filter_metadata_t in the code, using
724 * the given name (by convention, this should be ${FUNCTIONNAME}_metadata)
725 * as extern "C" linkage. Note that the return value is a function-returning-
726 * pointer-to-constant-data.
727 */
728 llvm::Function *embed_metadata_getter(const std::string &metadata_getter_name,
729 const std::string &function_name, const std::vector<LoweredArgument> &args,
730 const MetadataNameMap &metadata_name_map);
731
732 /** Embed a constant expression as a global variable. */
733 llvm::Constant *embed_constant_expr(Expr e, llvm::Type *t);
734 llvm::Constant *embed_constant_scalar_value_t(const Expr &e);
735
736 llvm::Function *add_argv_wrapper(llvm::Function *fn, const std::string &name,
737 bool result_in_argv, std::vector<bool> &arg_is_buffer);
738
739 llvm::Value *codegen_vector_load(const Type &type, const std::string &name, const Expr &base,
740 const Buffer<> &image, const Parameter &param, const ModulusRemainder &alignment,
741 llvm::Value *vpred = nullptr, bool slice_to_native = true, llvm::Value *stride = nullptr);
742
743 virtual void codegen_predicated_load(const Load *op);
744 virtual void codegen_predicated_store(const Store *op);
745
746 void codegen_atomic_rmw(const Store *op);
747
748 void init_codegen(const std::string &name);
749 std::unique_ptr<llvm::Module> finish_codegen();
750
751 /** A helper routine for generating folded vector reductions. */
752 template<typename Op>
753 bool try_to_fold_vector_reduce(const Expr &a, Expr b);
754
755 /** Records the StructType for pointer values returned from
756 * make_struct intrinsic. Required for opaque pointer support.
757 * This map should never grow without bound as each entry
758 * represents a unique struct type created by a closure or similar.
759 */
760 std::map<llvm::Value *, llvm::Type *> struct_type_recovery;
761};
762
763} // namespace Internal
764
765/** Given a Halide module, generate an llvm::Module. */
766std::unique_ptr<llvm::Module> codegen_llvm(const Module &module,
767 llvm::LLVMContext &context);
768
769} // namespace Halide
770
771#endif
IRVisitor.h
Defines the base class for things that recursively walk over the IR.
Module.h
Defines Module, an IR container that fully describes a Halide program.
Scope.h
Defines the Scope class, which is used for keeping track of names in a scope while traversing IR.
Target.h
Defines the structure that describes a Halide target.
Halide::Buffer<>
Halide::Internal::CodeGen_LLVM
A code generator abstract base class.
Definition: CodeGen_LLVM.h:60
Halide::Internal::CodeGen_LLVM::get_splat
llvm::Constant * get_splat(int lanes, llvm::Constant *value, VectorTypeConstraint type_constraint=VectorTypeConstraint::None) const
Halide::Internal::CodeGen_LLVM::value
llvm::Value * value
Definition: CodeGen_LLVM.h:169
Halide::Internal::CodeGen_LLVM::visit
void visit(const Let *) override
Halide::Internal::CodeGen_LLVM::find_vector_runtime_function
std::pair< llvm::Function *, int > find_vector_runtime_function(const std::string &name, int lanes)
Go looking for a vector version of a runtime function.
Halide::Internal::CodeGen_LLVM::metadata_t_type
llvm::StructType * metadata_t_type
Definition: CodeGen_LLVM.h:237
Halide::Internal::CodeGen_LLVM::end_func
virtual void end_func(const std::vector< LoweredArgument > &args)
Halide::Internal::CodeGen_LLVM::argument_t_type
llvm::StructType * argument_t_type
Definition: CodeGen_LLVM.h:238
Halide::Internal::CodeGen_LLVM::create_binary_blob
llvm::Constant * create_binary_blob(const std::vector< char > &data, const std::string &name, bool constant=true)
Put a binary blob in the module as a global variable and return a pointer to it.
Halide::Internal::CodeGen_LLVM::visit
void visit(const Select *) override
Halide::Internal::CodeGen_LLVM::maximum_vector_bits
virtual int maximum_vector_bits() const
Used to decide whether to break a vector up into multiple smaller operations.
Definition: CodeGen_LLVM.h:142
Halide::Internal::CodeGen_LLVM::visit
void visit(const Or *) override
Halide::Internal::CodeGen_LLVM::f64_t
llvm::Type * f64_t
Definition: CodeGen_LLVM.h:232
Halide::Internal::CodeGen_LLVM::visit
void visit(const AssertStmt *) override
Halide::Internal::CodeGen_LLVM::emit_atomic_stores
bool emit_atomic_stores
Emit atomic store instructions?
Definition: CodeGen_LLVM.h:562
Halide::Internal::CodeGen_LLVM::sym_pop
void sym_pop(const std::string &name)
Remove an entry for the symbol table, revealing any previous entries with the same name.
Halide::Internal::CodeGen_LLVM::init_context
virtual void init_context()
Grab all the context specific internal state.
Halide::Internal::CodeGen_LLVM::target_vscale
virtual int target_vscale() const
For architectures that have vscale vectors, return the constant vscale to use.
Definition: CodeGen_LLVM.h:149
Halide::Internal::CodeGen_LLVM::device_interface_t_type
llvm::StructType * device_interface_t_type
Definition: CodeGen_LLVM.h:240
Halide::Internal::CodeGen_LLVM::visit
void visit(const Mul *) override
Halide::Internal::CodeGen_LLVM::semaphore_t_type
llvm::StructType * semaphore_t_type
Definition: CodeGen_LLVM.h:242
Halide::Internal::CodeGen_LLVM::supports_atomic_add
virtual bool supports_atomic_add(const Type &t) const
Halide::Internal::CodeGen_LLVM::llvm_type_of
virtual llvm::Type * llvm_type_of(const Type &) const
Get the llvm type equivalent to the given halide type in the current context.
Halide::Internal::CodeGen_LLVM::i64_t
llvm::Type * i64_t
Definition: CodeGen_LLVM.h:232
Halide::Internal::CodeGen_LLVM::current_function_args
std::vector< LoweredArgument > current_function_args
Definition: CodeGen_LLVM.h:173
Halide::Internal::CodeGen_LLVM::get_target
const Target & get_target() const
The target we're generating code for.
Definition: CodeGen_LLVM.h:69
Halide::Internal::CodeGen_LLVM::return_with_error_code
void return_with_error_code(llvm::Value *error_code)
Return the the pipeline with the given error code.
Halide::Internal::CodeGen_LLVM::wild_f64_
Expr wild_f64_
Definition: CodeGen_LLVM.h:256
Halide::Internal::CodeGen_LLVM::call_intrin
llvm::Value * call_intrin(const llvm::Type *t, int intrin_lanes, const std::string &name, std::vector< llvm::Value * >, bool scalable_vector_result=false, bool is_reduction=false)
Halide::Internal::CodeGen_LLVM::compile
virtual std::unique_ptr< llvm::Module > compile(const Module &module)
Takes a halide Module and compiles it to an llvm Module.
Halide::Internal::CodeGen_LLVM::codegen_buffer_pointer
llvm::Value * codegen_buffer_pointer(const std::string &buffer, Type type, Expr index)
Halide::Internal::CodeGen_LLVM::reverse_vector
llvm::Value * reverse_vector(llvm::Value *vec)
Reverse elements in a vector.
Halide::Internal::CodeGen_LLVM::get_llvm_intrin
llvm::Function * get_llvm_intrin(llvm::Type *ret_type, const std::string &name, const std::vector< llvm::Type * > &arg_types)
Halide::Internal::CodeGen_LLVM::mcpu_tune
virtual std::string mcpu_tune() const =0
Halide::Internal::CodeGen_LLVM::ptr_t
llvm::PointerType * ptr_t
Definition: CodeGen_LLVM.h:233
Halide::Internal::CodeGen_LLVM::try_vector_predication_intrinsic
bool try_vector_predication_intrinsic(const std::string &name, VPResultType result_type, int32_t length, MaskVariant mask, std::vector< VPArg > args)
Generate an intrisic call if use_llvm_vp_intrinsics is true and length is greater than 1.
Halide::Internal::CodeGen_LLVM::trigger_destructor
void trigger_destructor(llvm::Function *destructor_fn, llvm::Value *stack_slot)
Call a destructor early.
Halide::Internal::CodeGen_LLVM::wild_i8x_
Expr wild_i8x_
Definition: CodeGen_LLVM.h:249
Halide::Internal::CodeGen_LLVM::is_scalable_vector
bool is_scalable_vector(llvm::Value *v) const
Halide::Internal::CodeGen_LLVM::visit
void visit(const Reinterpret *) override
Halide::Internal::CodeGen_LLVM::visit
void visit(const Min *) override
Halide::Internal::CodeGen_LLVM::visit
void visit(const For *) override
Halide::Internal::CodeGen_LLVM::codegen_buffer_pointer
llvm::Value * codegen_buffer_pointer(llvm::Value *base_address, Type type, llvm::Value *index)
Halide::Internal::CodeGen_LLVM::visit
void visit(const Not *) override
Halide::Internal::CodeGen_LLVM::visit
void visit(const Sub *) override
Halide::Internal::CodeGen_LLVM::sym_push
void sym_push(const std::string &name, llvm::Value *value)
Add an entry to the symbol table, hiding previous entries with the same name.
Halide::Internal::CodeGen_LLVM::type_t_type
llvm::StructType * type_t_type
Definition: CodeGen_LLVM.h:235
Halide::Internal::CodeGen_LLVM::VectorTypeConstraint
VectorTypeConstraint
Interface to abstract vector code generation as LLVM is now providing multiple options to express eve...
Definition: CodeGen_LLVM.h:602
Halide::Internal::CodeGen_LLVM::VectorTypeConstraint::None
@ None
Halide::Internal::CodeGen_LLVM::DestructorType
DestructorType
Some destructors should always be called.
Definition: CodeGen_LLVM.h:271
Halide::Internal::CodeGen_LLVM::Always
@ Always
Definition: CodeGen_LLVM.h:271
Halide::Internal::CodeGen_LLVM::OnError
@ OnError
Definition: CodeGen_LLVM.h:272
Halide::Internal::CodeGen_LLVM::OnSuccess
@ OnSuccess
Definition: CodeGen_LLVM.h:273
Halide::Internal::CodeGen_LLVM::target
Halide::Target target
The target we're generating code for.
Definition: CodeGen_LLVM.h:197
Halide::Internal::CodeGen_LLVM::begin_func
virtual void begin_func(LinkageType linkage, const std::string &simple_name, const std::string &extern_name, const std::vector< LoweredArgument > &args)
Helper functions for compiling Halide functions to llvm functions.
Halide::Internal::CodeGen_LLVM::try_vector_predication_comparison
bool try_vector_predication_comparison(const std::string &name, const Type &result_type, MaskVariant mask, llvm::Value *a, llvm::Value *b, const char *cmp_op)
Generate a vector predicated comparison intrinsic call if use_llvm_vp_intrinsics is true and result_t...
Halide::Internal::CodeGen_LLVM::fast_fp_math_md
llvm::MDNode * fast_fp_math_md
Definition: CodeGen_LLVM.h:171
Halide::Internal::CodeGen_LLVM::scalar_value_t_type
llvm::StructType * scalar_value_t_type
Definition: CodeGen_LLVM.h:239
Halide::Internal::CodeGen_LLVM::builder
std::unique_ptr< llvm::IRBuilder< llvm::ConstantFolder, llvm::IRBuilderDefaultInserter > > builder
Definition: CodeGen_LLVM.h:168
Halide::Internal::CodeGen_LLVM::i16_t
llvm::Type * i16_t
Definition: CodeGen_LLVM.h:232
Halide::Internal::CodeGen_LLVM::use_llvm_vp_intrinsics
bool use_llvm_vp_intrinsics
Controls use of vector predicated intrinsics for vector operations.
Definition: CodeGen_LLVM.h:677
Halide::Internal::CodeGen_LLVM::create_broadcast
llvm::Value * create_broadcast(llvm::Value *, int lanes)
Widen an llvm scalar into an llvm vector with the given number of lanes.
Halide::Internal::CodeGen_LLVM::module
std::unique_ptr< llvm::Module > module
Definition: CodeGen_LLVM.h:165
Halide::Internal::CodeGen_LLVM::wild_i32_
Expr wild_i32_
Definition: CodeGen_LLVM.h:255
Halide::Internal::CodeGen_LLVM::mcpu_target
virtual std::string mcpu_target() const =0
What should be passed as -mcpu (warning: implies attrs!), -mattrs, and related for compilation.
Halide::Internal::CodeGen_LLVM::visit
void visit(const Max *) override
Halide::Internal::CodeGen_LLVM::codegen
void codegen(const Stmt &)
Emit code that runs a statement.
Halide::Internal::CodeGen_LLVM::compile_buffer
virtual void compile_buffer(const Buffer<> &buffer)
Halide::Internal::CodeGen_LLVM::promote_indices
virtual bool promote_indices() const
Should indexing math be promoted to 64-bit on platforms with 64-bit pointers?
Definition: CodeGen_LLVM.h:133
Halide::Internal::CodeGen_LLVM::function
llvm::Function * function
Definition: CodeGen_LLVM.h:166
Halide::Internal::CodeGen_LLVM::visit
void visit(const GE *) override
Halide::Internal::CodeGen_LLVM::wild_u1_
Expr wild_u1_
Definition: CodeGen_LLVM.h:254
Halide::Internal::CodeGen_LLVM::wild_i64_
Expr wild_i64_
Definition: CodeGen_LLVM.h:255
Halide::Internal::CodeGen_LLVM::visit
void visit(const Variable *) override
Halide::Internal::CodeGen_LLVM::declare_intrin_overload
void declare_intrin_overload(const std::string &name, const Type &ret_type, llvm::Function *impl, std::vector< Type > arg_types)
Halide::Internal::CodeGen_LLVM::codegen_dense_vector_load
llvm::Value * codegen_dense_vector_load(const Load *load, llvm::Value *vpred=nullptr, bool slice_to_native=true)
Generate a basic dense vector load, with an optional predicate and control over whether or not we sho...
Halide::Internal::CodeGen_LLVM::wild_f64x_
Expr wild_f64x_
Definition: CodeGen_LLVM.h:251
Halide::Internal::CodeGen_LLVM::wild_f32x_
Expr wild_f32x_
Definition: CodeGen_LLVM.h:251
Halide::Internal::CodeGen_LLVM::visit
void visit(const Atomic *) override
Halide::Internal::CodeGen_LLVM::wild_u16_
Expr wild_u16_
Definition: CodeGen_LLVM.h:254
Halide::Internal::CodeGen_LLVM::create_assertion
void create_assertion(llvm::Value *condition, const Expr &message, llvm::Value *error_code=nullptr)
Codegen an assertion.
Halide::Internal::CodeGen_LLVM::upgrade_type_for_storage
virtual Type upgrade_type_for_storage(const Type &) const
Return the type that a given Halide type should be stored/loaded from memory as.
Halide::Internal::CodeGen_LLVM::visit
void visit(const Shuffle *) override
Halide::Internal::CodeGen_LLVM::very_likely_branch
llvm::MDNode * very_likely_branch
Definition: CodeGen_LLVM.h:170
Halide::Internal::CodeGen_LLVM::visit
void visit(const Allocate *) override=0
Generate code for an allocate node.
Halide::Internal::CodeGen_LLVM::visit
void visit(const Prefetch *) override
Halide::Internal::CodeGen_LLVM::visit
void visit(const Provide *) override
These IR nodes should have been removed during lowering.
Halide::Internal::CodeGen_LLVM::upgrade_type_for_arithmetic
virtual Type upgrade_type_for_arithmetic(const Type &) const
Return the type in which arithmetic should be done for the given storage type.
Halide::Internal::CodeGen_LLVM::any_strict_float
bool any_strict_float
Definition: CodeGen_LLVM.h:176
Halide::Internal::CodeGen_LLVM::normalize_fixed_scalable_vector_type
llvm::Value * normalize_fixed_scalable_vector_type(llvm::Type *desired_type, llvm::Value *result)
Ensure that a vector value is either fixed or vscale depending to match desired_type.
Halide::Internal::CodeGen_LLVM::visit
void visit(const Div *) override
Halide::Internal::CodeGen_LLVM::halide_buffer_t_type
llvm::StructType * halide_buffer_t_type
Definition: CodeGen_LLVM.h:234
Halide::Internal::CodeGen_LLVM::WarningKind
WarningKind
Warning messages which we want to avoid displaying number of times.
Definition: CodeGen_LLVM.h:686
Halide::Internal::CodeGen_LLVM::visit
void visit(const EQ *) override
Halide::Internal::CodeGen_LLVM::native_vector_bits
virtual int native_vector_bits() const =0
What's the natural vector bit-width to use for loads, stores, etc.
Halide::Internal::CodeGen_LLVM::wild_u8x_
Expr wild_u8x_
Definition: CodeGen_LLVM.h:249
Halide::Internal::CodeGen_LLVM::match_vector_type_scalable
llvm::Value * match_vector_type_scalable(llvm::Value *value, llvm::Type *guide)
Halide::Internal::CodeGen_LLVM::dimension_t_type
llvm::StructType * dimension_t_type
Definition: CodeGen_LLVM.h:236
Halide::Internal::CodeGen_LLVM::onetime_warnings
std::map< WarningKind, std::string > onetime_warnings
Definition: CodeGen_LLVM.h:689
Halide::Internal::CodeGen_LLVM::i8_t
llvm::Type * i8_t
Definition: CodeGen_LLVM.h:232
Halide::Internal::CodeGen_LLVM::compile_trampolines
static std::unique_ptr< llvm::Module > compile_trampolines(const Target &target, llvm::LLVMContext &context, const std::string &suffix, const std::vector< std::pair< std::string, ExternSignature > > &externs)
Halide::Internal::CodeGen_LLVM::compile_func
virtual void compile_func(const LoweredFunc &func, const std::string &simple_name, const std::string &extern_name)
Compile a specific halide declaration into the llvm Module.
Halide::Internal::CodeGen_LLVM::signature_to_type
llvm::FunctionType * signature_to_type(const ExternSignature &signature)
Given a Halide ExternSignature, return the equivalent llvm::FunctionType.
Halide::Internal::CodeGen_LLVM::create_string_constant
llvm::Constant * create_string_constant(const std::string &str)
Put a string constant in the module as a global variable and return a pointer to it.
Halide::Internal::CodeGen_LLVM::i1_t
llvm::Type * i1_t
Definition: CodeGen_LLVM.h:232
Halide::Internal::CodeGen_LLVM::concat_vectors
virtual llvm::Value * concat_vectors(const std::vector< llvm::Value * > &)
Concatenate a bunch of llvm vectors.
Halide::Internal::CodeGen_LLVM::context
llvm::LLVMContext * context
Definition: CodeGen_LLVM.h:167
Halide::Internal::CodeGen_LLVM::set_strict_fp_math
void set_strict_fp_math()
Change floating-point math op emission to use strict flags.
Halide::Internal::CodeGen_LLVM::wild_i16x_
Expr wild_i16x_
Definition: CodeGen_LLVM.h:249
Halide::Internal::CodeGen_LLVM::wild_u64x_
Expr wild_u64x_
Definition: CodeGen_LLVM.h:250
Halide::Internal::CodeGen_LLVM::create_alloca_at_entry
llvm::Value * create_alloca_at_entry(llvm::Type *type, int n, bool zero_initialize=false, const std::string &name="")
Perform an alloca at the function entrypoint.
Halide::Internal::CodeGen_LLVM::f32_t
llvm::Type * f32_t
Definition: CodeGen_LLVM.h:232
Halide::Internal::CodeGen_LLVM::match_vector_type_scalable
llvm::Value * match_vector_type_scalable(llvm::Value *value, llvm::Value *guide)
Halide::Internal::CodeGen_LLVM::wild_u32_
Expr wild_u32_
Definition: CodeGen_LLVM.h:255
Halide::Internal::CodeGen_LLVM::get_allocation_name
virtual std::string get_allocation_name(const std::string &n)
Get a unique name for the actual block of memory that an allocate node uses.
Definition: CodeGen_LLVM.h:347
Halide::Internal::CodeGen_LLVM::use_soft_float_abi
virtual bool use_soft_float_abi() const =0
Halide::Internal::CodeGen_LLVM::mabi
virtual std::string mabi() const
Halide::Internal::CodeGen_LLVM::visit
void visit(const Evaluate *) override
Halide::Internal::CodeGen_LLVM::intrinsics
std::map< std::string, std::vector< Intrinsic > > intrinsics
Mapping of intrinsic functions to the various overloads implementing it.
Definition: CodeGen_LLVM.h:476
Halide::Internal::CodeGen_LLVM::mangle_llvm_type
std::string mangle_llvm_type(llvm::Type *type)
Return type string for LLVM type using LLVM IR intrinsic type mangling.
Halide::Internal::CodeGen_LLVM::visit
void visit(const LE *) override
Halide::Internal::CodeGen_LLVM::codegen_buffer_pointer
llvm::Value * codegen_buffer_pointer(const std::string &buffer, Type type, llvm::Value *index)
Generate a pointer into a named buffer at a given index, of a given type.
Halide::Internal::CodeGen_LLVM::visit
void visit(const NE *) override
Halide::Internal::CodeGen_LLVM::add_tbaa_metadata
void add_tbaa_metadata(llvm::Instruction *inst, std::string buffer, const Expr &index)
Mark a load or store with type-based-alias-analysis metadata so that llvm knows it can reorder loads ...
Halide::Internal::CodeGen_LLVM::visit
void visit(const And *) override
Halide::Internal::CodeGen_LLVM::use_pic
virtual bool use_pic() const
Halide::Internal::CodeGen_LLVM::scalarize
void scalarize(const Expr &)
Codegen a vector Expr by codegenning each lane and combining.
Halide::Internal::CodeGen_LLVM::visit
void visit(const StringImm *) override
Halide::Internal::CodeGen_LLVM::in_strict_float
bool in_strict_float
Definition: CodeGen_LLVM.h:175
Halide::Internal::CodeGen_LLVM::codegen_vector_reduce
virtual void codegen_vector_reduce(const VectorReduce *op, const Expr &init)
Compile a horizontal reduction that starts with an explicit initial value.
Halide::Internal::CodeGen_LLVM::codegen_asserts
void codegen_asserts(const std::vector< const AssertStmt * > &asserts)
Codegen a block of asserts with pure conditions.
Halide::Internal::CodeGen_LLVM::get_llvm_intrin
llvm::Function * get_llvm_intrin(const Type &ret_type, const std::string &name, const std::vector< Type > &arg_types, bool scalars_are_vectors=false)
Get an LLVM intrinsic declaration.
Halide::Internal::CodeGen_LLVM::get_requested_alloca_total
size_t get_requested_alloca_total() const
Definition: CodeGen_LLVM.h:85
Halide::Internal::CodeGen_LLVM::new_for_target
static std::unique_ptr< CodeGen_LLVM > new_for_target(const Target &target, llvm::LLVMContext &context)
Create an instance of CodeGen_LLVM suitable for the target.
Halide::Internal::CodeGen_LLVM::upgrade_type_for_argument_passing
virtual Type upgrade_type_for_argument_passing(const Type &) const
Return the type that a Halide type should be passed in and out of functions as.
Halide::Internal::CodeGen_LLVM::visit
void visit(const GT *) override
Halide::Internal::CodeGen_LLVM::fixed_to_scalable_vector_type
llvm::Value * fixed_to_scalable_vector_type(llvm::Value *fixed)
Convert an LLVM fixed vector value to the corresponding vscale vector value.
Halide::Internal::CodeGen_LLVM::visit
void visit(const Cast *) override
Halide::Internal::CodeGen_LLVM::CodeGen_LLVM
CodeGen_LLVM(const Target &t)
Halide::Internal::CodeGen_LLVM::visit
void visit(const Ramp *) override
Halide::Internal::CodeGen_LLVM::visit
void visit(const Broadcast *) override
Halide::Internal::CodeGen_LLVM::visit
void visit(const Mod *) override
Halide::Internal::CodeGen_LLVM::make_halide_type_t
llvm::Value * make_halide_type_t(const Type &)
Turn a Halide Type into an llvm::Value representing a constant halide_type_t.
Halide::Internal::CodeGen_LLVM::visit
void visit(const Call *) override
Halide::Internal::CodeGen_LLVM::initialize_llvm
static void initialize_llvm()
Initialize internal llvm state for the enabled targets.
Halide::Internal::CodeGen_LLVM::simple_call_intrin
llvm::Value * simple_call_intrin(const std::string &intrin, const std::vector< llvm::Value * > &args, llvm::Type *result_type)
call_intrin does far too much to be useful and generally breaks things when one has carefully set thi...
Halide::Internal::CodeGen_LLVM::set_context
void set_context(llvm::LLVMContext &context)
Tell the code generator which LLVM context to use.
Halide::Internal::CodeGen_LLVM::visit
void visit(const Store *) override
Halide::Internal::CodeGen_LLVM::get_vector_num_elements
int get_vector_num_elements(const llvm::Type *t)
Get number of vector elements, taking into account scalable vectors.
Halide::Internal::CodeGen_LLVM::call_intrin
llvm::Value * call_intrin(const Type &t, int intrin_lanes, llvm::Function *intrin, std::vector< Expr >)
Halide::Internal::CodeGen_LLVM::register_destructor
llvm::Value * register_destructor(llvm::Function *destructor_fn, llvm::Value *obj, DestructorType when)
Halide::Internal::CodeGen_LLVM::wild_u8_
Expr wild_u8_
Definition: CodeGen_LLVM.h:254
Halide::Internal::CodeGen_LLVM::wild_i8_
Expr wild_i8_
Definition: CodeGen_LLVM.h:254
Halide::Internal::CodeGen_LLVM::codegen
llvm::Value * codegen(const Expr &)
Emit code that evaluates an expression, and return the llvm representation of the result of the expre...
Halide::Internal::CodeGen_LLVM::wild_i16_
Expr wild_i16_
Definition: CodeGen_LLVM.h:254
Halide::Internal::CodeGen_LLVM::optimize_module
void optimize_module()
Run all of llvm's optimization passes on the module.
Halide::Internal::CodeGen_LLVM::codegen_buffer_pointer
llvm::Value * codegen_buffer_pointer(llvm::Value *base_address, Type type, Expr index)
Halide::Internal::CodeGen_LLVM::set_fast_fp_math
void set_fast_fp_math()
Change floating-point math op emission to use fast flags.
Halide::Internal::CodeGen_LLVM::sym_get
llvm::Value * sym_get(const std::string &name, bool must_succeed=true) const
Fetch an entry from the symbol table.
Halide::Internal::CodeGen_LLVM::visit
void visit(const ProducerConsumer *) override
Halide::Internal::CodeGen_LLVM::shuffle_vectors
virtual llvm::Value * shuffle_vectors(llvm::Value *a, llvm::Value *b, const std::vector< int > &indices)
Create an LLVM shuffle vectors instruction.
Halide::Internal::CodeGen_LLVM::visit
void visit(const LT *) override
Halide::Internal::CodeGen_LLVM::visit
void visit(const Load *) override
Halide::Internal::CodeGen_LLVM::MaskVariant
std::variant< NoMask, AllEnabledMask, llvm::Value * > MaskVariant
Predication mask using the above two types for special cases and an llvm::Value for the general one.
Definition: CodeGen_LLVM.h:649
Halide::Internal::CodeGen_LLVM::is_power_of_two
bool is_power_of_two(int x) const
Halide::Internal::CodeGen_LLVM::convert_fixed_or_scalable_vector_type
llvm::Value * convert_fixed_or_scalable_vector_type(llvm::Value *arg, llvm::Type *desired_type)
Convert between two LLVM vectors of potentially different scalable/fixed and size.
Halide::Internal::CodeGen_LLVM::slice_vector
virtual llvm::Value * slice_vector(llvm::Value *vec, int start, int extent)
Take a slice of lanes out of an llvm vector.
Halide::Internal::CodeGen_LLVM::declare_intrin_overload
llvm::Function * declare_intrin_overload(const std::string &name, const Type &ret_type, const std::string &impl_name, std::vector< Type > arg_types, bool scalars_are_vectors=false)
Declare an intrinsic function that participates in overload resolution.
Halide::Internal::CodeGen_LLVM::inside_atomic_mutex_node
bool inside_atomic_mutex_node
Are we inside an atomic node that uses mutex locks? This is used for detecting deadlocks from nested ...
Definition: CodeGen_LLVM.h:559
Halide::Internal::CodeGen_LLVM::visit
void visit(const FloatImm *) override
Halide::Internal::CodeGen_LLVM::visit
void visit(const IntImm *) override
Generate code for various IR nodes.
Halide::Internal::CodeGen_LLVM::function_does_not_access_memory
void function_does_not_access_memory(llvm::Function *fn)
Add the appropriate function attribute to tell LLVM that the function doesn't access memory.
Halide::Internal::CodeGen_LLVM::strict_fp_math_md
llvm::MDNode * strict_fp_math_md
Definition: CodeGen_LLVM.h:172
Halide::Internal::CodeGen_LLVM::call_intrin
llvm::Value * call_intrin(const Type &t, int intrin_lanes, const std::string &name, std::vector< Expr >)
Generate a call to a vector intrinsic or runtime inlined function.
Halide::Internal::CodeGen_LLVM::get_destructor_block
llvm::BasicBlock * get_destructor_block()
Retrieves the block containing the error handling code.
Halide::Internal::CodeGen_LLVM::wild_f32_
Expr wild_f32_
Definition: CodeGen_LLVM.h:256
Halide::Internal::CodeGen_LLVM::init_module
virtual void init_module()
Initialize the CodeGen_LLVM internal state to compile a fresh module.
Halide::Internal::CodeGen_LLVM::get_user_context
llvm::Value * get_user_context() const
The user_context argument.
Halide::Internal::CodeGen_LLVM::wild_u64_
Expr wild_u64_
Definition: CodeGen_LLVM.h:255
Halide::Internal::CodeGen_LLVM::mattrs
virtual std::string mattrs() const =0
Halide::Internal::CodeGen_LLVM::visit
void visit(const Realize *) override
Halide::Internal::CodeGen_LLVM::visit
void visit(const VectorReduce *) override
Halide::Internal::CodeGen_LLVM::visit
void visit(const IfThenElse *) override
Halide::Internal::CodeGen_LLVM::i32_t
llvm::Type * i32_t
Definition: CodeGen_LLVM.h:232
Halide::Internal::CodeGen_LLVM::requested_alloca_total
size_t requested_alloca_total
A (very) conservative guess at the size of all alloca() storage requested (including alignment paddin...
Definition: CodeGen_LLVM.h:452
Halide::Internal::CodeGen_LLVM::wild_u1x_
Expr wild_u1x_
Some wildcard variables used for peephole optimizations in subclasses.
Definition: CodeGen_LLVM.h:249
Halide::Internal::CodeGen_LLVM::wild_u16x_
Expr wild_u16x_
Definition: CodeGen_LLVM.h:249
Halide::Internal::CodeGen_LLVM::pseudostack_slot_t_type
llvm::StructType * pseudostack_slot_t_type
Definition: CodeGen_LLVM.h:241
Halide::Internal::CodeGen_LLVM::wild_u32x_
Expr wild_u32x_
Definition: CodeGen_LLVM.h:250
Halide::Internal::CodeGen_LLVM::llvm_type_of
llvm::Type * llvm_type_of(llvm::LLVMContext *context, Halide::Type t, int effective_vscale) const
Get the llvm type equivalent to a given halide type.
Halide::Internal::CodeGen_LLVM::scalable_to_fixed_vector_type
llvm::Value * scalable_to_fixed_vector_type(llvm::Value *scalable)
Convert an LLVM vscale vector value to the corresponding fixed vector value.
Halide::Internal::CodeGen_LLVM::wild_i64x_
Expr wild_i64x_
Definition: CodeGen_LLVM.h:250
Halide::Internal::CodeGen_LLVM::call_intrin
llvm::Value * call_intrin(const llvm::Type *t, int intrin_lanes, llvm::Function *intrin, std::vector< llvm::Value * >, bool is_reduction=false)
Halide::Internal::CodeGen_LLVM::sym_exists
bool sym_exists(const std::string &name) const
Test if an item exists in the symbol table.
Halide::Internal::CodeGen_LLVM::match_vector_type_scalable
llvm::Value * match_vector_type_scalable(llvm::Value *value, VectorTypeConstraint constraint)
Make sure a value type has the same scalable/fixed vector type as a guide.
Halide::Internal::CodeGen_LLVM::visit
void visit(const Free *) override=0
Generate code for a free node.
Halide::Internal::CodeGen_LLVM::shuffle_vectors
llvm::Value * shuffle_vectors(llvm::Value *v, const std::vector< int > &indices)
Shorthand for shuffling a single vector.
Halide::Internal::CodeGen_LLVM::wild_i32x_
Expr wild_i32x_
Definition: CodeGen_LLVM.h:250
Halide::Internal::CodeGen_LLVM::visit
void visit(const Add *) override
Halide::Internal::CodeGen_LLVM::visit
void visit(const Block *) override
Halide::Internal::CodeGen_LLVM::supports_call_as_float16
virtual bool supports_call_as_float16(const Call *op) const
Can we call this operation with float16 type? This is used to avoid "emulated" equivalent code-gen in...
Halide::Internal::CodeGen_LLVM::visit
void visit(const UIntImm *) override
Halide::Internal::CodeGen_LLVM::visit
void visit(const LetStmt *) override
Halide::Internal::CodeGen_LLVM::void_t
llvm::Type * void_t
Some useful llvm types.
Definition: CodeGen_LLVM.h:232
Halide::Internal::CodeGen_LLVM::f16_t
llvm::Type * f16_t
Definition: CodeGen_LLVM.h:232
Halide::Internal::CodeGen_LLVM::call_overloaded_intrin
llvm::Value * call_overloaded_intrin(const Type &result_type, const std::string &name, const std::vector< Expr > &args)
Call an overloaded intrinsic function.
Halide::Internal::CodeGen_LLVM::interleave_vectors
virtual llvm::Value * interleave_vectors(const std::vector< llvm::Value * > &)
Implementation of the intrinsic call to interleave_vectors.
Halide::Internal::CodeGen_LLVM::get_vector_type
llvm::Type * get_vector_type(llvm::Type *, int n, VectorTypeConstraint type_constraint=VectorTypeConstraint::None) const
Halide::Internal::IRVisitor
A base class for algorithms that need to recursively walk over the IR.
Definition: IRVisitor.h:19
Halide::Internal::IRVisitor::visit
virtual void visit(const IntImm *)
Halide::Internal::Scope< llvm::Value * >
Halide::Module
A halide module.
Definition: Module.h:142
Halide::Parameter
A reference-counted handle to a parameter to a halide pipeline.
Definition: Parameter.h:40
Halide::Internal::ArgInfoKind::Function
@ Function
Halide::Internal::SyntheticParamType::Type
@ Type
Halide::Internal::Monotonic::Constant
@ Constant
Halide
This file defines the class FunctionDAG, which is our representation of a Halide pipeline,...
Definition: AbstractGenerator.h:19
Halide::LinkageType
LinkageType
Type of linkage a function in a lowered Halide module can have.
Definition: Module.h:52
Halide::LinkageType::Internal
@ Internal
Not visible externally, similar to 'static' linkage in C.
Halide::MetadataNameMap
std::map< std::string, std::string > MetadataNameMap
Definition: Module.h:138
Halide::codegen_llvm
std::unique_ptr< llvm::Module > codegen_llvm(const Module &module, llvm::LLVMContext &context)
Given a Halide module, generate an llvm::Module.
llvm
Definition: CodeGen_Internal.h:18
int32_t
signed __INT32_TYPE__ int32_t
Definition: runtime_internal.h:24
Halide::Expr
A fragment of Halide syntax.
Definition: Expr.h:258
Halide::ExternSignature
Definition: Pipeline.h:501
Halide::Internal::Add
The sum of two expressions.
Definition: IR.h:66
Halide::Internal::Allocate
Allocate a scratch area called with the given name, type, and size.
Definition: IR.h:381
Halide::Internal::And
Logical and - are both expressions true.
Definition: IR.h:185
Halide::Internal::AssertStmt
If the 'condition' is false, then evaluate and return the message, which should be a call to an error...
Definition: IR.h:304
Halide::Internal::Atomic
Lock all the Store nodes in the body statement.
Definition: IR.h:1006
Halide::Internal::Block
A sequence of statements to be executed in-order.
Definition: IR.h:452
Halide::Internal::Broadcast
A vector with 'lanes' elements, in which every element is 'value'.
Definition: IR.h:269
Halide::Internal::Call
A function call.
Definition: IR.h:500
Halide::Internal::Cast
The actual IR nodes begin here.
Definition: IR.h:40
Halide::Internal::CodeGen_LLVM::AllEnabledMask
Type indicating mask to use is all true – all lanes enabled.
Definition: CodeGen_LLVM.h:644
Halide::Internal::CodeGen_LLVM::Intrinsic
Description of an intrinsic function overload.
Definition: CodeGen_LLVM.h:466
Halide::Internal::CodeGen_LLVM::Intrinsic::Intrinsic
Intrinsic(Type result_type, std::vector< Type > arg_types, llvm::Function *impl)
Definition: CodeGen_LLVM.h:471
Halide::Internal::CodeGen_LLVM::Intrinsic::impl
llvm::Function * impl
Definition: CodeGen_LLVM.h:469
Halide::Internal::CodeGen_LLVM::Intrinsic::result_type
Type result_type
Definition: CodeGen_LLVM.h:467
Halide::Internal::CodeGen_LLVM::Intrinsic::arg_types
std::vector< Type > arg_types
Definition: CodeGen_LLVM.h:468
Halide::Internal::CodeGen_LLVM::NoMask
Type indicating an intrinsic does not take a mask.
Definition: CodeGen_LLVM.h:640
Halide::Internal::CodeGen_LLVM::ScopedFastMath
If any_strict_float is true, sets fast math flags for the lifetime of this object,...
Definition: CodeGen_LLVM.h:190
Halide::Internal::CodeGen_LLVM::ScopedFastMath::codegen
CodeGen_LLVM * codegen
Definition: CodeGen_LLVM.h:191
Halide::Internal::CodeGen_LLVM::ScopedFastMath::~ScopedFastMath
~ScopedFastMath()
Halide::Internal::CodeGen_LLVM::ScopedFastMath::ScopedFastMath
ScopedFastMath(CodeGen_LLVM *)
Halide::Internal::CodeGen_LLVM::VPArg
Support for generating LLVM vector predication intrinsics ("@llvm.vp.*" and "@llvm....
Definition: CodeGen_LLVM.h:629
Halide::Internal::CodeGen_LLVM::VPArg::value
llvm::Value * value
Definition: CodeGen_LLVM.h:630
Halide::Internal::CodeGen_LLVM::VPArg::mangle_index
std::optional< size_t > mangle_index
Definition: CodeGen_LLVM.h:632
Halide::Internal::CodeGen_LLVM::VPArg::alignment
int alignment
Definition: CodeGen_LLVM.h:633
Halide::Internal::CodeGen_LLVM::VPArg::VPArg
VPArg(llvm::Value *value, std::optional< size_t > mangle_index=std::nullopt, int32_t alignment=0)
Definition: CodeGen_LLVM.h:634
Halide::Internal::CodeGen_LLVM::VPResultType
Definition: CodeGen_LLVM.h:660
Halide::Internal::CodeGen_LLVM::VPResultType::type
llvm::Type * type
Definition: CodeGen_LLVM.h:661
Halide::Internal::CodeGen_LLVM::VPResultType::VPResultType
VPResultType(llvm::Type *type, std::optional< size_t > mangle_index=std::nullopt)
Definition: CodeGen_LLVM.h:663
Halide::Internal::CodeGen_LLVM::VPResultType::mangle_index
std::optional< size_t > mangle_index
Definition: CodeGen_LLVM.h:662
Halide::Internal::Div
The ratio of two expressions.
Definition: IR.h:93
Halide::Internal::EQ
Is the first expression equal to the second.
Definition: IR.h:131
Halide::Internal::Evaluate
Evaluate and discard an expression, presumably because it has some side-effect.
Definition: IR.h:486
Halide::Internal::FloatImm
Floating point constants.
Definition: Expr.h:236
Halide::Internal::For
A for loop.
Definition: IR.h:856
Halide::Internal::Free
Free the resources associated with the given buffer.
Definition: IR.h:423
Halide::Internal::GE
Is the first expression greater than or equal to the second.
Definition: IR.h:176
Halide::Internal::GT
Is the first expression greater than the second.
Definition: IR.h:167
Halide::Internal::IfThenElse
An if-then-else block.
Definition: IR.h:476
Halide::Internal::IntImm
Integer constants.
Definition: Expr.h:218
Halide::Internal::LE
Is the first expression less than or equal to the second.
Definition: IR.h:158
Halide::Internal::LT
Is the first expression less than the second.
Definition: IR.h:149
Halide::Internal::Let
A let expression, like you might find in a functional language.
Definition: IR.h:281
Halide::Internal::LetStmt
The statement form of a let node.
Definition: IR.h:292
Halide::Internal::Load
Load a value from a named symbol if predicate is true.
Definition: IR.h:227
Halide::Internal::LoweredFunc
Definition of a lowered function.
Definition: Module.h:101
Halide::Internal::Max
The greater of two values.
Definition: IR.h:122
Halide::Internal::Min
The lesser of two values.
Definition: IR.h:113
Halide::Internal::Mod
The remainder of a / b.
Definition: IR.h:104
Halide::Internal::ModulusRemainder
The result of modulus_remainder analysis.
Definition: ModulusRemainder.h:33
Halide::Internal::Mul
The product of two expressions.
Definition: IR.h:84
Halide::Internal::NE
Is the first expression not equal to the second.
Definition: IR.h:140
Halide::Internal::Not
Logical not - true if the expression false.
Definition: IR.h:203
Halide::Internal::Or
Logical or - is at least one of the expression true.
Definition: IR.h:194
Halide::Internal::Prefetch
Represent a multi-dimensional region of a Func or an ImageParam that needs to be prefetched.
Definition: IR.h:968
Halide::Internal::ProducerConsumer
This node is a helpful annotation to do with permissions.
Definition: IR.h:325
Halide::Internal::Provide
This defines the value of a function at a multi-dimensional location.
Definition: IR.h:364
Halide::Internal::Ramp
A linear ramp vector node.
Definition: IR.h:257
Halide::Internal::Realize
Allocate a multi-dimensional buffer of the given type and size.
Definition: IR.h:437
Halide::Internal::Reinterpret
Reinterpret value as another type, without affecting any of the bits (on little-endian systems).
Definition: IR.h:57
Halide::Internal::Select
A ternary operator.
Definition: IR.h:214
Halide::Internal::Shuffle
Construct a new vector by taking elements from another sequence of vectors.
Definition: IR.h:896
Halide::Internal::Stmt
A reference-counted handle to a statement node.
Definition: Expr.h:427
Halide::Internal::Store
Store a 'value' to the buffer called 'name' at a given 'index' if 'predicate' is true.
Definition: IR.h:343
Halide::Internal::StringImm
String constants.
Definition: Expr.h:245
Halide::Internal::Sub
The difference of two expressions.
Definition: IR.h:75
Halide::Internal::UIntImm
Unsigned integer constants.
Definition: Expr.h:227
Halide::Internal::Variable
A named variable.
Definition: IR.h:811
Halide::Internal::VectorReduce
Horizontally reduce a vector to a scalar or narrower vector using the given commutative and associati...
Definition: IR.h:1024
Halide::Target
A struct representing a target machine and os to generate code for.
Definition: Target.h:19
Halide::Type
Types in the halide type system.
Definition: Type.h:281
error_code
int error_code
Definition: vulkan_resources.h:182