1//===- MveEmitter.cpp - Generate arm_mve.h for use with clang -*- C++ -*-=====//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This set of linked tablegen backends is responsible for emitting the bits
10// and pieces that implement <arm_mve.h>, which is defined by the ACLE standard
11// and provides a set of types and functions for (more or less) direct access
12// to the MVE instruction set, including the scalar shifts as well as the
13// vector instructions.
14//
15// MVE's standard intrinsic functions are unusual in that they have a system of
16// polymorphism. For example, the function vaddq() can behave like vaddq_u16(),
17// vaddq_f32(), vaddq_s8(), etc., depending on the types of the vector
18// arguments you give it.
19//
20// This constrains the implementation strategies. The usual approach to making
21// the user-facing functions polymorphic would be to either use
22// __attribute__((overloadable)) to make a set of vaddq() functions that are
23// all inline wrappers on the underlying clang builtins, or to define a single
24// vaddq() macro which expands to an instance of _Generic.
25//
26// The inline-wrappers approach would work fine for most intrinsics, except for
27// the ones that take an argument required to be a compile-time constant,
28// because if you wrap an inline function around a call to a builtin, the
29// constant nature of the argument is not passed through.
30//
31// The _Generic approach can be made to work with enough effort, but it takes a
32// lot of machinery, because of the design feature of _Generic that even the
33// untaken branches are required to pass all front-end validity checks such as
34// type-correctness. You can work around that by nesting further _Generics all
35// over the place to coerce things to the right type in untaken branches, but
36// what you get out is complicated, hard to guarantee its correctness, and
37// worst of all, gives _completely unreadable_ error messages if the user gets
38// the types wrong for an intrinsic call.
39//
40// Therefore, my strategy is to introduce a new __attribute__ that allows a
41// function to be mapped to a clang builtin even though it doesn't have the
42// same name, and then declare all the user-facing MVE function names with that
43// attribute, mapping each one directly to the clang builtin. And the
44// polymorphic ones have __attribute__((overloadable)) as well. So once the
45// compiler has resolved the overload, it knows the internal builtin ID of the
46// selected function, and can check the immediate arguments against that; and
47// if the user gets the types wrong in a call to a polymorphic intrinsic, they
48// get a completely clear error message showing all the declarations of that
49// function in the header file and explaining why each one doesn't fit their
50// call.
51//
52// The downside of this is that if every clang builtin has to correspond
53// exactly to a user-facing ACLE intrinsic, then you can't save work in the
54// frontend by doing it in the header file: CGBuiltin.cpp has to do the entire
55// job of converting an ACLE intrinsic call into LLVM IR. So the Tablegen
56// description for an MVE intrinsic has to contain a full description of the
57// sequence of IRBuilder calls that clang will need to make.
58//
59//===----------------------------------------------------------------------===//
60
61#include "llvm/ADT/APInt.h"
62#include "llvm/ADT/StringRef.h"
63#include "llvm/ADT/StringSwitch.h"
64#include "llvm/Support/Casting.h"
65#include "llvm/Support/raw_ostream.h"
66#include "llvm/TableGen/Error.h"
67#include "llvm/TableGen/Record.h"
68#include "llvm/TableGen/StringToOffsetTable.h"
69#include <cassert>
70#include <cstddef>
71#include <cstdint>
72#include <list>
73#include <map>
74#include <memory>
75#include <set>
76#include <string>
77#include <vector>
78
79using namespace llvm;
80
81namespace {
82
83class EmitterBase;
84class Result;
85
86// -----------------------------------------------------------------------------
87// A system of classes to represent all the types we'll need to deal with in
88// the prototypes of intrinsics.
89//
90// Query methods include finding out the C name of a type; the "LLVM name" in
91// the sense of a C++ code snippet that can be used in the codegen function;
92// the suffix that represents the type in the ACLE intrinsic naming scheme
93// (e.g. 's32' represents int32_t in intrinsics such as vaddq_s32); whether the
94// type is floating-point related (hence should be under #ifdef in the MVE
95// header so that it isn't included in integer-only MVE mode); and the type's
96// size in bits. Not all subtypes support all these queries.
97
98class Type {
99public:
100 enum class TypeKind {
101 // Void appears as a return type (for store intrinsics, which are pure
102 // side-effect). It's also used as the parameter type in the Tablegen
103 // when an intrinsic doesn't need to come in various suffixed forms like
104 // vfooq_s8,vfooq_u16,vfooq_f32.
105 Void,
106
107 // Scalar is used for ordinary int and float types of all sizes.
108 Scalar,
109
110 // Vector is used for anything that occupies exactly one MVE vector
111 // register, i.e. {uint,int,float}NxM_t.
112 Vector,
113
114 // MultiVector is used for the {uint,int,float}NxMxK_t types used by the
115 // interleaving load/store intrinsics v{ld,st}{2,4}q.
116 MultiVector,
117
118 // Predicate is used by all the predicated intrinsics. Its C
119 // representation is mve_pred16_t (which is just an alias for uint16_t).
120 // But we give more detail here, by indicating that a given predicate
121 // instruction is logically regarded as a vector of i1 containing the
122 // same number of lanes as the input vector type. So our Predicate type
123 // comes with a lane count, which we use to decide which kind of <n x i1>
124 // we'll invoke the pred_i2v IR intrinsic to translate it into.
125 Predicate,
126
127 // Pointer is used for pointer types (obviously), and comes with a flag
128 // indicating whether it's a pointer to a const or mutable instance of
129 // the pointee type.
130 Pointer,
131 };
132
133private:
134 const TypeKind TKind;
135
136protected:
137 Type(TypeKind K) : TKind(K) {}
138
139public:
140 TypeKind typeKind() const { return TKind; }
141 virtual ~Type() = default;
142 virtual bool requiresFloat() const = 0;
143 virtual bool requiresMVE() const = 0;
144 virtual unsigned sizeInBits() const = 0;
145 virtual std::string cName() const = 0;
146 virtual std::string llvmName() const {
147 PrintFatalError(Msg: "no LLVM type name available for type " + cName());
148 }
149 virtual std::string acleSuffix(std::string) const {
150 PrintFatalError(Msg: "no ACLE suffix available for this type");
151 }
152};
153
154enum class ScalarTypeKind { SignedInt, UnsignedInt, Float };
155inline std::string toLetter(ScalarTypeKind kind) {
156 switch (kind) {
157 case ScalarTypeKind::SignedInt:
158 return "s";
159 case ScalarTypeKind::UnsignedInt:
160 return "u";
161 case ScalarTypeKind::Float:
162 return "f";
163 }
164 llvm_unreachable("Unhandled ScalarTypeKind enum");
165}
166inline std::string toCPrefix(ScalarTypeKind kind) {
167 switch (kind) {
168 case ScalarTypeKind::SignedInt:
169 return "int";
170 case ScalarTypeKind::UnsignedInt:
171 return "uint";
172 case ScalarTypeKind::Float:
173 return "float";
174 }
175 llvm_unreachable("Unhandled ScalarTypeKind enum");
176}
177
178class VoidType : public Type {
179public:
180 VoidType() : Type(TypeKind::Void) {}
181 unsigned sizeInBits() const override { return 0; }
182 bool requiresFloat() const override { return false; }
183 bool requiresMVE() const override { return false; }
184 std::string cName() const override { return "void"; }
185
186 static bool classof(const Type *T) { return T->typeKind() == TypeKind::Void; }
187 std::string acleSuffix(std::string) const override { return ""; }
188};
189
190class PointerType : public Type {
191 const Type *Pointee;
192 bool Const;
193
194public:
195 PointerType(const Type *Pointee, bool Const)
196 : Type(TypeKind::Pointer), Pointee(Pointee), Const(Const) {}
197 unsigned sizeInBits() const override { return 32; }
198 bool requiresFloat() const override { return Pointee->requiresFloat(); }
199 bool requiresMVE() const override { return Pointee->requiresMVE(); }
200 std::string cName() const override {
201 std::string Name = Pointee->cName();
202
203 // The syntax for a pointer in C is different when the pointee is
204 // itself a pointer. The MVE intrinsics don't contain any double
205 // pointers, so we don't need to worry about that wrinkle.
206 assert(!isa<PointerType>(Pointee) && "Pointer to pointer not supported");
207
208 if (Const)
209 Name = "const " + Name;
210 return Name + " *";
211 }
212 std::string llvmName() const override {
213 return "llvm::PointerType::getUnqual(" + Pointee->llvmName() + ")";
214 }
215 const Type *getPointeeType() const { return Pointee; }
216
217 static bool classof(const Type *T) {
218 return T->typeKind() == TypeKind::Pointer;
219 }
220};
221
222// Base class for all the types that have a name of the form
223// [prefix][numbers]_t, like int32_t, uint16x8_t, float32x4x2_t.
224//
225// For this sub-hierarchy we invent a cNameBase() method which returns the
226// whole name except for the trailing "_t", so that Vector and MultiVector can
227// append an extra "x2" or whatever to their element type's cNameBase(). Then
228// the main cName() query method puts "_t" on the end for the final type name.
229
230class CRegularNamedType : public Type {
231 using Type::Type;
232 virtual std::string cNameBase() const = 0;
233
234public:
235 std::string cName() const override { return cNameBase() + "_t"; }
236};
237
238class ScalarType : public CRegularNamedType {
239 ScalarTypeKind Kind;
240 unsigned Bits;
241 std::string NameOverride;
242
243public:
244 ScalarType(const Record *Record) : CRegularNamedType(TypeKind::Scalar) {
245 Kind = StringSwitch<ScalarTypeKind>(Record->getValueAsString(FieldName: "kind"))
246 .Case(S: "s", Value: ScalarTypeKind::SignedInt)
247 .Case(S: "u", Value: ScalarTypeKind::UnsignedInt)
248 .Case(S: "f", Value: ScalarTypeKind::Float);
249 Bits = Record->getValueAsInt(FieldName: "size");
250 NameOverride = std::string(Record->getValueAsString(FieldName: "nameOverride"));
251 }
252 unsigned sizeInBits() const override { return Bits; }
253 ScalarTypeKind kind() const { return Kind; }
254 std::string suffix() const { return toLetter(kind: Kind) + utostr(X: Bits); }
255 std::string cNameBase() const override {
256 return toCPrefix(kind: Kind) + utostr(X: Bits);
257 }
258 std::string cName() const override {
259 if (NameOverride.empty())
260 return CRegularNamedType::cName();
261 return NameOverride;
262 }
263 std::string llvmName() const override {
264 if (Kind == ScalarTypeKind::Float) {
265 if (Bits == 16)
266 return "HalfTy";
267 if (Bits == 32)
268 return "FloatTy";
269 if (Bits == 64)
270 return "DoubleTy";
271 PrintFatalError(Msg: "bad size for floating type");
272 }
273 return "Int" + utostr(X: Bits) + "Ty";
274 }
275 std::string acleSuffix(std::string overrideLetter) const override {
276 return "_" + (overrideLetter.size() ? overrideLetter : toLetter(kind: Kind))
277 + utostr(X: Bits);
278 }
279 bool isInteger() const { return Kind != ScalarTypeKind::Float; }
280 bool requiresFloat() const override { return !isInteger(); }
281 bool requiresMVE() const override { return false; }
282 bool hasNonstandardName() const { return !NameOverride.empty(); }
283
284 static bool classof(const Type *T) {
285 return T->typeKind() == TypeKind::Scalar;
286 }
287};
288
289class VectorType : public CRegularNamedType {
290 const ScalarType *Element;
291 unsigned Lanes;
292
293public:
294 VectorType(const ScalarType *Element, unsigned Lanes)
295 : CRegularNamedType(TypeKind::Vector), Element(Element), Lanes(Lanes) {}
296 unsigned sizeInBits() const override { return Lanes * Element->sizeInBits(); }
297 unsigned lanes() const { return Lanes; }
298 bool requiresFloat() const override { return Element->requiresFloat(); }
299 bool requiresMVE() const override { return true; }
300 std::string cNameBase() const override {
301 return Element->cNameBase() + "x" + utostr(X: Lanes);
302 }
303 std::string llvmName() const override {
304 return "llvm::FixedVectorType::get(" + Element->llvmName() + ", " +
305 utostr(X: Lanes) + ")";
306 }
307
308 static bool classof(const Type *T) {
309 return T->typeKind() == TypeKind::Vector;
310 }
311};
312
313class MultiVectorType : public CRegularNamedType {
314 const VectorType *Element;
315 unsigned Registers;
316
317public:
318 MultiVectorType(unsigned Registers, const VectorType *Element)
319 : CRegularNamedType(TypeKind::MultiVector), Element(Element),
320 Registers(Registers) {}
321 unsigned sizeInBits() const override {
322 return Registers * Element->sizeInBits();
323 }
324 unsigned registers() const { return Registers; }
325 bool requiresFloat() const override { return Element->requiresFloat(); }
326 bool requiresMVE() const override { return true; }
327 std::string cNameBase() const override {
328 return Element->cNameBase() + "x" + utostr(X: Registers);
329 }
330
331 // MultiVectorType doesn't override llvmName, because we don't expect to do
332 // automatic code generation for the MVE intrinsics that use it: the {vld2,
333 // vld4, vst2, vst4} family are the only ones that use these types, so it was
334 // easier to hand-write the codegen for dealing with these structs than to
335 // build in lots of extra automatic machinery that would only be used once.
336
337 static bool classof(const Type *T) {
338 return T->typeKind() == TypeKind::MultiVector;
339 }
340};
341
342class PredicateType : public CRegularNamedType {
343 unsigned Lanes;
344
345public:
346 PredicateType(unsigned Lanes)
347 : CRegularNamedType(TypeKind::Predicate), Lanes(Lanes) {}
348 unsigned sizeInBits() const override { return 16; }
349 std::string cNameBase() const override { return "mve_pred16"; }
350 bool requiresFloat() const override { return false; };
351 bool requiresMVE() const override { return true; }
352 std::string llvmName() const override {
353 return "llvm::FixedVectorType::get(Builder.getInt1Ty(), " + utostr(X: Lanes) +
354 ")";
355 }
356
357 static bool classof(const Type *T) {
358 return T->typeKind() == TypeKind::Predicate;
359 }
360};
361
362// -----------------------------------------------------------------------------
363// Class to facilitate merging together the code generation for many intrinsics
364// by means of varying a few constant or type parameters.
365//
366// Most obviously, the intrinsics in a single parametrised family will have
367// code generation sequences that only differ in a type or two, e.g. vaddq_s8
368// and vaddq_u16 will look the same apart from putting a different vector type
369// in the call to CGM.getIntrinsic(). But also, completely different intrinsics
370// will often code-generate in the same way, with only a different choice of
371// _which_ IR intrinsic they lower to (e.g. vaddq_m_s8 and vmulq_m_s8), but
372// marshalling the arguments and return values of the IR intrinsic in exactly
373// the same way. And others might differ only in some other kind of constant,
374// such as a lane index.
375//
376// So, when we generate the IR-building code for all these intrinsics, we keep
377// track of every value that could possibly be pulled out of the code and
378// stored ahead of time in a local variable. Then we group together intrinsics
379// by textual equivalence of the code that would result if _all_ those
380// parameters were stored in local variables. That gives us maximal sets that
381// can be implemented by a single piece of IR-building code by changing
382// parameter values ahead of time.
383//
384// After we've done that, we do a second pass in which we only allocate _some_
385// of the parameters into local variables, by tracking which ones have the same
386// values as each other (so that a single variable can be reused) and which
387// ones are the same across the whole set (so that no variable is needed at
388// all).
389//
390// Hence the class below. Its allocParam method is invoked during code
391// generation by every method of a Result subclass (see below) that wants to
392// give it the opportunity to pull something out into a switchable parameter.
393// It returns a variable name for the parameter, or (if it's being used in the
394// second pass once we've decided that some parameters don't need to be stored
395// in variables after all) it might just return the input expression unchanged.
396
397struct CodeGenParamAllocator {
398 // Accumulated during code generation
399 std::vector<std::string> *ParamTypes = nullptr;
400 std::vector<std::string> *ParamValues = nullptr;
401
402 // Provided ahead of time in pass 2, to indicate which parameters are being
403 // assigned to what. This vector contains an entry for each call to
404 // allocParam expected during code gen (which we counted up in pass 1), and
405 // indicates the number of the parameter variable that should be returned, or
406 // -1 if this call shouldn't allocate a parameter variable at all.
407 //
408 // We rely on the recursive code generation working identically in passes 1
409 // and 2, so that the same list of calls to allocParam happen in the same
410 // order. That guarantees that the parameter numbers recorded in pass 1 will
411 // match the entries in this vector that store what EmitterBase::EmitBuiltinCG
412 // decided to do about each one in pass 2.
413 std::vector<int> *ParamNumberMap = nullptr;
414
415 // Internally track how many things we've allocated
416 unsigned nparams = 0;
417
418 std::string allocParam(StringRef Type, StringRef Value) {
419 unsigned ParamNumber;
420
421 if (!ParamNumberMap) {
422 // In pass 1, unconditionally assign a new parameter variable to every
423 // value we're asked to process.
424 ParamNumber = nparams++;
425 } else {
426 // In pass 2, consult the map provided by the caller to find out which
427 // variable we should be keeping things in.
428 int MapValue = (*ParamNumberMap)[nparams++];
429 if (MapValue < 0)
430 return std::string(Value);
431 ParamNumber = MapValue;
432 }
433
434 // If we've allocated a new parameter variable for the first time, store
435 // its type and value to be retrieved after codegen.
436 if (ParamTypes && ParamTypes->size() == ParamNumber)
437 ParamTypes->push_back(x: std::string(Type));
438 if (ParamValues && ParamValues->size() == ParamNumber)
439 ParamValues->push_back(x: std::string(Value));
440
441 // Unimaginative naming scheme for parameter variables.
442 return "Param" + utostr(X: ParamNumber);
443 }
444};
445
446// -----------------------------------------------------------------------------
447// System of classes that represent all the intermediate values used during
448// code-generation for an intrinsic.
449//
450// The base class 'Result' can represent a value of the LLVM type 'Value', or
451// sometimes 'Address' (for loads/stores, including an alignment requirement).
452//
453// In the case where the Tablegen provides a value in the codegen dag as a
454// plain integer literal, the Result object we construct here will be one that
455// returns true from hasIntegerConstantValue(). This allows the generated C++
456// code to use the constant directly in contexts which can take a literal
457// integer, such as Builder.CreateExtractValue(thing, 1), without going to the
458// effort of calling llvm::ConstantInt::get() and then pulling the constant
459// back out of the resulting llvm:Value later.
460
461class Result {
462public:
463 // Convenient shorthand for the pointer type we'll be using everywhere.
464 using Ptr = std::shared_ptr<Result>;
465
466private:
467 Ptr Predecessor;
468 std::string VarName;
469 bool VarNameUsed = false;
470 unsigned Visited = 0;
471
472public:
473 virtual ~Result() = default;
474 using Scope = std::map<std::string, Ptr>;
475 virtual void genCode(raw_ostream &OS, CodeGenParamAllocator &) const = 0;
476 virtual bool hasIntegerConstantValue() const { return false; }
477 virtual uint32_t integerConstantValue() const { return 0; }
478 virtual bool hasIntegerValue() const { return false; }
479 virtual std::string getIntegerValue(const std::string &) {
480 llvm_unreachable("non-working Result::getIntegerValue called");
481 }
482 virtual std::string typeName() const { return "Value *"; }
483
484 // Mostly, when a code-generation operation has a dependency on prior
485 // operations, it's because it uses the output values of those operations as
486 // inputs. But there's one exception, which is the use of 'seq' in Tablegen
487 // to indicate that operations have to be performed in sequence regardless of
488 // whether they use each others' output values.
489 //
490 // So, the actual generation of code is done by depth-first search, using the
491 // prerequisites() method to get a list of all the other Results that have to
492 // be computed before this one. That method divides into the 'predecessor',
493 // set by setPredecessor() while processing a 'seq' dag node, and the list
494 // returned by 'morePrerequisites', which each subclass implements to return
495 // a list of the Results it uses as input to whatever its own computation is
496 // doing.
497
498 virtual void morePrerequisites(std::vector<Ptr> &output) const {}
499 std::vector<Ptr> prerequisites() const {
500 std::vector<Ptr> ToRet;
501 if (Predecessor)
502 ToRet.push_back(x: Predecessor);
503 morePrerequisites(output&: ToRet);
504 return ToRet;
505 }
506
507 void setPredecessor(Ptr p) {
508 // If the user has nested one 'seq' node inside another, and this
509 // method is called on the return value of the inner 'seq' (i.e.
510 // the final item inside it), then we can't link _this_ node to p,
511 // because it already has a predecessor. Instead, walk the chain
512 // until we find the first item in the inner seq, and link that to
513 // p, so that nesting seqs has the obvious effect of linking
514 // everything together into one long sequential chain.
515 Result *r = this;
516 while (r->Predecessor)
517 r = r->Predecessor.get();
518 r->Predecessor = p;
519 }
520
521 // Each Result will be assigned a variable name in the output code, but not
522 // all those variable names will actually be used (e.g. the return value of
523 // Builder.CreateStore has void type, so nobody will want to refer to it). To
524 // prevent annoying compiler warnings, we track whether each Result's
525 // variable name was ever actually mentioned in subsequent statements, so
526 // that it can be left out of the final generated code.
527 std::string varname() {
528 VarNameUsed = true;
529 return VarName;
530 }
531 void setVarname(const StringRef s) { VarName = std::string(s); }
532 bool varnameUsed() const { return VarNameUsed; }
533
534 // Emit code to generate this result as a Value *.
535 virtual std::string asValue() {
536 return varname();
537 }
538
539 // Code generation happens in multiple passes. This method tracks whether a
540 // Result has yet been visited in a given pass, without the need for a
541 // tedious loop in between passes that goes through and resets a 'visited'
542 // flag back to false: you just set Pass=1 the first time round, and Pass=2
543 // the second time.
544 bool needsVisiting(unsigned Pass) {
545 bool ToRet = Visited < Pass;
546 Visited = Pass;
547 return ToRet;
548 }
549};
550
551// Result subclass that retrieves one of the arguments to the clang builtin
552// function. In cases where the argument has pointer type, we call
553// EmitPointerWithAlignment and store the result in a variable of type Address,
554// so that load and store IR nodes can know the right alignment. Otherwise, we
555// call EmitScalarExpr.
556//
557// There are aggregate parameters in the MVE intrinsics API, but we don't deal
558// with them in this Tablegen back end: they only arise in the vld2q/vld4q and
559// vst2q/vst4q family, which is few enough that we just write the code by hand
560// for those in CGBuiltin.cpp.
561class BuiltinArgResult : public Result {
562public:
563 unsigned ArgNum;
564 bool AddressType;
565 bool Immediate;
566 BuiltinArgResult(unsigned ArgNum, bool AddressType, bool Immediate)
567 : ArgNum(ArgNum), AddressType(AddressType), Immediate(Immediate) {}
568 void genCode(raw_ostream &OS, CodeGenParamAllocator &) const override {
569 OS << (AddressType ? "EmitPointerWithAlignment" : "EmitScalarExpr")
570 << "(E->getArg(" << ArgNum << "))";
571 }
572 std::string typeName() const override {
573 return AddressType ? "Address" : Result::typeName();
574 }
575 // Emit code to generate this result as a Value *.
576 std::string asValue() override {
577 if (AddressType)
578 return "(" + varname() + ".emitRawPointer(*this))";
579 return Result::asValue();
580 }
581 bool hasIntegerValue() const override { return Immediate; }
582 std::string getIntegerValue(const std::string &IntType) override {
583 return "GetIntegerConstantValue<" + IntType + ">(E->getArg(" +
584 utostr(X: ArgNum) + "), getContext())";
585 }
586};
587
588// Result subclass for an integer literal appearing in Tablegen. This may need
589// to be turned into an llvm::Result by means of llvm::ConstantInt::get(), or
590// it may be used directly as an integer, depending on which IRBuilder method
591// it's being passed to.
592class IntLiteralResult : public Result {
593public:
594 const ScalarType *IntegerType;
595 uint32_t IntegerValue;
596 IntLiteralResult(const ScalarType *IntegerType, uint32_t IntegerValue)
597 : IntegerType(IntegerType), IntegerValue(IntegerValue) {}
598 void genCode(raw_ostream &OS,
599 CodeGenParamAllocator &ParamAlloc) const override {
600 OS << "llvm::ConstantInt::get("
601 << ParamAlloc.allocParam(Type: "llvm::Type *", Value: IntegerType->llvmName())
602 << ", ";
603 OS << ParamAlloc.allocParam(Type: IntegerType->cName(), Value: utostr(X: IntegerValue))
604 << ")";
605 }
606 bool hasIntegerConstantValue() const override { return true; }
607 uint32_t integerConstantValue() const override { return IntegerValue; }
608};
609
610// Result subclass representing a cast between different integer types. We use
611// our own ScalarType abstraction as the representation of the target type,
612// which gives both size and signedness.
613class IntCastResult : public Result {
614public:
615 const ScalarType *IntegerType;
616 Ptr V;
617 IntCastResult(const ScalarType *IntegerType, Ptr V)
618 : IntegerType(IntegerType), V(V) {}
619 void genCode(raw_ostream &OS,
620 CodeGenParamAllocator &ParamAlloc) const override {
621 OS << "Builder.CreateIntCast(" << V->varname() << ", "
622 << ParamAlloc.allocParam(Type: "llvm::Type *", Value: IntegerType->llvmName()) << ", "
623 << ParamAlloc.allocParam(Type: "bool",
624 Value: IntegerType->kind() == ScalarTypeKind::SignedInt
625 ? "true"
626 : "false")
627 << ")";
628 }
629 void morePrerequisites(std::vector<Ptr> &output) const override {
630 output.push_back(x: V);
631 }
632};
633
634// Result subclass representing a cast between different pointer types.
635class PointerCastResult : public Result {
636public:
637 const PointerType *PtrType;
638 Ptr V;
639 PointerCastResult(const PointerType *PtrType, Ptr V)
640 : PtrType(PtrType), V(V) {}
641 void genCode(raw_ostream &OS,
642 CodeGenParamAllocator &ParamAlloc) const override {
643 OS << "Builder.CreatePointerCast(" << V->asValue() << ", "
644 << ParamAlloc.allocParam(Type: "llvm::Type *", Value: PtrType->llvmName()) << ")";
645 }
646 void morePrerequisites(std::vector<Ptr> &output) const override {
647 output.push_back(x: V);
648 }
649};
650
651// Result subclass representing a call to an IRBuilder method. Each IRBuilder
652// method we want to use will have a Tablegen record giving the method name and
653// describing any important details of how to call it, such as whether a
654// particular argument should be an integer constant instead of an llvm::Value.
655class IRBuilderResult : public Result {
656public:
657 StringRef CallPrefix;
658 std::vector<Ptr> Args;
659 std::set<unsigned> AddressArgs;
660 std::map<unsigned, std::string> IntegerArgs;
661 IRBuilderResult(StringRef CallPrefix, const std::vector<Ptr> &Args,
662 const std::set<unsigned> &AddressArgs,
663 const std::map<unsigned, std::string> &IntegerArgs)
664 : CallPrefix(CallPrefix), Args(Args), AddressArgs(AddressArgs),
665 IntegerArgs(IntegerArgs) {}
666 void genCode(raw_ostream &OS,
667 CodeGenParamAllocator &ParamAlloc) const override {
668 OS << CallPrefix;
669 const char *Sep = "";
670 for (unsigned i = 0, e = Args.size(); i < e; ++i) {
671 Ptr Arg = Args[i];
672 auto it = IntegerArgs.find(x: i);
673
674 OS << Sep;
675 Sep = ", ";
676
677 if (it != IntegerArgs.end()) {
678 if (Arg->hasIntegerConstantValue())
679 OS << "static_cast<" << it->second << ">("
680 << ParamAlloc.allocParam(Type: it->second,
681 Value: utostr(X: Arg->integerConstantValue()))
682 << ")";
683 else if (Arg->hasIntegerValue())
684 OS << ParamAlloc.allocParam(Type: it->second,
685 Value: Arg->getIntegerValue(it->second));
686 } else {
687 OS << Arg->varname();
688 }
689 }
690 OS << ")";
691 }
692 void morePrerequisites(std::vector<Ptr> &output) const override {
693 for (unsigned i = 0, e = Args.size(); i < e; ++i) {
694 Ptr Arg = Args[i];
695 if (IntegerArgs.find(x: i) != IntegerArgs.end())
696 continue;
697 output.push_back(x: Arg);
698 }
699 }
700};
701
702// Result subclass representing making an Address out of a Value.
703class AddressResult : public Result {
704public:
705 Ptr Arg;
706 const Type *Ty;
707 unsigned Align;
708 AddressResult(Ptr Arg, const Type *Ty, unsigned Align)
709 : Arg(Arg), Ty(Ty), Align(Align) {}
710 void genCode(raw_ostream &OS,
711 CodeGenParamAllocator &ParamAlloc) const override {
712 OS << "Address(" << Arg->varname() << ", " << Ty->llvmName()
713 << ", CharUnits::fromQuantity(" << Align << "))";
714 }
715 std::string typeName() const override {
716 return "Address";
717 }
718 void morePrerequisites(std::vector<Ptr> &output) const override {
719 output.push_back(x: Arg);
720 }
721};
722
723// Result subclass representing a call to an IR intrinsic, which we first have
724// to look up using an Intrinsic::ID constant and an array of types.
725class IRIntrinsicResult : public Result {
726public:
727 std::string IntrinsicID;
728 std::vector<const Type *> ParamTypes;
729 std::vector<Ptr> Args;
730 IRIntrinsicResult(StringRef IntrinsicID,
731 const std::vector<const Type *> &ParamTypes,
732 const std::vector<Ptr> &Args)
733 : IntrinsicID(std::string(IntrinsicID)), ParamTypes(ParamTypes),
734 Args(Args) {}
735 void genCode(raw_ostream &OS,
736 CodeGenParamAllocator &ParamAlloc) const override {
737 std::string IntNo = ParamAlloc.allocParam(
738 Type: "Intrinsic::ID", Value: "Intrinsic::" + IntrinsicID);
739 OS << "Builder.CreateCall(CGM.getIntrinsic(" << IntNo;
740 if (!ParamTypes.empty()) {
741 OS << ", {";
742 const char *Sep = "";
743 for (auto T : ParamTypes) {
744 OS << Sep << ParamAlloc.allocParam(Type: "llvm::Type *", Value: T->llvmName());
745 Sep = ", ";
746 }
747 OS << "}";
748 }
749 OS << "), {";
750 const char *Sep = "";
751 for (auto Arg : Args) {
752 OS << Sep << Arg->asValue();
753 Sep = ", ";
754 }
755 OS << "})";
756 }
757 void morePrerequisites(std::vector<Ptr> &output) const override {
758 output.insert(position: output.end(), first: Args.begin(), last: Args.end());
759 }
760};
761
762// Result subclass that specifies a type, for use in IRBuilder operations such
763// as CreateBitCast that take a type argument.
764class TypeResult : public Result {
765public:
766 const Type *T;
767 TypeResult(const Type *T) : T(T) {}
768 void genCode(raw_ostream &OS, CodeGenParamAllocator &) const override {
769 OS << T->llvmName();
770 }
771 std::string typeName() const override {
772 return "llvm::Type *";
773 }
774};
775
776// -----------------------------------------------------------------------------
777// Class that describes a single ACLE intrinsic.
778//
779// A Tablegen record will typically describe more than one ACLE intrinsic, by
780// means of setting the 'list<Type> Params' field to a list of multiple
781// parameter types, so as to define vaddq_{s8,u8,...,f16,f32} all in one go.
782// We'll end up with one instance of ACLEIntrinsic for *each* parameter type,
783// rather than a single one for all of them. Hence, the constructor takes both
784// a Tablegen record and the current value of the parameter type.
785
786class ACLEIntrinsic {
787 // Structure documenting that one of the intrinsic's arguments is required to
788 // be a compile-time constant integer, and what constraints there are on its
789 // value. Used when generating Sema checking code.
790 struct ImmediateArg {
791 enum class BoundsType { ExplicitRange, UInt };
792 BoundsType boundsType;
793 int64_t i1, i2;
794 StringRef ExtraCheckType, ExtraCheckArgs;
795 const Type *ArgType;
796 };
797
798 // For polymorphic intrinsics, FullName is the explicit name that uniquely
799 // identifies this variant of the intrinsic, and ShortName is the name it
800 // shares with at least one other intrinsic.
801 std::string ShortName, FullName;
802
803 // Name of the architecture extension, used in the Clang builtin name
804 StringRef BuiltinExtension;
805
806 // A very small number of intrinsics _only_ have a polymorphic
807 // variant (vuninitializedq taking an unevaluated argument).
808 bool PolymorphicOnly;
809
810 // Another rarely-used flag indicating that the builtin doesn't
811 // evaluate its argument(s) at all.
812 bool NonEvaluating;
813
814 // True if the intrinsic needs only the C header part (no codegen, semantic
815 // checks, etc). Used for redeclaring MVE intrinsics in the arm_cde.h header.
816 bool HeaderOnly;
817
818 const Type *ReturnType;
819 std::vector<const Type *> ArgTypes;
820 std::map<unsigned, ImmediateArg> ImmediateArgs;
821 Result::Ptr Code;
822
823 std::map<std::string, std::string> CustomCodeGenArgs;
824
825 // Recursive function that does the internals of code generation.
826 void genCodeDfs(Result::Ptr V, std::list<Result::Ptr> &Used,
827 unsigned Pass) const {
828 if (!V->needsVisiting(Pass))
829 return;
830
831 for (Result::Ptr W : V->prerequisites())
832 genCodeDfs(V: W, Used, Pass);
833
834 Used.push_back(x: V);
835 }
836
837public:
838 const std::string &shortName() const { return ShortName; }
839 const std::string &fullName() const { return FullName; }
840 StringRef builtinExtension() const { return BuiltinExtension; }
841 const Type *returnType() const { return ReturnType; }
842 const std::vector<const Type *> &argTypes() const { return ArgTypes; }
843 bool requiresFloat() const {
844 if (ReturnType->requiresFloat())
845 return true;
846 for (const Type *T : ArgTypes)
847 if (T->requiresFloat())
848 return true;
849 return false;
850 }
851 bool requiresMVE() const {
852 return ReturnType->requiresMVE() ||
853 any_of(Range: ArgTypes, P: [](const Type *T) { return T->requiresMVE(); });
854 }
855 bool polymorphic() const { return ShortName != FullName; }
856 bool polymorphicOnly() const { return PolymorphicOnly; }
857 bool nonEvaluating() const { return NonEvaluating; }
858 bool headerOnly() const { return HeaderOnly; }
859
860 // External entry point for code generation, called from EmitterBase.
861 void genCode(raw_ostream &OS, CodeGenParamAllocator &ParamAlloc,
862 unsigned Pass) const {
863 assert(!headerOnly() && "Called genCode for header-only intrinsic");
864 if (!hasCode()) {
865 for (auto kv : CustomCodeGenArgs)
866 OS << " " << kv.first << " = " << kv.second << ";\n";
867 OS << " break; // custom code gen\n";
868 return;
869 }
870 std::list<Result::Ptr> Used;
871 genCodeDfs(V: Code, Used, Pass);
872
873 unsigned varindex = 0;
874 for (Result::Ptr V : Used)
875 if (V->varnameUsed())
876 V->setVarname("Val" + utostr(X: varindex++));
877
878 for (Result::Ptr V : Used) {
879 OS << " ";
880 if (V == Used.back()) {
881 assert(!V->varnameUsed());
882 OS << "return "; // FIXME: what if the top-level thing is void?
883 } else if (V->varnameUsed()) {
884 std::string Type = V->typeName();
885 OS << V->typeName();
886 if (!StringRef(Type).ends_with(Suffix: "*"))
887 OS << " ";
888 OS << V->varname() << " = ";
889 }
890 V->genCode(OS, ParamAlloc);
891 OS << ";\n";
892 }
893 }
894 bool hasCode() const { return Code != nullptr; }
895
896 static std::string signedHexLiteral(const llvm::APInt &iOrig) {
897 llvm::APInt i = iOrig.trunc(width: 64);
898 SmallString<40> s;
899 i.toString(Str&: s, Radix: 16, Signed: true, formatAsCLiteral: true);
900 return std::string(s);
901 }
902
903 std::string genSema() const {
904 assert(!headerOnly() && "Called genSema for header-only intrinsic");
905 std::vector<std::string> SemaChecks;
906
907 for (const auto &kv : ImmediateArgs) {
908 const ImmediateArg &IA = kv.second;
909
910 llvm::APInt lo(128, 0), hi(128, 0);
911 switch (IA.boundsType) {
912 case ImmediateArg::BoundsType::ExplicitRange:
913 lo = IA.i1;
914 hi = IA.i2;
915 break;
916 case ImmediateArg::BoundsType::UInt:
917 lo = 0;
918 hi = llvm::APInt::getMaxValue(numBits: IA.i1).zext(width: 128);
919 break;
920 }
921
922 std::string Index = utostr(X: kv.first);
923
924 // Emit a range check if the legal range of values for the
925 // immediate is smaller than the _possible_ range of values for
926 // its type.
927 unsigned ArgTypeBits = IA.ArgType->sizeInBits();
928 llvm::APInt ArgTypeRange = llvm::APInt::getMaxValue(numBits: ArgTypeBits).zext(width: 128);
929 llvm::APInt ActualRange = (hi-lo).trunc(width: 64).sext(width: 128);
930 if (ActualRange.ult(RHS: ArgTypeRange))
931 SemaChecks.push_back(x: "SemaRef.BuiltinConstantArgRange(TheCall, " +
932 Index + ", " + signedHexLiteral(iOrig: lo) + ", " +
933 signedHexLiteral(iOrig: hi) + ")");
934
935 if (!IA.ExtraCheckType.empty()) {
936 std::string Suffix;
937 if (!IA.ExtraCheckArgs.empty()) {
938 std::string tmp;
939 StringRef Arg = IA.ExtraCheckArgs;
940 if (Arg == "!lanesize") {
941 tmp = utostr(X: IA.ArgType->sizeInBits());
942 Arg = tmp;
943 }
944 Suffix = (Twine(", ") + Arg).str();
945 }
946 SemaChecks.push_back(x: (Twine("SemaRef.BuiltinConstantArg") +
947 IA.ExtraCheckType + "(TheCall, " + Index +
948 Suffix + ")")
949 .str());
950 }
951
952 assert(!SemaChecks.empty());
953 }
954 if (SemaChecks.empty())
955 return "";
956 return join(Begin: std::begin(cont&: SemaChecks), End: std::end(cont&: SemaChecks),
957 Separator: " ||\n ") +
958 ";\n";
959 }
960
961 ACLEIntrinsic(EmitterBase &ME, Record *R, const Type *Param);
962};
963
964// -----------------------------------------------------------------------------
965// The top-level class that holds all the state from analyzing the entire
966// Tablegen input.
967
968class EmitterBase {
969protected:
970 // EmitterBase holds a collection of all the types we've instantiated.
971 VoidType Void;
972 std::map<std::string, std::unique_ptr<ScalarType>> ScalarTypes;
973 std::map<std::tuple<ScalarTypeKind, unsigned, unsigned>,
974 std::unique_ptr<VectorType>>
975 VectorTypes;
976 std::map<std::pair<std::string, unsigned>, std::unique_ptr<MultiVectorType>>
977 MultiVectorTypes;
978 std::map<unsigned, std::unique_ptr<PredicateType>> PredicateTypes;
979 std::map<std::string, std::unique_ptr<PointerType>> PointerTypes;
980
981 // And all the ACLEIntrinsic instances we've created.
982 std::map<std::string, std::unique_ptr<ACLEIntrinsic>> ACLEIntrinsics;
983
984public:
985 // Methods to create a Type object, or return the right existing one from the
986 // maps stored in this object.
987 const VoidType *getVoidType() { return &Void; }
988 const ScalarType *getScalarType(StringRef Name) {
989 return ScalarTypes[std::string(Name)].get();
990 }
991 const ScalarType *getScalarType(Record *R) {
992 return getScalarType(Name: R->getName());
993 }
994 const VectorType *getVectorType(const ScalarType *ST, unsigned Lanes) {
995 std::tuple<ScalarTypeKind, unsigned, unsigned> key(ST->kind(),
996 ST->sizeInBits(), Lanes);
997 if (VectorTypes.find(x: key) == VectorTypes.end())
998 VectorTypes[key] = std::make_unique<VectorType>(args&: ST, args&: Lanes);
999 return VectorTypes[key].get();
1000 }
1001 const VectorType *getVectorType(const ScalarType *ST) {
1002 return getVectorType(ST, Lanes: 128 / ST->sizeInBits());
1003 }
1004 const MultiVectorType *getMultiVectorType(unsigned Registers,
1005 const VectorType *VT) {
1006 std::pair<std::string, unsigned> key(VT->cNameBase(), Registers);
1007 if (MultiVectorTypes.find(x: key) == MultiVectorTypes.end())
1008 MultiVectorTypes[key] = std::make_unique<MultiVectorType>(args&: Registers, args&: VT);
1009 return MultiVectorTypes[key].get();
1010 }
1011 const PredicateType *getPredicateType(unsigned Lanes) {
1012 unsigned key = Lanes;
1013 if (PredicateTypes.find(x: key) == PredicateTypes.end())
1014 PredicateTypes[key] = std::make_unique<PredicateType>(args&: Lanes);
1015 return PredicateTypes[key].get();
1016 }
1017 const PointerType *getPointerType(const Type *T, bool Const) {
1018 PointerType PT(T, Const);
1019 std::string key = PT.cName();
1020 if (PointerTypes.find(x: key) == PointerTypes.end())
1021 PointerTypes[key] = std::make_unique<PointerType>(args&: PT);
1022 return PointerTypes[key].get();
1023 }
1024
1025 // Methods to construct a type from various pieces of Tablegen. These are
1026 // always called in the context of setting up a particular ACLEIntrinsic, so
1027 // there's always an ambient parameter type (because we're iterating through
1028 // the Params list in the Tablegen record for the intrinsic), which is used
1029 // to expand Tablegen classes like 'Vector' which mean something different in
1030 // each member of a parametric family.
1031 const Type *getType(Record *R, const Type *Param);
1032 const Type *getType(DagInit *D, const Type *Param);
1033 const Type *getType(Init *I, const Type *Param);
1034
1035 // Functions that translate the Tablegen representation of an intrinsic's
1036 // code generation into a collection of Value objects (which will then be
1037 // reprocessed to read out the actual C++ code included by CGBuiltin.cpp).
1038 Result::Ptr getCodeForDag(DagInit *D, const Result::Scope &Scope,
1039 const Type *Param);
1040 Result::Ptr getCodeForDagArg(DagInit *D, unsigned ArgNum,
1041 const Result::Scope &Scope, const Type *Param);
1042 Result::Ptr getCodeForArg(unsigned ArgNum, const Type *ArgType, bool Promote,
1043 bool Immediate);
1044
1045 void GroupSemaChecks(std::map<std::string, std::set<std::string>> &Checks);
1046
1047 // Constructor and top-level functions.
1048
1049 EmitterBase(RecordKeeper &Records);
1050 virtual ~EmitterBase() = default;
1051
1052 virtual void EmitHeader(raw_ostream &OS) = 0;
1053 virtual void EmitBuiltinDef(raw_ostream &OS) = 0;
1054 virtual void EmitBuiltinSema(raw_ostream &OS) = 0;
1055 void EmitBuiltinCG(raw_ostream &OS);
1056 void EmitBuiltinAliases(raw_ostream &OS);
1057};
1058
1059const Type *EmitterBase::getType(Init *I, const Type *Param) {
1060 if (auto Dag = dyn_cast<DagInit>(Val: I))
1061 return getType(D: Dag, Param);
1062 if (auto Def = dyn_cast<DefInit>(Val: I))
1063 return getType(R: Def->getDef(), Param);
1064
1065 PrintFatalError(Msg: "Could not convert this value into a type");
1066}
1067
1068const Type *EmitterBase::getType(Record *R, const Type *Param) {
1069 // Pass to a subfield of any wrapper records. We don't expect more than one
1070 // of these: immediate operands are used as plain numbers rather than as
1071 // llvm::Value, so it's meaningless to promote their type anyway.
1072 if (R->isSubClassOf(Name: "Immediate"))
1073 R = R->getValueAsDef(FieldName: "type");
1074 else if (R->isSubClassOf(Name: "unpromoted"))
1075 R = R->getValueAsDef(FieldName: "underlying_type");
1076
1077 if (R->getName() == "Void")
1078 return getVoidType();
1079 if (R->isSubClassOf(Name: "PrimitiveType"))
1080 return getScalarType(R);
1081 if (R->isSubClassOf(Name: "ComplexType"))
1082 return getType(D: R->getValueAsDag(FieldName: "spec"), Param);
1083
1084 PrintFatalError(ErrorLoc: R->getLoc(), Msg: "Could not convert this record into a type");
1085}
1086
1087const Type *EmitterBase::getType(DagInit *D, const Type *Param) {
1088 // The meat of the getType system: types in the Tablegen are represented by a
1089 // dag whose operators select sub-cases of this function.
1090
1091 Record *Op = cast<DefInit>(Val: D->getOperator())->getDef();
1092 if (!Op->isSubClassOf(Name: "ComplexTypeOp"))
1093 PrintFatalError(
1094 Msg: "Expected ComplexTypeOp as dag operator in type expression");
1095
1096 if (Op->getName() == "CTO_Parameter") {
1097 if (isa<VoidType>(Val: Param))
1098 PrintFatalError(Msg: "Parametric type in unparametrised context");
1099 return Param;
1100 }
1101
1102 if (Op->getName() == "CTO_Vec") {
1103 const Type *Element = getType(I: D->getArg(Num: 0), Param);
1104 if (D->getNumArgs() == 1) {
1105 return getVectorType(ST: cast<ScalarType>(Val: Element));
1106 } else {
1107 const Type *ExistingVector = getType(I: D->getArg(Num: 1), Param);
1108 return getVectorType(ST: cast<ScalarType>(Val: Element),
1109 Lanes: cast<VectorType>(Val: ExistingVector)->lanes());
1110 }
1111 }
1112
1113 if (Op->getName() == "CTO_Pred") {
1114 const Type *Element = getType(I: D->getArg(Num: 0), Param);
1115 return getPredicateType(Lanes: 128 / Element->sizeInBits());
1116 }
1117
1118 if (Op->isSubClassOf(Name: "CTO_Tuple")) {
1119 unsigned Registers = Op->getValueAsInt(FieldName: "n");
1120 const Type *Element = getType(I: D->getArg(Num: 0), Param);
1121 return getMultiVectorType(Registers, VT: cast<VectorType>(Val: Element));
1122 }
1123
1124 if (Op->isSubClassOf(Name: "CTO_Pointer")) {
1125 const Type *Pointee = getType(I: D->getArg(Num: 0), Param);
1126 return getPointerType(T: Pointee, Const: Op->getValueAsBit(FieldName: "const"));
1127 }
1128
1129 if (Op->getName() == "CTO_CopyKind") {
1130 const ScalarType *STSize = cast<ScalarType>(Val: getType(I: D->getArg(Num: 0), Param));
1131 const ScalarType *STKind = cast<ScalarType>(Val: getType(I: D->getArg(Num: 1), Param));
1132 for (const auto &kv : ScalarTypes) {
1133 const ScalarType *RT = kv.second.get();
1134 if (RT->kind() == STKind->kind() && RT->sizeInBits() == STSize->sizeInBits())
1135 return RT;
1136 }
1137 PrintFatalError(Msg: "Cannot find a type to satisfy CopyKind");
1138 }
1139
1140 if (Op->isSubClassOf(Name: "CTO_ScaleSize")) {
1141 const ScalarType *STKind = cast<ScalarType>(Val: getType(I: D->getArg(Num: 0), Param));
1142 int Num = Op->getValueAsInt(FieldName: "num"), Denom = Op->getValueAsInt(FieldName: "denom");
1143 unsigned DesiredSize = STKind->sizeInBits() * Num / Denom;
1144 for (const auto &kv : ScalarTypes) {
1145 const ScalarType *RT = kv.second.get();
1146 if (RT->kind() == STKind->kind() && RT->sizeInBits() == DesiredSize)
1147 return RT;
1148 }
1149 PrintFatalError(Msg: "Cannot find a type to satisfy ScaleSize");
1150 }
1151
1152 PrintFatalError(Msg: "Bad operator in type dag expression");
1153}
1154
1155Result::Ptr EmitterBase::getCodeForDag(DagInit *D, const Result::Scope &Scope,
1156 const Type *Param) {
1157 Record *Op = cast<DefInit>(Val: D->getOperator())->getDef();
1158
1159 if (Op->getName() == "seq") {
1160 Result::Scope SubScope = Scope;
1161 Result::Ptr PrevV = nullptr;
1162 for (unsigned i = 0, e = D->getNumArgs(); i < e; ++i) {
1163 // We don't use getCodeForDagArg here, because the argument name
1164 // has different semantics in a seq
1165 Result::Ptr V =
1166 getCodeForDag(D: cast<DagInit>(Val: D->getArg(Num: i)), Scope: SubScope, Param);
1167 StringRef ArgName = D->getArgNameStr(Num: i);
1168 if (!ArgName.empty())
1169 SubScope[std::string(ArgName)] = V;
1170 if (PrevV)
1171 V->setPredecessor(PrevV);
1172 PrevV = V;
1173 }
1174 return PrevV;
1175 } else if (Op->isSubClassOf(Name: "Type")) {
1176 if (D->getNumArgs() != 1)
1177 PrintFatalError(Msg: "Type casts should have exactly one argument");
1178 const Type *CastType = getType(R: Op, Param);
1179 Result::Ptr Arg = getCodeForDagArg(D, ArgNum: 0, Scope, Param);
1180 if (const auto *ST = dyn_cast<ScalarType>(Val: CastType)) {
1181 if (!ST->requiresFloat()) {
1182 if (Arg->hasIntegerConstantValue())
1183 return std::make_shared<IntLiteralResult>(
1184 args&: ST, args: Arg->integerConstantValue());
1185 else
1186 return std::make_shared<IntCastResult>(args&: ST, args&: Arg);
1187 }
1188 } else if (const auto *PT = dyn_cast<PointerType>(Val: CastType)) {
1189 return std::make_shared<PointerCastResult>(args&: PT, args&: Arg);
1190 }
1191 PrintFatalError(Msg: "Unsupported type cast");
1192 } else if (Op->getName() == "address") {
1193 if (D->getNumArgs() != 2)
1194 PrintFatalError(Msg: "'address' should have two arguments");
1195 Result::Ptr Arg = getCodeForDagArg(D, ArgNum: 0, Scope, Param);
1196
1197 const Type *Ty = nullptr;
1198 if (auto *DI = dyn_cast<DagInit>(Val: D->getArg(Num: 0)))
1199 if (auto *PTy = dyn_cast<PointerType>(Val: getType(I: DI->getOperator(), Param)))
1200 Ty = PTy->getPointeeType();
1201 if (!Ty)
1202 PrintFatalError(Msg: "'address' pointer argument should be a pointer");
1203
1204 unsigned Alignment;
1205 if (auto *II = dyn_cast<IntInit>(Val: D->getArg(Num: 1))) {
1206 Alignment = II->getValue();
1207 } else {
1208 PrintFatalError(Msg: "'address' alignment argument should be an integer");
1209 }
1210 return std::make_shared<AddressResult>(args&: Arg, args&: Ty, args&: Alignment);
1211 } else if (Op->getName() == "unsignedflag") {
1212 if (D->getNumArgs() != 1)
1213 PrintFatalError(Msg: "unsignedflag should have exactly one argument");
1214 Record *TypeRec = cast<DefInit>(Val: D->getArg(Num: 0))->getDef();
1215 if (!TypeRec->isSubClassOf(Name: "Type"))
1216 PrintFatalError(Msg: "unsignedflag's argument should be a type");
1217 if (const auto *ST = dyn_cast<ScalarType>(Val: getType(R: TypeRec, Param))) {
1218 return std::make_shared<IntLiteralResult>(
1219 args: getScalarType(Name: "u32"), args: ST->kind() == ScalarTypeKind::UnsignedInt);
1220 } else {
1221 PrintFatalError(Msg: "unsignedflag's argument should be a scalar type");
1222 }
1223 } else if (Op->getName() == "bitsize") {
1224 if (D->getNumArgs() != 1)
1225 PrintFatalError(Msg: "bitsize should have exactly one argument");
1226 Record *TypeRec = cast<DefInit>(Val: D->getArg(Num: 0))->getDef();
1227 if (!TypeRec->isSubClassOf(Name: "Type"))
1228 PrintFatalError(Msg: "bitsize's argument should be a type");
1229 if (const auto *ST = dyn_cast<ScalarType>(Val: getType(R: TypeRec, Param))) {
1230 return std::make_shared<IntLiteralResult>(args: getScalarType(Name: "u32"),
1231 args: ST->sizeInBits());
1232 } else {
1233 PrintFatalError(Msg: "bitsize's argument should be a scalar type");
1234 }
1235 } else {
1236 std::vector<Result::Ptr> Args;
1237 for (unsigned i = 0, e = D->getNumArgs(); i < e; ++i)
1238 Args.push_back(x: getCodeForDagArg(D, ArgNum: i, Scope, Param));
1239 if (Op->isSubClassOf(Name: "IRBuilderBase")) {
1240 std::set<unsigned> AddressArgs;
1241 std::map<unsigned, std::string> IntegerArgs;
1242 for (Record *sp : Op->getValueAsListOfDefs(FieldName: "special_params")) {
1243 unsigned Index = sp->getValueAsInt(FieldName: "index");
1244 if (sp->isSubClassOf(Name: "IRBuilderAddrParam")) {
1245 AddressArgs.insert(x: Index);
1246 } else if (sp->isSubClassOf(Name: "IRBuilderIntParam")) {
1247 IntegerArgs[Index] = std::string(sp->getValueAsString(FieldName: "type"));
1248 }
1249 }
1250 return std::make_shared<IRBuilderResult>(args: Op->getValueAsString(FieldName: "prefix"),
1251 args&: Args, args&: AddressArgs, args&: IntegerArgs);
1252 } else if (Op->isSubClassOf(Name: "IRIntBase")) {
1253 std::vector<const Type *> ParamTypes;
1254 for (Record *RParam : Op->getValueAsListOfDefs(FieldName: "params"))
1255 ParamTypes.push_back(x: getType(R: RParam, Param));
1256 std::string IntName = std::string(Op->getValueAsString(FieldName: "intname"));
1257 if (Op->getValueAsBit(FieldName: "appendKind"))
1258 IntName += "_" + toLetter(kind: cast<ScalarType>(Val: Param)->kind());
1259 return std::make_shared<IRIntrinsicResult>(args&: IntName, args&: ParamTypes, args&: Args);
1260 } else {
1261 PrintFatalError(Msg: "Unsupported dag node " + Op->getName());
1262 }
1263 }
1264}
1265
1266Result::Ptr EmitterBase::getCodeForDagArg(DagInit *D, unsigned ArgNum,
1267 const Result::Scope &Scope,
1268 const Type *Param) {
1269 Init *Arg = D->getArg(Num: ArgNum);
1270 StringRef Name = D->getArgNameStr(Num: ArgNum);
1271
1272 if (!Name.empty()) {
1273 if (!isa<UnsetInit>(Val: Arg))
1274 PrintFatalError(
1275 Msg: "dag operator argument should not have both a value and a name");
1276 auto it = Scope.find(x: std::string(Name));
1277 if (it == Scope.end())
1278 PrintFatalError(Msg: "unrecognized variable name '" + Name + "'");
1279 return it->second;
1280 }
1281
1282 // Sometimes the Arg is a bit. Prior to multiclass template argument
1283 // checking, integers would sneak through the bit declaration,
1284 // but now they really are bits.
1285 if (auto *BI = dyn_cast<BitInit>(Val: Arg))
1286 return std::make_shared<IntLiteralResult>(args: getScalarType(Name: "u32"),
1287 args: BI->getValue());
1288
1289 if (auto *II = dyn_cast<IntInit>(Val: Arg))
1290 return std::make_shared<IntLiteralResult>(args: getScalarType(Name: "u32"),
1291 args: II->getValue());
1292
1293 if (auto *DI = dyn_cast<DagInit>(Val: Arg))
1294 return getCodeForDag(D: DI, Scope, Param);
1295
1296 if (auto *DI = dyn_cast<DefInit>(Val: Arg)) {
1297 Record *Rec = DI->getDef();
1298 if (Rec->isSubClassOf(Name: "Type")) {
1299 const Type *T = getType(R: Rec, Param);
1300 return std::make_shared<TypeResult>(args&: T);
1301 }
1302 }
1303
1304 PrintError(Msg: "bad DAG argument type for code generation");
1305 PrintNote(Msg: "DAG: " + D->getAsString());
1306 if (TypedInit *Typed = dyn_cast<TypedInit>(Val: Arg))
1307 PrintNote(Msg: "argument type: " + Typed->getType()->getAsString());
1308 PrintFatalNote(Msg: "argument number " + Twine(ArgNum) + ": " + Arg->getAsString());
1309}
1310
1311Result::Ptr EmitterBase::getCodeForArg(unsigned ArgNum, const Type *ArgType,
1312 bool Promote, bool Immediate) {
1313 Result::Ptr V = std::make_shared<BuiltinArgResult>(
1314 args&: ArgNum, args: isa<PointerType>(Val: ArgType), args&: Immediate);
1315
1316 if (Promote) {
1317 if (const auto *ST = dyn_cast<ScalarType>(Val: ArgType)) {
1318 if (ST->isInteger() && ST->sizeInBits() < 32)
1319 V = std::make_shared<IntCastResult>(args: getScalarType(Name: "u32"), args&: V);
1320 } else if (const auto *PT = dyn_cast<PredicateType>(Val: ArgType)) {
1321 V = std::make_shared<IntCastResult>(args: getScalarType(Name: "u32"), args&: V);
1322 V = std::make_shared<IRIntrinsicResult>(args: "arm_mve_pred_i2v",
1323 args: std::vector<const Type *>{PT},
1324 args: std::vector<Result::Ptr>{V});
1325 }
1326 }
1327
1328 return V;
1329}
1330
1331ACLEIntrinsic::ACLEIntrinsic(EmitterBase &ME, Record *R, const Type *Param)
1332 : ReturnType(ME.getType(R: R->getValueAsDef(FieldName: "ret"), Param)) {
1333 // Derive the intrinsic's full name, by taking the name of the
1334 // Tablegen record (or override) and appending the suffix from its
1335 // parameter type. (If the intrinsic is unparametrised, its
1336 // parameter type will be given as Void, which returns the empty
1337 // string for acleSuffix.)
1338 StringRef BaseName =
1339 (R->isSubClassOf(Name: "NameOverride") ? R->getValueAsString(FieldName: "basename")
1340 : R->getName());
1341 StringRef overrideLetter = R->getValueAsString(FieldName: "overrideKindLetter");
1342 FullName =
1343 (Twine(BaseName) + Param->acleSuffix(std::string(overrideLetter))).str();
1344
1345 // Derive the intrinsic's polymorphic name, by removing components from the
1346 // full name as specified by its 'pnt' member ('polymorphic name type'),
1347 // which indicates how many type suffixes to remove, and any other piece of
1348 // the name that should be removed.
1349 Record *PolymorphicNameType = R->getValueAsDef(FieldName: "pnt");
1350 SmallVector<StringRef, 8> NameParts;
1351 StringRef(FullName).split(A&: NameParts, Separator: '_');
1352 for (unsigned i = 0, e = PolymorphicNameType->getValueAsInt(
1353 FieldName: "NumTypeSuffixesToDiscard");
1354 i < e; ++i)
1355 NameParts.pop_back();
1356 if (!PolymorphicNameType->isValueUnset(FieldName: "ExtraSuffixToDiscard")) {
1357 StringRef ExtraSuffix =
1358 PolymorphicNameType->getValueAsString(FieldName: "ExtraSuffixToDiscard");
1359 auto it = NameParts.end();
1360 while (it != NameParts.begin()) {
1361 --it;
1362 if (*it == ExtraSuffix) {
1363 NameParts.erase(CI: it);
1364 break;
1365 }
1366 }
1367 }
1368 ShortName = join(Begin: std::begin(cont&: NameParts), End: std::end(cont&: NameParts), Separator: "_");
1369
1370 BuiltinExtension = R->getValueAsString(FieldName: "builtinExtension");
1371
1372 PolymorphicOnly = R->getValueAsBit(FieldName: "polymorphicOnly");
1373 NonEvaluating = R->getValueAsBit(FieldName: "nonEvaluating");
1374 HeaderOnly = R->getValueAsBit(FieldName: "headerOnly");
1375
1376 // Process the intrinsic's argument list.
1377 DagInit *ArgsDag = R->getValueAsDag(FieldName: "args");
1378 Result::Scope Scope;
1379 for (unsigned i = 0, e = ArgsDag->getNumArgs(); i < e; ++i) {
1380 Init *TypeInit = ArgsDag->getArg(Num: i);
1381
1382 bool Promote = true;
1383 if (auto TypeDI = dyn_cast<DefInit>(Val: TypeInit))
1384 if (TypeDI->getDef()->isSubClassOf(Name: "unpromoted"))
1385 Promote = false;
1386
1387 // Work out the type of the argument, for use in the function prototype in
1388 // the header file.
1389 const Type *ArgType = ME.getType(I: TypeInit, Param);
1390 ArgTypes.push_back(x: ArgType);
1391
1392 // If the argument is a subclass of Immediate, record the details about
1393 // what values it can take, for Sema checking.
1394 bool Immediate = false;
1395 if (auto TypeDI = dyn_cast<DefInit>(Val: TypeInit)) {
1396 Record *TypeRec = TypeDI->getDef();
1397 if (TypeRec->isSubClassOf(Name: "Immediate")) {
1398 Immediate = true;
1399
1400 Record *Bounds = TypeRec->getValueAsDef(FieldName: "bounds");
1401 ImmediateArg &IA = ImmediateArgs[i];
1402 if (Bounds->isSubClassOf(Name: "IB_ConstRange")) {
1403 IA.boundsType = ImmediateArg::BoundsType::ExplicitRange;
1404 IA.i1 = Bounds->getValueAsInt(FieldName: "lo");
1405 IA.i2 = Bounds->getValueAsInt(FieldName: "hi");
1406 } else if (Bounds->getName() == "IB_UEltValue") {
1407 IA.boundsType = ImmediateArg::BoundsType::UInt;
1408 IA.i1 = Param->sizeInBits();
1409 } else if (Bounds->getName() == "IB_LaneIndex") {
1410 IA.boundsType = ImmediateArg::BoundsType::ExplicitRange;
1411 IA.i1 = 0;
1412 IA.i2 = 128 / Param->sizeInBits() - 1;
1413 } else if (Bounds->isSubClassOf(Name: "IB_EltBit")) {
1414 IA.boundsType = ImmediateArg::BoundsType::ExplicitRange;
1415 IA.i1 = Bounds->getValueAsInt(FieldName: "base");
1416 const Type *T = ME.getType(R: Bounds->getValueAsDef(FieldName: "type"), Param);
1417 IA.i2 = IA.i1 + T->sizeInBits() - 1;
1418 } else {
1419 PrintFatalError(Msg: "unrecognised ImmediateBounds subclass");
1420 }
1421
1422 IA.ArgType = ArgType;
1423
1424 if (!TypeRec->isValueUnset(FieldName: "extra")) {
1425 IA.ExtraCheckType = TypeRec->getValueAsString(FieldName: "extra");
1426 if (!TypeRec->isValueUnset(FieldName: "extraarg"))
1427 IA.ExtraCheckArgs = TypeRec->getValueAsString(FieldName: "extraarg");
1428 }
1429 }
1430 }
1431
1432 // The argument will usually have a name in the arguments dag, which goes
1433 // into the variable-name scope that the code gen will refer to.
1434 StringRef ArgName = ArgsDag->getArgNameStr(Num: i);
1435 if (!ArgName.empty())
1436 Scope[std::string(ArgName)] =
1437 ME.getCodeForArg(ArgNum: i, ArgType, Promote, Immediate);
1438 }
1439
1440 // Finally, go through the codegen dag and translate it into a Result object
1441 // (with an arbitrary DAG of depended-on Results hanging off it).
1442 DagInit *CodeDag = R->getValueAsDag(FieldName: "codegen");
1443 Record *MainOp = cast<DefInit>(Val: CodeDag->getOperator())->getDef();
1444 if (MainOp->isSubClassOf(Name: "CustomCodegen")) {
1445 // Or, if it's the special case of CustomCodegen, just accumulate
1446 // a list of parameters we're going to assign to variables before
1447 // breaking from the loop.
1448 CustomCodeGenArgs["CustomCodeGenType"] =
1449 (Twine("CustomCodeGen::") + MainOp->getValueAsString(FieldName: "type")).str();
1450 for (unsigned i = 0, e = CodeDag->getNumArgs(); i < e; ++i) {
1451 StringRef Name = CodeDag->getArgNameStr(Num: i);
1452 if (Name.empty()) {
1453 PrintFatalError(Msg: "Operands to CustomCodegen should have names");
1454 } else if (auto *II = dyn_cast<IntInit>(Val: CodeDag->getArg(Num: i))) {
1455 CustomCodeGenArgs[std::string(Name)] = itostr(X: II->getValue());
1456 } else if (auto *SI = dyn_cast<StringInit>(Val: CodeDag->getArg(Num: i))) {
1457 CustomCodeGenArgs[std::string(Name)] = std::string(SI->getValue());
1458 } else {
1459 PrintFatalError(Msg: "Operands to CustomCodegen should be integers");
1460 }
1461 }
1462 } else {
1463 Code = ME.getCodeForDag(D: CodeDag, Scope, Param);
1464 }
1465}
1466
1467EmitterBase::EmitterBase(RecordKeeper &Records) {
1468 // Construct the whole EmitterBase.
1469
1470 // First, look up all the instances of PrimitiveType. This gives us the list
1471 // of vector typedefs we have to put in arm_mve.h, and also allows us to
1472 // collect all the useful ScalarType instances into a big list so that we can
1473 // use it for operations such as 'find the unsigned version of this signed
1474 // integer type'.
1475 for (Record *R : Records.getAllDerivedDefinitions(ClassName: "PrimitiveType"))
1476 ScalarTypes[std::string(R->getName())] = std::make_unique<ScalarType>(args&: R);
1477
1478 // Now go through the instances of Intrinsic, and for each one, iterate
1479 // through its list of type parameters making an ACLEIntrinsic for each one.
1480 for (Record *R : Records.getAllDerivedDefinitions(ClassName: "Intrinsic")) {
1481 for (Record *RParam : R->getValueAsListOfDefs(FieldName: "params")) {
1482 const Type *Param = getType(R: RParam, Param: getVoidType());
1483 auto Intrinsic = std::make_unique<ACLEIntrinsic>(args&: *this, args&: R, args&: Param);
1484 ACLEIntrinsics[Intrinsic->fullName()] = std::move(Intrinsic);
1485 }
1486 }
1487}
1488
1489/// A wrapper on raw_string_ostream that contains its own buffer rather than
1490/// having to point it at one elsewhere. (In other words, it works just like
1491/// std::ostringstream; also, this makes it convenient to declare a whole array
1492/// of them at once.)
1493///
1494/// We have to set this up using multiple inheritance, to ensure that the
1495/// string member has been constructed before raw_string_ostream's constructor
1496/// is given a pointer to it.
1497class string_holder {
1498protected:
1499 std::string S;
1500};
1501class raw_self_contained_string_ostream : private string_holder,
1502 public raw_string_ostream {
1503public:
1504 raw_self_contained_string_ostream() : raw_string_ostream(S) {}
1505};
1506
1507const char LLVMLicenseHeader[] =
1508 " *\n"
1509 " *\n"
1510 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM"
1511 " Exceptions.\n"
1512 " * See https://llvm.org/LICENSE.txt for license information.\n"
1513 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1514 " *\n"
1515 " *===-----------------------------------------------------------------"
1516 "------===\n"
1517 " */\n"
1518 "\n";
1519
1520// Machinery for the grouping of intrinsics by similar codegen.
1521//
1522// The general setup is that 'MergeableGroup' stores the things that a set of
1523// similarly shaped intrinsics have in common: the text of their code
1524// generation, and the number and type of their parameter variables.
1525// MergeableGroup is the key in a std::map whose value is a set of
1526// OutputIntrinsic, which stores the ways in which a particular intrinsic
1527// specializes the MergeableGroup's generic description: the function name and
1528// the _values_ of the parameter variables.
1529
1530struct ComparableStringVector : std::vector<std::string> {
1531 // Infrastructure: a derived class of vector<string> which comes with an
1532 // ordering, so that it can be used as a key in maps and an element in sets.
1533 // There's no requirement on the ordering beyond being deterministic.
1534 bool operator<(const ComparableStringVector &rhs) const {
1535 if (size() != rhs.size())
1536 return size() < rhs.size();
1537 for (size_t i = 0, e = size(); i < e; ++i)
1538 if ((*this)[i] != rhs[i])
1539 return (*this)[i] < rhs[i];
1540 return false;
1541 }
1542};
1543
1544struct OutputIntrinsic {
1545 const ACLEIntrinsic *Int;
1546 std::string Name;
1547 ComparableStringVector ParamValues;
1548 bool operator<(const OutputIntrinsic &rhs) const {
1549 if (Name != rhs.Name)
1550 return Name < rhs.Name;
1551 return ParamValues < rhs.ParamValues;
1552 }
1553};
1554struct MergeableGroup {
1555 std::string Code;
1556 ComparableStringVector ParamTypes;
1557 bool operator<(const MergeableGroup &rhs) const {
1558 if (Code != rhs.Code)
1559 return Code < rhs.Code;
1560 return ParamTypes < rhs.ParamTypes;
1561 }
1562};
1563
1564void EmitterBase::EmitBuiltinCG(raw_ostream &OS) {
1565 // Pass 1: generate code for all the intrinsics as if every type or constant
1566 // that can possibly be abstracted out into a parameter variable will be.
1567 // This identifies the sets of intrinsics we'll group together into a single
1568 // piece of code generation.
1569
1570 std::map<MergeableGroup, std::set<OutputIntrinsic>> MergeableGroupsPrelim;
1571
1572 for (const auto &kv : ACLEIntrinsics) {
1573 const ACLEIntrinsic &Int = *kv.second;
1574 if (Int.headerOnly())
1575 continue;
1576
1577 MergeableGroup MG;
1578 OutputIntrinsic OI;
1579
1580 OI.Int = &Int;
1581 OI.Name = Int.fullName();
1582 CodeGenParamAllocator ParamAllocPrelim{.ParamTypes: &MG.ParamTypes, .ParamValues: &OI.ParamValues};
1583 raw_string_ostream OS(MG.Code);
1584 Int.genCode(OS, ParamAlloc&: ParamAllocPrelim, Pass: 1);
1585 OS.flush();
1586
1587 MergeableGroupsPrelim[MG].insert(x: OI);
1588 }
1589
1590 // Pass 2: for each of those groups, optimize the parameter variable set by
1591 // eliminating 'parameters' that are the same for all intrinsics in the
1592 // group, and merging together pairs of parameter variables that take the
1593 // same values as each other for all intrinsics in the group.
1594
1595 std::map<MergeableGroup, std::set<OutputIntrinsic>> MergeableGroups;
1596
1597 for (const auto &kv : MergeableGroupsPrelim) {
1598 const MergeableGroup &MG = kv.first;
1599 std::vector<int> ParamNumbers;
1600 std::map<ComparableStringVector, int> ParamNumberMap;
1601
1602 // Loop over the parameters for this group.
1603 for (size_t i = 0, e = MG.ParamTypes.size(); i < e; ++i) {
1604 // Is this parameter the same for all intrinsics in the group?
1605 const OutputIntrinsic &OI_first = *kv.second.begin();
1606 bool Constant = all_of(Range: kv.second, P: [&](const OutputIntrinsic &OI) {
1607 return OI.ParamValues[i] == OI_first.ParamValues[i];
1608 });
1609
1610 // If so, record it as -1, meaning 'no parameter variable needed'. Then
1611 // the corresponding call to allocParam in pass 2 will not generate a
1612 // variable at all, and just use the value inline.
1613 if (Constant) {
1614 ParamNumbers.push_back(x: -1);
1615 continue;
1616 }
1617
1618 // Otherwise, make a list of the values this parameter takes for each
1619 // intrinsic, and see if that value vector matches anything we already
1620 // have. We also record the parameter type, so that we don't accidentally
1621 // match up two parameter variables with different types. (Not that
1622 // there's much chance of them having textually equivalent values, but in
1623 // _principle_ it could happen.)
1624 ComparableStringVector key;
1625 key.push_back(x: MG.ParamTypes[i]);
1626 for (const auto &OI : kv.second)
1627 key.push_back(x: OI.ParamValues[i]);
1628
1629 auto Found = ParamNumberMap.find(x: key);
1630 if (Found != ParamNumberMap.end()) {
1631 // Yes, an existing parameter variable can be reused for this.
1632 ParamNumbers.push_back(x: Found->second);
1633 continue;
1634 }
1635
1636 // No, we need a new parameter variable.
1637 int ExistingIndex = ParamNumberMap.size();
1638 ParamNumberMap[key] = ExistingIndex;
1639 ParamNumbers.push_back(x: ExistingIndex);
1640 }
1641
1642 // Now we're ready to do the pass 2 code generation, which will emit the
1643 // reduced set of parameter variables we've just worked out.
1644
1645 for (const auto &OI_prelim : kv.second) {
1646 const ACLEIntrinsic *Int = OI_prelim.Int;
1647
1648 MergeableGroup MG;
1649 OutputIntrinsic OI;
1650
1651 OI.Int = OI_prelim.Int;
1652 OI.Name = OI_prelim.Name;
1653 CodeGenParamAllocator ParamAlloc{.ParamTypes: &MG.ParamTypes, .ParamValues: &OI.ParamValues,
1654 .ParamNumberMap: &ParamNumbers};
1655 raw_string_ostream OS(MG.Code);
1656 Int->genCode(OS, ParamAlloc, Pass: 2);
1657 OS.flush();
1658
1659 MergeableGroups[MG].insert(x: OI);
1660 }
1661 }
1662
1663 // Output the actual C++ code.
1664
1665 for (const auto &kv : MergeableGroups) {
1666 const MergeableGroup &MG = kv.first;
1667
1668 // List of case statements in the main switch on BuiltinID, and an open
1669 // brace.
1670 const char *prefix = "";
1671 for (const auto &OI : kv.second) {
1672 OS << prefix << "case ARM::BI__builtin_arm_" << OI.Int->builtinExtension()
1673 << "_" << OI.Name << ":";
1674
1675 prefix = "\n";
1676 }
1677 OS << " {\n";
1678
1679 if (!MG.ParamTypes.empty()) {
1680 // If we've got some parameter variables, then emit their declarations...
1681 for (size_t i = 0, e = MG.ParamTypes.size(); i < e; ++i) {
1682 StringRef Type = MG.ParamTypes[i];
1683 OS << " " << Type;
1684 if (!Type.ends_with(Suffix: "*"))
1685 OS << " ";
1686 OS << " Param" << utostr(X: i) << ";\n";
1687 }
1688
1689 // ... and an inner switch on BuiltinID that will fill them in with each
1690 // individual intrinsic's values.
1691 OS << " switch (BuiltinID) {\n";
1692 for (const auto &OI : kv.second) {
1693 OS << " case ARM::BI__builtin_arm_" << OI.Int->builtinExtension()
1694 << "_" << OI.Name << ":\n";
1695 for (size_t i = 0, e = MG.ParamTypes.size(); i < e; ++i)
1696 OS << " Param" << utostr(X: i) << " = " << OI.ParamValues[i] << ";\n";
1697 OS << " break;\n";
1698 }
1699 OS << " }\n";
1700 }
1701
1702 // And finally, output the code, and close the outer pair of braces. (The
1703 // code will always end with a 'return' statement, so we need not insert a
1704 // 'break' here.)
1705 OS << MG.Code << "}\n";
1706 }
1707}
1708
1709void EmitterBase::EmitBuiltinAliases(raw_ostream &OS) {
1710 // Build a sorted table of:
1711 // - intrinsic id number
1712 // - full name
1713 // - polymorphic name or -1
1714 StringToOffsetTable StringTable;
1715 OS << "static const IntrinToName MapData[] = {\n";
1716 for (const auto &kv : ACLEIntrinsics) {
1717 const ACLEIntrinsic &Int = *kv.second;
1718 if (Int.headerOnly())
1719 continue;
1720 int32_t ShortNameOffset =
1721 Int.polymorphic() ? StringTable.GetOrAddStringOffset(Str: Int.shortName())
1722 : -1;
1723 OS << " { ARM::BI__builtin_arm_" << Int.builtinExtension() << "_"
1724 << Int.fullName() << ", "
1725 << StringTable.GetOrAddStringOffset(Str: Int.fullName()) << ", "
1726 << ShortNameOffset << "},\n";
1727 }
1728 OS << "};\n\n";
1729
1730 OS << "ArrayRef<IntrinToName> Map(MapData);\n\n";
1731
1732 OS << "static const char IntrinNames[] = {\n";
1733 StringTable.EmitString(O&: OS);
1734 OS << "};\n\n";
1735}
1736
1737void EmitterBase::GroupSemaChecks(
1738 std::map<std::string, std::set<std::string>> &Checks) {
1739 for (const auto &kv : ACLEIntrinsics) {
1740 const ACLEIntrinsic &Int = *kv.second;
1741 if (Int.headerOnly())
1742 continue;
1743 std::string Check = Int.genSema();
1744 if (!Check.empty())
1745 Checks[Check].insert(x: Int.fullName());
1746 }
1747}
1748
1749// -----------------------------------------------------------------------------
1750// The class used for generating arm_mve.h and related Clang bits
1751//
1752
1753class MveEmitter : public EmitterBase {
1754public:
1755 MveEmitter(RecordKeeper &Records) : EmitterBase(Records){};
1756 void EmitHeader(raw_ostream &OS) override;
1757 void EmitBuiltinDef(raw_ostream &OS) override;
1758 void EmitBuiltinSema(raw_ostream &OS) override;
1759};
1760
1761void MveEmitter::EmitHeader(raw_ostream &OS) {
1762 // Accumulate pieces of the header file that will be enabled under various
1763 // different combinations of #ifdef. The index into parts[] is made up of
1764 // the following bit flags.
1765 constexpr unsigned Float = 1;
1766 constexpr unsigned UseUserNamespace = 2;
1767
1768 constexpr unsigned NumParts = 4;
1769 raw_self_contained_string_ostream parts[NumParts];
1770
1771 // Write typedefs for all the required vector types, and a few scalar
1772 // types that don't already have the name we want them to have.
1773
1774 parts[0] << "typedef uint16_t mve_pred16_t;\n";
1775 parts[Float] << "typedef __fp16 float16_t;\n"
1776 "typedef float float32_t;\n";
1777 for (const auto &kv : ScalarTypes) {
1778 const ScalarType *ST = kv.second.get();
1779 if (ST->hasNonstandardName())
1780 continue;
1781 raw_ostream &OS = parts[ST->requiresFloat() ? Float : 0];
1782 const VectorType *VT = getVectorType(ST);
1783
1784 OS << "typedef __attribute__((__neon_vector_type__(" << VT->lanes()
1785 << "), __clang_arm_mve_strict_polymorphism)) " << ST->cName() << " "
1786 << VT->cName() << ";\n";
1787
1788 // Every vector type also comes with a pair of multi-vector types for
1789 // the VLD2 and VLD4 instructions.
1790 for (unsigned n = 2; n <= 4; n += 2) {
1791 const MultiVectorType *MT = getMultiVectorType(Registers: n, VT);
1792 OS << "typedef struct { " << VT->cName() << " val[" << n << "]; } "
1793 << MT->cName() << ";\n";
1794 }
1795 }
1796 parts[0] << "\n";
1797 parts[Float] << "\n";
1798
1799 // Write declarations for all the intrinsics.
1800
1801 for (const auto &kv : ACLEIntrinsics) {
1802 const ACLEIntrinsic &Int = *kv.second;
1803
1804 // We generate each intrinsic twice, under its full unambiguous
1805 // name and its shorter polymorphic name (if the latter exists).
1806 for (bool Polymorphic : {false, true}) {
1807 if (Polymorphic && !Int.polymorphic())
1808 continue;
1809 if (!Polymorphic && Int.polymorphicOnly())
1810 continue;
1811
1812 // We also generate each intrinsic under a name like __arm_vfooq
1813 // (which is in C language implementation namespace, so it's
1814 // safe to define in any conforming user program) and a shorter
1815 // one like vfooq (which is in user namespace, so a user might
1816 // reasonably have used it for something already). If so, they
1817 // can #define __ARM_MVE_PRESERVE_USER_NAMESPACE before
1818 // including the header, which will suppress the shorter names
1819 // and leave only the implementation-namespace ones. Then they
1820 // have to write __arm_vfooq everywhere, of course.
1821
1822 for (bool UserNamespace : {false, true}) {
1823 raw_ostream &OS = parts[(Int.requiresFloat() ? Float : 0) |
1824 (UserNamespace ? UseUserNamespace : 0)];
1825
1826 // Make the name of the function in this declaration.
1827
1828 std::string FunctionName =
1829 Polymorphic ? Int.shortName() : Int.fullName();
1830 if (!UserNamespace)
1831 FunctionName = "__arm_" + FunctionName;
1832
1833 // Make strings for the types involved in the function's
1834 // prototype.
1835
1836 std::string RetTypeName = Int.returnType()->cName();
1837 if (!StringRef(RetTypeName).ends_with(Suffix: "*"))
1838 RetTypeName += " ";
1839
1840 std::vector<std::string> ArgTypeNames;
1841 for (const Type *ArgTypePtr : Int.argTypes())
1842 ArgTypeNames.push_back(x: ArgTypePtr->cName());
1843 std::string ArgTypesString =
1844 join(Begin: std::begin(cont&: ArgTypeNames), End: std::end(cont&: ArgTypeNames), Separator: ", ");
1845
1846 // Emit the actual declaration. All these functions are
1847 // declared 'static inline' without a body, which is fine
1848 // provided clang recognizes them as builtins, and has the
1849 // effect that this type signature is used in place of the one
1850 // that Builtins.td didn't provide. That's how we can get
1851 // structure types that weren't defined until this header was
1852 // included to be part of the type signature of a builtin that
1853 // was known to clang already.
1854 //
1855 // The declarations use __attribute__(__clang_arm_builtin_alias),
1856 // so that each function declared will be recognized as the
1857 // appropriate MVE builtin in spite of its user-facing name.
1858 //
1859 // (That's better than making them all wrapper functions,
1860 // partly because it avoids any compiler error message citing
1861 // the wrapper function definition instead of the user's code,
1862 // and mostly because some MVE intrinsics have arguments
1863 // required to be compile-time constants, and that property
1864 // can't be propagated through a wrapper function. It can be
1865 // propagated through a macro, but macros can't be overloaded
1866 // on argument types very easily - you have to use _Generic,
1867 // which makes error messages very confusing when the user
1868 // gets it wrong.)
1869 //
1870 // Finally, the polymorphic versions of the intrinsics are
1871 // also defined with __attribute__(overloadable), so that when
1872 // the same name is defined with several type signatures, the
1873 // right thing happens. Each one of the overloaded
1874 // declarations is given a different builtin id, which
1875 // has exactly the effect we want: first clang resolves the
1876 // overload to the right function, then it knows which builtin
1877 // it's referring to, and then the Sema checking for that
1878 // builtin can check further things like the constant
1879 // arguments.
1880 //
1881 // One more subtlety is the newline just before the return
1882 // type name. That's a cosmetic tweak to make the error
1883 // messages legible if the user gets the types wrong in a call
1884 // to a polymorphic function: this way, clang will print just
1885 // the _final_ line of each declaration in the header, to show
1886 // the type signatures that would have been legal. So all the
1887 // confusing machinery with __attribute__ is left out of the
1888 // error message, and the user sees something that's more or
1889 // less self-documenting: "here's a list of actually readable
1890 // type signatures for vfooq(), and here's why each one didn't
1891 // match your call".
1892
1893 OS << "static __inline__ __attribute__(("
1894 << (Polymorphic ? "__overloadable__, " : "")
1895 << "__clang_arm_builtin_alias(__builtin_arm_mve_" << Int.fullName()
1896 << ")))\n"
1897 << RetTypeName << FunctionName << "(" << ArgTypesString << ");\n";
1898 }
1899 }
1900 }
1901 for (auto &part : parts)
1902 part << "\n";
1903
1904 // Now we've finished accumulating bits and pieces into the parts[] array.
1905 // Put it all together to write the final output file.
1906
1907 OS << "/*===---- arm_mve.h - ARM MVE intrinsics "
1908 "-----------------------------------===\n"
1909 << LLVMLicenseHeader
1910 << "#ifndef __ARM_MVE_H\n"
1911 "#define __ARM_MVE_H\n"
1912 "\n"
1913 "#if !__ARM_FEATURE_MVE\n"
1914 "#error \"MVE support not enabled\"\n"
1915 "#endif\n"
1916 "\n"
1917 "#include <stdint.h>\n"
1918 "\n"
1919 "#ifdef __cplusplus\n"
1920 "extern \"C\" {\n"
1921 "#endif\n"
1922 "\n";
1923
1924 for (size_t i = 0; i < NumParts; ++i) {
1925 std::vector<std::string> conditions;
1926 if (i & Float)
1927 conditions.push_back(x: "(__ARM_FEATURE_MVE & 2)");
1928 if (i & UseUserNamespace)
1929 conditions.push_back(x: "(!defined __ARM_MVE_PRESERVE_USER_NAMESPACE)");
1930
1931 std::string condition =
1932 join(Begin: std::begin(cont&: conditions), End: std::end(cont&: conditions), Separator: " && ");
1933 if (!condition.empty())
1934 OS << "#if " << condition << "\n\n";
1935 OS << parts[i].str();
1936 if (!condition.empty())
1937 OS << "#endif /* " << condition << " */\n\n";
1938 }
1939
1940 OS << "#ifdef __cplusplus\n"
1941 "} /* extern \"C\" */\n"
1942 "#endif\n"
1943 "\n"
1944 "#endif /* __ARM_MVE_H */\n";
1945}
1946
1947void MveEmitter::EmitBuiltinDef(raw_ostream &OS) {
1948 for (const auto &kv : ACLEIntrinsics) {
1949 const ACLEIntrinsic &Int = *kv.second;
1950 OS << "BUILTIN(__builtin_arm_mve_" << Int.fullName()
1951 << ", \"\", \"n\")\n";
1952 }
1953
1954 std::set<std::string> ShortNamesSeen;
1955
1956 for (const auto &kv : ACLEIntrinsics) {
1957 const ACLEIntrinsic &Int = *kv.second;
1958 if (Int.polymorphic()) {
1959 StringRef Name = Int.shortName();
1960 if (ShortNamesSeen.find(x: std::string(Name)) == ShortNamesSeen.end()) {
1961 OS << "BUILTIN(__builtin_arm_mve_" << Name << ", \"vi.\", \"nt";
1962 if (Int.nonEvaluating())
1963 OS << "u"; // indicate that this builtin doesn't evaluate its args
1964 OS << "\")\n";
1965 ShortNamesSeen.insert(x: std::string(Name));
1966 }
1967 }
1968 }
1969}
1970
1971void MveEmitter::EmitBuiltinSema(raw_ostream &OS) {
1972 std::map<std::string, std::set<std::string>> Checks;
1973 GroupSemaChecks(Checks);
1974
1975 for (const auto &kv : Checks) {
1976 for (StringRef Name : kv.second)
1977 OS << "case ARM::BI__builtin_arm_mve_" << Name << ":\n";
1978 OS << " return " << kv.first;
1979 }
1980}
1981
1982// -----------------------------------------------------------------------------
1983// Class that describes an ACLE intrinsic implemented as a macro.
1984//
1985// This class is used when the intrinsic is polymorphic in 2 or 3 types, but we
1986// want to avoid a combinatorial explosion by reinterpreting the arguments to
1987// fixed types.
1988
1989class FunctionMacro {
1990 std::vector<StringRef> Params;
1991 StringRef Definition;
1992
1993public:
1994 FunctionMacro(const Record &R);
1995
1996 const std::vector<StringRef> &getParams() const { return Params; }
1997 StringRef getDefinition() const { return Definition; }
1998};
1999
2000FunctionMacro::FunctionMacro(const Record &R) {
2001 Params = R.getValueAsListOfStrings(FieldName: "params");
2002 Definition = R.getValueAsString(FieldName: "definition");
2003}
2004
2005// -----------------------------------------------------------------------------
2006// The class used for generating arm_cde.h and related Clang bits
2007//
2008
2009class CdeEmitter : public EmitterBase {
2010 std::map<StringRef, FunctionMacro> FunctionMacros;
2011
2012public:
2013 CdeEmitter(RecordKeeper &Records);
2014 void EmitHeader(raw_ostream &OS) override;
2015 void EmitBuiltinDef(raw_ostream &OS) override;
2016 void EmitBuiltinSema(raw_ostream &OS) override;
2017};
2018
2019CdeEmitter::CdeEmitter(RecordKeeper &Records) : EmitterBase(Records) {
2020 for (Record *R : Records.getAllDerivedDefinitions(ClassName: "FunctionMacro"))
2021 FunctionMacros.emplace(args: R->getName(), args: FunctionMacro(*R));
2022}
2023
2024void CdeEmitter::EmitHeader(raw_ostream &OS) {
2025 // Accumulate pieces of the header file that will be enabled under various
2026 // different combinations of #ifdef. The index into parts[] is one of the
2027 // following:
2028 constexpr unsigned None = 0;
2029 constexpr unsigned MVE = 1;
2030 constexpr unsigned MVEFloat = 2;
2031
2032 constexpr unsigned NumParts = 3;
2033 raw_self_contained_string_ostream parts[NumParts];
2034
2035 // Write typedefs for all the required vector types, and a few scalar
2036 // types that don't already have the name we want them to have.
2037
2038 parts[MVE] << "typedef uint16_t mve_pred16_t;\n";
2039 parts[MVEFloat] << "typedef __fp16 float16_t;\n"
2040 "typedef float float32_t;\n";
2041 for (const auto &kv : ScalarTypes) {
2042 const ScalarType *ST = kv.second.get();
2043 if (ST->hasNonstandardName())
2044 continue;
2045 // We don't have float64x2_t
2046 if (ST->kind() == ScalarTypeKind::Float && ST->sizeInBits() == 64)
2047 continue;
2048 raw_ostream &OS = parts[ST->requiresFloat() ? MVEFloat : MVE];
2049 const VectorType *VT = getVectorType(ST);
2050
2051 OS << "typedef __attribute__((__neon_vector_type__(" << VT->lanes()
2052 << "), __clang_arm_mve_strict_polymorphism)) " << ST->cName() << " "
2053 << VT->cName() << ";\n";
2054 }
2055 parts[MVE] << "\n";
2056 parts[MVEFloat] << "\n";
2057
2058 // Write declarations for all the intrinsics.
2059
2060 for (const auto &kv : ACLEIntrinsics) {
2061 const ACLEIntrinsic &Int = *kv.second;
2062
2063 // We generate each intrinsic twice, under its full unambiguous
2064 // name and its shorter polymorphic name (if the latter exists).
2065 for (bool Polymorphic : {false, true}) {
2066 if (Polymorphic && !Int.polymorphic())
2067 continue;
2068 if (!Polymorphic && Int.polymorphicOnly())
2069 continue;
2070
2071 raw_ostream &OS =
2072 parts[Int.requiresFloat() ? MVEFloat
2073 : Int.requiresMVE() ? MVE : None];
2074
2075 // Make the name of the function in this declaration.
2076 std::string FunctionName =
2077 "__arm_" + (Polymorphic ? Int.shortName() : Int.fullName());
2078
2079 // Make strings for the types involved in the function's
2080 // prototype.
2081 std::string RetTypeName = Int.returnType()->cName();
2082 if (!StringRef(RetTypeName).ends_with(Suffix: "*"))
2083 RetTypeName += " ";
2084
2085 std::vector<std::string> ArgTypeNames;
2086 for (const Type *ArgTypePtr : Int.argTypes())
2087 ArgTypeNames.push_back(x: ArgTypePtr->cName());
2088 std::string ArgTypesString =
2089 join(Begin: std::begin(cont&: ArgTypeNames), End: std::end(cont&: ArgTypeNames), Separator: ", ");
2090
2091 // Emit the actual declaration. See MveEmitter::EmitHeader for detailed
2092 // comments
2093 OS << "static __inline__ __attribute__(("
2094 << (Polymorphic ? "__overloadable__, " : "")
2095 << "__clang_arm_builtin_alias(__builtin_arm_" << Int.builtinExtension()
2096 << "_" << Int.fullName() << ")))\n"
2097 << RetTypeName << FunctionName << "(" << ArgTypesString << ");\n";
2098 }
2099 }
2100
2101 for (const auto &kv : FunctionMacros) {
2102 StringRef Name = kv.first;
2103 const FunctionMacro &FM = kv.second;
2104
2105 raw_ostream &OS = parts[MVE];
2106 OS << "#define "
2107 << "__arm_" << Name << "(" << join(R: FM.getParams(), Separator: ", ") << ") "
2108 << FM.getDefinition() << "\n";
2109 }
2110
2111 for (auto &part : parts)
2112 part << "\n";
2113
2114 // Now we've finished accumulating bits and pieces into the parts[] array.
2115 // Put it all together to write the final output file.
2116
2117 OS << "/*===---- arm_cde.h - ARM CDE intrinsics "
2118 "-----------------------------------===\n"
2119 << LLVMLicenseHeader
2120 << "#ifndef __ARM_CDE_H\n"
2121 "#define __ARM_CDE_H\n"
2122 "\n"
2123 "#if !__ARM_FEATURE_CDE\n"
2124 "#error \"CDE support not enabled\"\n"
2125 "#endif\n"
2126 "\n"
2127 "#include <stdint.h>\n"
2128 "\n"
2129 "#ifdef __cplusplus\n"
2130 "extern \"C\" {\n"
2131 "#endif\n"
2132 "\n";
2133
2134 for (size_t i = 0; i < NumParts; ++i) {
2135 std::string condition;
2136 if (i == MVEFloat)
2137 condition = "__ARM_FEATURE_MVE & 2";
2138 else if (i == MVE)
2139 condition = "__ARM_FEATURE_MVE";
2140
2141 if (!condition.empty())
2142 OS << "#if " << condition << "\n\n";
2143 OS << parts[i].str();
2144 if (!condition.empty())
2145 OS << "#endif /* " << condition << " */\n\n";
2146 }
2147
2148 OS << "#ifdef __cplusplus\n"
2149 "} /* extern \"C\" */\n"
2150 "#endif\n"
2151 "\n"
2152 "#endif /* __ARM_CDE_H */\n";
2153}
2154
2155void CdeEmitter::EmitBuiltinDef(raw_ostream &OS) {
2156 for (const auto &kv : ACLEIntrinsics) {
2157 if (kv.second->headerOnly())
2158 continue;
2159 const ACLEIntrinsic &Int = *kv.second;
2160 OS << "BUILTIN(__builtin_arm_cde_" << Int.fullName()
2161 << ", \"\", \"ncU\")\n";
2162 }
2163}
2164
2165void CdeEmitter::EmitBuiltinSema(raw_ostream &OS) {
2166 std::map<std::string, std::set<std::string>> Checks;
2167 GroupSemaChecks(Checks);
2168
2169 for (const auto &kv : Checks) {
2170 for (StringRef Name : kv.second)
2171 OS << "case ARM::BI__builtin_arm_cde_" << Name << ":\n";
2172 OS << " Err = " << kv.first << " break;\n";
2173 }
2174}
2175
2176} // namespace
2177
2178namespace clang {
2179
2180// MVE
2181
2182void EmitMveHeader(RecordKeeper &Records, raw_ostream &OS) {
2183 MveEmitter(Records).EmitHeader(OS);
2184}
2185
2186void EmitMveBuiltinDef(RecordKeeper &Records, raw_ostream &OS) {
2187 MveEmitter(Records).EmitBuiltinDef(OS);
2188}
2189
2190void EmitMveBuiltinSema(RecordKeeper &Records, raw_ostream &OS) {
2191 MveEmitter(Records).EmitBuiltinSema(OS);
2192}
2193
2194void EmitMveBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
2195 MveEmitter(Records).EmitBuiltinCG(OS);
2196}
2197
2198void EmitMveBuiltinAliases(RecordKeeper &Records, raw_ostream &OS) {
2199 MveEmitter(Records).EmitBuiltinAliases(OS);
2200}
2201
2202// CDE
2203
2204void EmitCdeHeader(RecordKeeper &Records, raw_ostream &OS) {
2205 CdeEmitter(Records).EmitHeader(OS);
2206}
2207
2208void EmitCdeBuiltinDef(RecordKeeper &Records, raw_ostream &OS) {
2209 CdeEmitter(Records).EmitBuiltinDef(OS);
2210}
2211
2212void EmitCdeBuiltinSema(RecordKeeper &Records, raw_ostream &OS) {
2213 CdeEmitter(Records).EmitBuiltinSema(OS);
2214}
2215
2216void EmitCdeBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
2217 CdeEmitter(Records).EmitBuiltinCG(OS);
2218}
2219
2220void EmitCdeBuiltinAliases(RecordKeeper &Records, raw_ostream &OS) {
2221 CdeEmitter(Records).EmitBuiltinAliases(OS);
2222}
2223
2224} // end namespace clang
2225