1//===-- SveEmitter.cpp - Generate arm_sve.h for use with clang ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This tablegen backend is responsible for emitting arm_sve.h, which includes
10// a declaration and definition of each function specified by the ARM C/C++
11// Language Extensions (ACLE).
12//
13// For details, visit:
14// https://developer.arm.com/architectures/system-architectures/software-standards/acle
15//
16// Each SVE instruction is implemented in terms of 1 or more functions which
17// are suffixed with the element type of the input vectors. Functions may be
18// implemented in terms of generic vector operations such as +, *, -, etc. or
19// by calling a __builtin_-prefixed function which will be handled by clang's
20// CodeGen library.
21//
22// See also the documentation in include/clang/Basic/arm_sve.td.
23//
24//===----------------------------------------------------------------------===//
25
26#include "llvm/ADT/ArrayRef.h"
27#include "llvm/ADT/STLExtras.h"
28#include "llvm/ADT/StringExtras.h"
29#include "llvm/ADT/StringMap.h"
30#include "llvm/Support/raw_ostream.h"
31#include "llvm/TableGen/AArch64ImmCheck.h"
32#include "llvm/TableGen/Error.h"
33#include "llvm/TableGen/Record.h"
34#include "llvm/TableGen/StringToOffsetTable.h"
35#include <array>
36#include <cctype>
37#include <set>
38#include <string>
39#include <tuple>
40
41using namespace llvm;
42
43enum ClassKind {
44 ClassNone,
45 ClassS, // signed/unsigned, e.g., "_s8", "_u8" suffix
46 ClassG, // Overloaded name without type suffix
47};
48
49enum class ACLEKind { SVE, SME };
50
51using TypeSpec = std::string;
52
53namespace {
54class SVEType {
55
56 enum TypeKind {
57 Invalid,
58 Void,
59 Float,
60 SInt,
61 UInt,
62 BFloat16,
63 MFloat8,
64 Svcount,
65 PrefetchOp,
66 PredicatePattern,
67 Predicate,
68 Fpm
69 };
70 TypeKind Kind;
71 bool Immediate, Constant, Pointer, DefaultType, IsScalable;
72 unsigned Bitwidth, ElementBitwidth, NumVectors;
73
74public:
75 SVEType() : SVEType("", 'v') {}
76
77 SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1)
78 : Kind(Invalid), Immediate(false), Constant(false), Pointer(false),
79 DefaultType(false), IsScalable(true), Bitwidth(128),
80 ElementBitwidth(~0U), NumVectors(NumVectors) {
81 if (!TS.empty())
82 applyTypespec(TS);
83 applyModifier(Mod: CharMod);
84 }
85
86 SVEType(const SVEType &Base, unsigned NumV) : SVEType(Base) {
87 NumVectors = NumV;
88 }
89
90 bool isPointer() const { return Pointer; }
91 bool isConstant() const { return Constant; }
92 bool isImmediate() const { return Immediate; }
93 bool isScalar() const { return NumVectors == 0; }
94 bool isVector() const { return NumVectors > 0; }
95 bool isScalableVector() const { return isVector() && IsScalable; }
96 bool isFixedLengthVector() const { return isVector() && !IsScalable; }
97 bool isChar() const { return ElementBitwidth == 8 && isInteger(); }
98 bool isVoid() const { return Kind == Void; }
99 bool isDefault() const { return DefaultType; }
100 bool isFloat() const { return Kind == Float; }
101 bool isBFloat() const { return Kind == BFloat16; }
102 bool isMFloat() const { return Kind == MFloat8; }
103 bool isFloatingPoint() const {
104 return Kind == Float || Kind == BFloat16 || Kind == MFloat8;
105 }
106 bool isInteger() const { return Kind == SInt || Kind == UInt; }
107 bool isSignedInteger() const { return Kind == SInt; }
108 bool isUnsignedInteger() const { return Kind == UInt; }
109 bool isScalarPredicate() const {
110 return Kind == Predicate && NumVectors == 0;
111 }
112 bool isPredicate() const { return Kind == Predicate; }
113 bool isPredicatePattern() const { return Kind == PredicatePattern; }
114 bool isPrefetchOp() const { return Kind == PrefetchOp; }
115 bool isSvcount() const { return Kind == Svcount; }
116 bool isFpm() const { return Kind == Fpm; }
117 bool isInvalid() const { return Kind == Invalid; }
118 unsigned getElementSizeInBits() const { return ElementBitwidth; }
119 unsigned getNumVectors() const { return NumVectors; }
120
121 unsigned getNumElements() const {
122 assert(ElementBitwidth != ~0U);
123 return isPredicate() ? 16 : (Bitwidth / ElementBitwidth);
124 }
125 unsigned getSizeInBits() const {
126 return Bitwidth;
127 }
128
129 /// Return the string representation of a type, which is an encoded
130 /// string for passing to the BUILTIN() macro in Builtins.def.
131 std::string builtin_str() const;
132
133 /// Return the C/C++ string representation of a type for use in the
134 /// arm_sve.h header file.
135 std::string str() const;
136
137private:
138 /// Creates the type based on the typespec string in TS.
139 void applyTypespec(StringRef TS);
140
141 /// Applies a prototype modifier to the type.
142 void applyModifier(char Mod);
143
144 /// Get the builtin base for this SVEType, e.g. 'Wi' for svint64_t.
145 std::string builtinBaseType() const;
146};
147
148class SVEEmitter;
149
150/// The main grunt class. This represents an instantiation of an intrinsic with
151/// a particular typespec and prototype.
152class Intrinsic {
153 /// The unmangled name.
154 std::string Name;
155
156 /// The name of the corresponding LLVM IR intrinsic.
157 std::string LLVMName;
158
159 /// Intrinsic prototype.
160 std::string Proto;
161
162 /// The base type spec for this intrinsic.
163 TypeSpec BaseTypeSpec;
164
165 /// The base class kind. Most intrinsics use ClassS, which has full type
166 /// info for integers (_s32/_u32), or ClassG which is used for overloaded
167 /// intrinsics.
168 ClassKind Class;
169
170 /// The architectural #ifdef guard.
171 std::string SVEGuard, SMEGuard;
172
173 // The merge suffix such as _m, _x or _z.
174 std::string MergeSuffix;
175
176 /// The types of return value [0] and parameters [1..].
177 std::vector<SVEType> Types;
178
179 /// The "base type", which is VarType('d', BaseTypeSpec).
180 SVEType BaseType;
181
182 uint64_t Flags;
183
184 SmallVector<ImmCheck, 2> ImmChecks;
185
186 bool SetsFPMR;
187
188public:
189 Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
190 StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName,
191 uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT,
192 ClassKind Class, SVEEmitter &Emitter, StringRef SVEGuard,
193 StringRef SMEGuard);
194
195 ~Intrinsic()=default;
196
197 std::string getName() const { return Name; }
198 std::string getLLVMName() const { return LLVMName; }
199 std::string getProto() const { return Proto; }
200 TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; }
201 SVEType getBaseType() const { return BaseType; }
202
203 StringRef getSVEGuard() const { return SVEGuard; }
204 StringRef getSMEGuard() const { return SMEGuard; }
205 std::string getGuard() const {
206 std::string Guard;
207 llvm::raw_string_ostream OS(Guard);
208 if (!SVEGuard.empty() && SMEGuard.empty())
209 OS << SVEGuard;
210 else if (SVEGuard.empty() && !SMEGuard.empty())
211 OS << SMEGuard;
212 else {
213 if (SVEGuard.find(s: ",") != std::string::npos ||
214 SVEGuard.find(s: "|") != std::string::npos)
215 OS << "(" << SVEGuard << ")";
216 else
217 OS << SVEGuard;
218 OS << "|";
219 if (SMEGuard.find(s: ",") != std::string::npos ||
220 SMEGuard.find(s: "|") != std::string::npos)
221 OS << "(" << SMEGuard << ")";
222 else
223 OS << SMEGuard;
224 }
225 return Guard;
226 }
227 ClassKind getClassKind() const { return Class; }
228
229 SVEType getReturnType() const { return Types[0]; }
230 ArrayRef<SVEType> getTypes() const { return Types; }
231 SVEType getParamType(unsigned I) const { return Types[I + 1]; }
232 unsigned getNumParams() const {
233 return Proto.size() - (2 * count(Range: Proto, Element: '.')) - 1;
234 }
235
236 uint64_t getFlags() const { return Flags; }
237 bool isFlagSet(uint64_t Flag) const { return Flags & Flag;}
238
239 ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; }
240
241 /// Return the type string for a BUILTIN() macro in Builtins.def.
242 std::string getBuiltinTypeStr();
243
244 /// Return the name, mangled with type information. The name is mangled for
245 /// ClassS, so will add type suffixes such as _u32/_s32.
246 std::string getMangledName(ClassKind CK = ClassS) const {
247 return mangleName(LocalCK: CK);
248 }
249
250 /// As above, but mangles the LLVM name instead.
251 std::string getMangledLLVMName() const { return mangleLLVMName(); }
252
253 /// Returns true if the intrinsic is overloaded, in that it should also generate
254 /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of
255 /// 'svld1_u32(..)'.
256 static bool isOverloadedIntrinsic(StringRef Name) {
257 return Name.contains(C: '[') && Name.contains(C: ']');
258 }
259
260 /// Return true if the intrinsic takes a splat operand.
261 bool hasSplat() const {
262 // These prototype modifiers are described in arm_sve.td.
263 return Proto.find_first_of(s: "ajfrKLR@!") != std::string::npos;
264 }
265
266 /// Return the parameter index of the splat operand.
267 unsigned getSplatIdx() const {
268 unsigned I = 1, Param = 0;
269 for (; I < Proto.size(); ++I, ++Param) {
270 assert(Proto[I] != '4' &&
271 "Handling for '4' prototype modifier not implemented");
272 if (Proto[I] == 'a' || Proto[I] == 'j' || Proto[I] == 'f' ||
273 Proto[I] == 'r' || Proto[I] == 'K' || Proto[I] == 'L' ||
274 Proto[I] == 'R' || Proto[I] == '@' || Proto[I] == '!')
275 break;
276
277 if (Proto[I] == '2')
278 Param += 1;
279
280 // Multivector modifier can be skipped
281 if (Proto[I] == '.') {
282 Param -= 1; // Adjust for the increment at the top of the loop
283 I += 2;
284 }
285 }
286 assert(I != Proto.size() && "Prototype has no splat operand");
287 return Param;
288 }
289
290 /// Emits the intrinsic declaration to the ostream.
291 void emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter, ACLEKind Kind) const;
292
293private:
294 std::string getMergeSuffix() const { return MergeSuffix; }
295 StringRef getFPMSuffix() const { return SetsFPMR ? "_fpm" : ""; }
296 std::string mangleName(ClassKind LocalCK) const;
297 std::string mangleLLVMName() const;
298 std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,
299 std::string Proto) const;
300};
301
302class SVEEmitter {
303private:
304 // The reinterpret builtins are generated separately because they
305 // need the cross product of all types (121 functions in total),
306 // which is inconvenient to specify in the arm_sve.td file or
307 // generate in CGBuiltin.cpp.
308 struct ReinterpretTypeInfo {
309 SVEType BaseType;
310 const char *Suffix;
311 };
312
313 static const std::array<ReinterpretTypeInfo, 13> Reinterprets;
314
315 const RecordKeeper &Records;
316 StringMap<uint64_t> EltTypes;
317 StringMap<uint64_t> MemEltTypes;
318 StringMap<uint64_t> FlagTypes;
319 StringMap<uint64_t> MergeTypes;
320 StringMap<uint64_t> ImmCheckTypes;
321 std::vector<llvm::StringRef> ImmCheckTypeNames;
322
323public:
324 SVEEmitter(const RecordKeeper &R) : Records(R) {
325 for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "EltType"))
326 EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
327 for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "MemEltType"))
328 MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
329 for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "FlagType"))
330 FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
331 for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "MergeType"))
332 MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
333 for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "ImmCheckType")) {
334 auto [it, inserted] = ImmCheckTypes.try_emplace(
335 Key: RV->getNameInitAsString(), Args: RV->getValueAsInt(FieldName: "Value"));
336 if (!inserted)
337 llvm_unreachable("Duplicate imm check");
338 if ((size_t)it->second >= ImmCheckTypeNames.size())
339 ImmCheckTypeNames.resize(new_size: (size_t)it->second + 1);
340 ImmCheckTypeNames[it->second] = it->first();
341 }
342 }
343
344 /// Returns the enum value for the immcheck type
345 unsigned getEnumValueForImmCheck(StringRef C) const {
346 auto It = ImmCheckTypes.find(Key: C);
347 if (It != ImmCheckTypes.end())
348 return It->getValue();
349 llvm_unreachable("Unsupported imm check");
350 }
351
352 /// Returns the enum value for the flag type
353 uint64_t getEnumValueForFlag(StringRef C) const {
354 auto Res = FlagTypes.find(Key: C);
355 if (Res != FlagTypes.end())
356 return Res->getValue();
357 llvm_unreachable("Unsupported flag");
358 }
359
360 /// Returns the name for the immcheck type
361 StringRef getImmCheckForEnumValue(unsigned Id) {
362 if ((size_t)Id < ImmCheckTypeNames.size())
363 return ImmCheckTypeNames[Id];
364 llvm_unreachable("Unsupported imm check");
365 }
366
367 // Returns the SVETypeFlags for a given value and mask.
368 uint64_t encodeFlag(uint64_t V, StringRef MaskName) const {
369 auto It = FlagTypes.find(Key: MaskName);
370 if (It != FlagTypes.end()) {
371 uint64_t Mask = It->getValue();
372 unsigned Shift = countr_zero(Val: Mask);
373 assert(Shift < 64 && "Mask value produced an invalid shift value");
374 return (V << Shift) & Mask;
375 }
376 llvm_unreachable("Unsupported flag");
377 }
378
379 // Returns the SVETypeFlags for the given element type.
380 uint64_t encodeEltType(StringRef EltName) {
381 auto It = EltTypes.find(Key: EltName);
382 if (It != EltTypes.end())
383 return encodeFlag(V: It->getValue(), MaskName: "EltTypeMask");
384 llvm_unreachable("Unsupported EltType");
385 }
386
387 // Returns the SVETypeFlags for the given memory element type.
388 uint64_t encodeMemoryElementType(uint64_t MT) {
389 return encodeFlag(V: MT, MaskName: "MemEltTypeMask");
390 }
391
392 // Returns the SVETypeFlags for the given merge type.
393 uint64_t encodeMergeType(uint64_t MT) {
394 return encodeFlag(V: MT, MaskName: "MergeTypeMask");
395 }
396
397 // Returns the SVETypeFlags for the given splat operand.
398 unsigned encodeSplatOperand(unsigned SplatIdx) {
399 assert(SplatIdx < 7 && "SplatIdx out of encodable range");
400 return encodeFlag(V: SplatIdx + 1, MaskName: "SplatOperandMask");
401 }
402
403 // Returns the SVETypeFlags value for the given SVEType.
404 uint64_t encodeTypeFlags(const SVEType &T);
405
406 /// Emit arm_sve.h.
407 void createHeader(raw_ostream &o);
408
409 // Emits core intrinsics in both arm_sme.h and arm_sve.h
410 void createCoreHeaderIntrinsics(raw_ostream &o, SVEEmitter &Emitter,
411 ACLEKind Kind);
412
413 /// Emit all the __builtin prototypes and code needed by Sema.
414 void createBuiltins(raw_ostream &o);
415
416 /// Emit all the __builtin prototypes in JSON format.
417 void createBuiltinsJSON(raw_ostream &o);
418
419 /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
420 void createCodeGenMap(raw_ostream &o);
421
422 /// Emit all the range checks for the immediates.
423 void createRangeChecks(raw_ostream &o);
424
425 // Emit all the ImmCheckTypes to arm_immcheck_types.inc
426 void createImmCheckTypes(raw_ostream &OS);
427
428 /// Create the SVETypeFlags used in CGBuiltins
429 void createTypeFlags(raw_ostream &o);
430
431 /// Emit arm_sme.h.
432 void createSMEHeader(raw_ostream &o);
433
434 /// Emit all the SME __builtin prototypes and code needed by Sema.
435 void createSMEBuiltins(raw_ostream &o);
436
437 /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
438 void createSMECodeGenMap(raw_ostream &o);
439
440 /// Create a table for a builtin's requirement for PSTATE.SM.
441 void createStreamingAttrs(raw_ostream &o, ACLEKind Kind);
442
443 /// Emit all the range checks for the immediates.
444 void createSMERangeChecks(raw_ostream &o);
445
446 /// Create a table for a builtin's requirement for PSTATE.ZA.
447 void createBuiltinZAState(raw_ostream &OS);
448
449 /// Create intrinsic and add it to \p Out
450 void createIntrinsic(const Record *R,
451 SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out);
452};
453
454const std::array<SVEEmitter::ReinterpretTypeInfo, 13> SVEEmitter::Reinterprets =
455 {._M_elems: {{.BaseType: SVEType("c", 'd'), .Suffix: "s8"},
456 {.BaseType: SVEType("Uc", 'd'), .Suffix: "u8"},
457 {.BaseType: SVEType("m", 'd'), .Suffix: "mf8"},
458 {.BaseType: SVEType("s", 'd'), .Suffix: "s16"},
459 {.BaseType: SVEType("Us", 'd'), .Suffix: "u16"},
460 {.BaseType: SVEType("i", 'd'), .Suffix: "s32"},
461 {.BaseType: SVEType("Ui", 'd'), .Suffix: "u32"},
462 {.BaseType: SVEType("l", 'd'), .Suffix: "s64"},
463 {.BaseType: SVEType("Ul", 'd'), .Suffix: "u64"},
464 {.BaseType: SVEType("h", 'd'), .Suffix: "f16"},
465 {.BaseType: SVEType("b", 'd'), .Suffix: "bf16"},
466 {.BaseType: SVEType("f", 'd'), .Suffix: "f32"},
467 {.BaseType: SVEType("d", 'd'), .Suffix: "f64"}}};
468
469} // end anonymous namespace
470
471//===----------------------------------------------------------------------===//
472// Type implementation
473//===----------------------------------------------------------------------===//
474
475std::string SVEType::builtinBaseType() const {
476 switch (Kind) {
477 case TypeKind::Void:
478 return "v";
479 case TypeKind::Svcount:
480 return "Qa";
481 case TypeKind::PrefetchOp:
482 case TypeKind::PredicatePattern:
483 return "i";
484 case TypeKind::Fpm:
485 return "UWi";
486 case TypeKind::Predicate:
487 return "b";
488 case TypeKind::BFloat16:
489 assert(ElementBitwidth == 16 && "Invalid BFloat16!");
490 return "y";
491 case TypeKind::MFloat8:
492 assert(ElementBitwidth == 8 && "Invalid MFloat8!");
493 return "m";
494 case TypeKind::Float:
495 switch (ElementBitwidth) {
496 case 16:
497 return "h";
498 case 32:
499 return "f";
500 case 64:
501 return "d";
502 default:
503 llvm_unreachable("Unhandled float width!");
504 }
505 case TypeKind::SInt:
506 case TypeKind::UInt:
507 switch (ElementBitwidth) {
508 case 1:
509 return "b";
510 case 8:
511 return "c";
512 case 16:
513 return "s";
514 case 32:
515 return "i";
516 case 64:
517 return "Wi";
518 case 128:
519 return "LLLi";
520 default:
521 llvm_unreachable("Unhandled bitwidth!");
522 }
523 case TypeKind::Invalid:
524 llvm_unreachable("Attempting to resolve builtin string from Invalid type!");
525 }
526 llvm_unreachable("Unhandled TypeKind!");
527}
528
529std::string SVEType::builtin_str() const {
530 std::string Prefix;
531
532 if (isScalableVector())
533 Prefix = "q" + llvm::utostr(X: getNumElements() * NumVectors);
534 else if (isFixedLengthVector())
535 Prefix = "V" + llvm::utostr(X: getNumElements() * NumVectors);
536 else if (isImmediate()) {
537 assert(!isFloatingPoint() && "fp immediates are not supported");
538 Prefix = "I";
539 }
540
541 // Make chars and integer pointers explicitly signed.
542 if ((ElementBitwidth == 8 || isPointer()) && isSignedInteger())
543 Prefix += "S";
544 else if (isUnsignedInteger())
545 Prefix += "U";
546
547 std::string BuiltinStr = Prefix + builtinBaseType();
548 if (isConstant())
549 BuiltinStr += "C";
550 if (isPointer())
551 BuiltinStr += "*";
552
553 return BuiltinStr;
554}
555
556std::string SVEType::str() const {
557 std::string TypeStr;
558
559 switch (Kind) {
560 case TypeKind::PrefetchOp:
561 return "enum svprfop";
562 case TypeKind::PredicatePattern:
563 return "enum svpattern";
564 case TypeKind::Fpm:
565 TypeStr += "fpm";
566 break;
567 case TypeKind::Void:
568 TypeStr += "void";
569 break;
570 case TypeKind::Float:
571 TypeStr += "float" + llvm::utostr(X: ElementBitwidth);
572 break;
573 case TypeKind::Svcount:
574 TypeStr += "svcount";
575 break;
576 case TypeKind::Predicate:
577 TypeStr += "bool";
578 break;
579 case TypeKind::BFloat16:
580 TypeStr += "bfloat16";
581 break;
582 case TypeKind::MFloat8:
583 TypeStr += "mfloat8";
584 break;
585 case TypeKind::SInt:
586 TypeStr += "int" + llvm::utostr(X: ElementBitwidth);
587 break;
588 case TypeKind::UInt:
589 TypeStr += "uint" + llvm::utostr(X: ElementBitwidth);
590 break;
591 case TypeKind::Invalid:
592 llvm_unreachable("Attempting to resolve type name from Invalid type!");
593 }
594
595 if (isFixedLengthVector())
596 TypeStr += "x" + llvm::utostr(X: getNumElements());
597 else if (isScalableVector())
598 TypeStr = "sv" + TypeStr;
599
600 if (NumVectors > 1)
601 TypeStr += "x" + llvm::utostr(X: NumVectors);
602 if (!isScalarPredicate() && !isVoid())
603 TypeStr += "_t";
604 if (isConstant())
605 TypeStr += " const";
606 if (isPointer())
607 TypeStr += " *";
608
609 return TypeStr;
610}
611
612void SVEType::applyTypespec(StringRef TS) {
613 for (char I : TS) {
614 switch (I) {
615 case 'Q':
616 assert(isInvalid() && "Unexpected use of typespec modifier");
617 Kind = Svcount;
618 break;
619 case 'P':
620 assert(isInvalid() && "Unexpected use of typespec modifier");
621 Kind = Predicate;
622 break;
623 case 'U':
624 assert(isInvalid() && "Unexpected use of typespec modifier");
625 Kind = UInt;
626 break;
627 case 'c':
628 Kind = isInvalid() ? SInt : Kind;
629 ElementBitwidth = 8;
630 break;
631 case 's':
632 Kind = isInvalid() ? SInt : Kind;
633 ElementBitwidth = 16;
634 break;
635 case 'i':
636 Kind = isInvalid() ? SInt : Kind;
637 ElementBitwidth = 32;
638 break;
639 case 'l':
640 Kind = isInvalid() ? SInt : Kind;
641 ElementBitwidth = 64;
642 break;
643 case 'q':
644 Kind = isInvalid() ? SInt : Kind;
645 ElementBitwidth = 128;
646 break;
647 case 'h':
648 assert(isInvalid() && "Unexpected use of typespec modifier");
649 Kind = Float;
650 ElementBitwidth = 16;
651 break;
652 case 'f':
653 assert(isInvalid() && "Unexpected use of typespec modifier");
654 Kind = Float;
655 ElementBitwidth = 32;
656 break;
657 case 'd':
658 assert(isInvalid() && "Unexpected use of typespec modifier");
659 Kind = Float;
660 ElementBitwidth = 64;
661 break;
662 case 'b':
663 assert(isInvalid() && "Unexpected use of typespec modifier");
664 Kind = BFloat16;
665 ElementBitwidth = 16;
666 break;
667 case 'm':
668 assert(isInvalid() && "Unexpected use of typespec modifier");
669 Kind = MFloat8;
670 ElementBitwidth = 8;
671 break;
672 default:
673 llvm_unreachable("Unhandled type code!");
674 }
675 }
676 assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
677}
678
679void SVEType::applyModifier(char Mod) {
680 switch (Mod) {
681 case 'v':
682 Kind = Void;
683 NumVectors = 0;
684 break;
685 case 'd':
686 DefaultType = true;
687 break;
688 case 'c':
689 Constant = true;
690 [[fallthrough]];
691 case 'p':
692 Pointer = true;
693 Bitwidth = ElementBitwidth;
694 NumVectors = 0;
695 break;
696 case 'e':
697 Kind = UInt;
698 ElementBitwidth /= 2;
699 break;
700 case 'y':
701 Kind = SInt;
702 ElementBitwidth /= 2;
703 break;
704 case 'h':
705 ElementBitwidth /= 2;
706 break;
707 case 'q':
708 ElementBitwidth /= 4;
709 break;
710 case 'b':
711 Kind = UInt;
712 ElementBitwidth /= 4;
713 break;
714 case 'o':
715 ElementBitwidth *= 4;
716 break;
717 case 'P':
718 Kind = Predicate;
719 Bitwidth = 16;
720 ElementBitwidth = 1;
721 break;
722 case '{':
723 IsScalable = false;
724 Bitwidth = 128;
725 NumVectors = 1;
726 break;
727 case 's':
728 case 'a':
729 Bitwidth = ElementBitwidth;
730 NumVectors = 0;
731 break;
732 case 'R':
733 ElementBitwidth /= 2;
734 NumVectors = 0;
735 break;
736 case 'r':
737 ElementBitwidth /= 4;
738 NumVectors = 0;
739 break;
740 case '@':
741 Kind = UInt;
742 ElementBitwidth /= 4;
743 NumVectors = 0;
744 break;
745 case 'K':
746 Kind = SInt;
747 Bitwidth = ElementBitwidth;
748 NumVectors = 0;
749 break;
750 case 'L':
751 Kind = UInt;
752 Bitwidth = ElementBitwidth;
753 NumVectors = 0;
754 break;
755 case 'u':
756 Kind = UInt;
757 break;
758 case 'x':
759 Kind = SInt;
760 break;
761 case 'i':
762 Kind = UInt;
763 ElementBitwidth = Bitwidth = 64;
764 NumVectors = 0;
765 Immediate = true;
766 break;
767 case 'I':
768 Kind = PredicatePattern;
769 ElementBitwidth = Bitwidth = 32;
770 NumVectors = 0;
771 Immediate = true;
772 break;
773 case 'J':
774 Kind = PrefetchOp;
775 ElementBitwidth = Bitwidth = 32;
776 NumVectors = 0;
777 Immediate = true;
778 break;
779 case 'k':
780 Kind = SInt;
781 ElementBitwidth = Bitwidth = 32;
782 NumVectors = 0;
783 break;
784 case 'l':
785 Kind = SInt;
786 ElementBitwidth = Bitwidth = 64;
787 NumVectors = 0;
788 break;
789 case 'm':
790 Kind = UInt;
791 ElementBitwidth = Bitwidth = 32;
792 NumVectors = 0;
793 break;
794 case '>':
795 Kind = Fpm;
796 ElementBitwidth = Bitwidth = 64;
797 NumVectors = 0;
798 break;
799 case 'n':
800 Kind = UInt;
801 ElementBitwidth = Bitwidth = 64;
802 NumVectors = 0;
803 break;
804 case 'w':
805 ElementBitwidth = 64;
806 break;
807 case 'j':
808 ElementBitwidth = Bitwidth = 64;
809 NumVectors = 0;
810 break;
811 case 'f':
812 Kind = UInt;
813 ElementBitwidth = Bitwidth = 64;
814 NumVectors = 0;
815 break;
816 case 'g':
817 Kind = UInt;
818 ElementBitwidth = 64;
819 break;
820 case '#':
821 Kind = SInt;
822 ElementBitwidth = 64;
823 break;
824 case '[':
825 Kind = UInt;
826 ElementBitwidth = 8;
827 break;
828 case 't':
829 Kind = SInt;
830 ElementBitwidth = 32;
831 break;
832 case 'z':
833 Kind = UInt;
834 ElementBitwidth = 32;
835 break;
836 case 'O':
837 Kind = Float;
838 ElementBitwidth = 16;
839 break;
840 case 'M':
841 Kind = Float;
842 ElementBitwidth = 32;
843 break;
844 case 'N':
845 Kind = Float;
846 ElementBitwidth = 64;
847 break;
848 case 'Q':
849 Kind = Void;
850 Constant = true;
851 Pointer = true;
852 NumVectors = 0;
853 break;
854 case 'S':
855 Kind = SInt;
856 Constant = true;
857 Pointer = true;
858 ElementBitwidth = Bitwidth = 8;
859 NumVectors = 0;
860 break;
861 case 'W':
862 Kind = UInt;
863 Constant = true;
864 Pointer = true;
865 ElementBitwidth = Bitwidth = 8;
866 NumVectors = 0;
867 break;
868 case 'T':
869 Kind = SInt;
870 Constant = true;
871 Pointer = true;
872 ElementBitwidth = Bitwidth = 16;
873 NumVectors = 0;
874 break;
875 case 'X':
876 Kind = UInt;
877 Constant = true;
878 Pointer = true;
879 ElementBitwidth = Bitwidth = 16;
880 NumVectors = 0;
881 break;
882 case 'Y':
883 Kind = UInt;
884 Constant = true;
885 Pointer = true;
886 ElementBitwidth = Bitwidth = 32;
887 NumVectors = 0;
888 break;
889 case 'U':
890 Kind = SInt;
891 Constant = true;
892 Pointer = true;
893 ElementBitwidth = Bitwidth = 32;
894 NumVectors = 0;
895 break;
896 case '%':
897 Kind = Void;
898 Pointer = true;
899 NumVectors = 0;
900 break;
901 case 'A':
902 Kind = SInt;
903 Pointer = true;
904 ElementBitwidth = Bitwidth = 8;
905 NumVectors = 0;
906 break;
907 case 'B':
908 Kind = SInt;
909 Pointer = true;
910 ElementBitwidth = Bitwidth = 16;
911 NumVectors = 0;
912 break;
913 case 'C':
914 Kind = SInt;
915 Pointer = true;
916 ElementBitwidth = Bitwidth = 32;
917 NumVectors = 0;
918 break;
919 case 'D':
920 Kind = SInt;
921 Pointer = true;
922 ElementBitwidth = Bitwidth = 64;
923 NumVectors = 0;
924 break;
925 case 'E':
926 Kind = UInt;
927 Pointer = true;
928 ElementBitwidth = Bitwidth = 8;
929 NumVectors = 0;
930 break;
931 case 'F':
932 Kind = UInt;
933 Pointer = true;
934 ElementBitwidth = Bitwidth = 16;
935 NumVectors = 0;
936 break;
937 case 'G':
938 Kind = UInt;
939 Pointer = true;
940 ElementBitwidth = Bitwidth = 32;
941 NumVectors = 0;
942 break;
943 case '$':
944 Kind = BFloat16;
945 ElementBitwidth = 16;
946 break;
947 case '}':
948 Kind = Svcount;
949 NumVectors = 0;
950 break;
951 case '~':
952 Kind = MFloat8;
953 ElementBitwidth = 8;
954 break;
955 case '!':
956 Kind = MFloat8;
957 Bitwidth = ElementBitwidth = 8;
958 NumVectors = 0;
959 break;
960 case '.':
961 llvm_unreachable(". is never a type in itself");
962 break;
963 default:
964 llvm_unreachable("Unhandled character!");
965 }
966}
967
968/// Returns the modifier and number of vectors for the given operand \p Op.
969std::pair<char, unsigned> getProtoModifier(StringRef Proto, unsigned Op) {
970 for (unsigned P = 0; !Proto.empty(); ++P) {
971 unsigned NumVectors = 1;
972 unsigned CharsToSkip = 1;
973 char Mod = Proto[0];
974 if (Mod == '2' || Mod == '3' || Mod == '4') {
975 NumVectors = Mod - '0';
976 Mod = 'd';
977 if (Proto.size() > 1 && Proto[1] == '.') {
978 Mod = Proto[2];
979 CharsToSkip = 3;
980 }
981 }
982
983 if (P == Op)
984 return {Mod, NumVectors};
985
986 Proto = Proto.drop_front(N: CharsToSkip);
987 }
988 llvm_unreachable("Unexpected Op");
989}
990
991//===----------------------------------------------------------------------===//
992// Intrinsic implementation
993//===----------------------------------------------------------------------===//
994
995Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
996 StringRef MergeSuffix, uint64_t MemoryElementTy,
997 StringRef LLVMName, uint64_t Flags,
998 ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class,
999 SVEEmitter &Emitter, StringRef SVEGuard,
1000 StringRef SMEGuard)
1001 : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()),
1002 BaseTypeSpec(BT), Class(Class), MergeSuffix(MergeSuffix.str()),
1003 BaseType(BT, 'd'), Flags(Flags), ImmChecks(Checks) {
1004
1005 auto FormatGuard = [](StringRef Guard, StringRef Base) -> std::string {
1006 if (Guard.empty() || Guard == Base)
1007 return Guard.str();
1008
1009 unsigned Depth = 0;
1010 for (auto &C : Guard) {
1011 switch (C) {
1012 default:
1013 break;
1014 case '|':
1015 if (Depth == 0)
1016 // Group top-level ORs before ANDing with the base feature.
1017 return Base.str() + ",(" + Guard.str() + ")";
1018 break;
1019 case '(':
1020 ++Depth;
1021 break;
1022 case ')':
1023 if (Depth == 0)
1024 llvm_unreachable("Mismatched parentheses!");
1025
1026 --Depth;
1027 break;
1028 }
1029 }
1030
1031 return Base.str() + "," + Guard.str();
1032 };
1033
1034 this->SVEGuard = FormatGuard(SVEGuard, "sve");
1035 this->SMEGuard = FormatGuard(SMEGuard, "sme");
1036
1037 // Types[0] is the return value.
1038 for (unsigned I = 0; I < (getNumParams() + 1); ++I) {
1039 char Mod;
1040 unsigned NumVectors;
1041 std::tie(args&: Mod, args&: NumVectors) = getProtoModifier(Proto, Op: I);
1042 SVEType T(BaseTypeSpec, Mod, NumVectors);
1043 Types.push_back(x: T);
1044 SetsFPMR = T.isFpm();
1045
1046 // Add range checks for immediates
1047 if (I > 0) {
1048 if (T.isPredicatePattern())
1049 ImmChecks.emplace_back(
1050 Args: I - 1, Args: Emitter.getEnumValueForImmCheck(C: "ImmCheck0_31"));
1051 else if (T.isPrefetchOp())
1052 ImmChecks.emplace_back(
1053 Args: I - 1, Args: Emitter.getEnumValueForImmCheck(C: "ImmCheck0_13"));
1054 }
1055 }
1056
1057 // Set flags based on properties
1058 this->Flags |= Emitter.encodeTypeFlags(T: BaseType);
1059 this->Flags |= Emitter.encodeMemoryElementType(MT: MemoryElementTy);
1060 this->Flags |= Emitter.encodeMergeType(MT: MergeTy);
1061 if (hasSplat())
1062 this->Flags |= Emitter.encodeSplatOperand(SplatIdx: getSplatIdx());
1063 if (SetsFPMR)
1064 this->Flags |= Emitter.getEnumValueForFlag(C: "SetsFPMR");
1065}
1066
1067std::string Intrinsic::getBuiltinTypeStr() {
1068 std::string S = getReturnType().builtin_str();
1069 for (unsigned I = 0; I < getNumParams(); ++I)
1070 S += getParamType(I).builtin_str();
1071
1072 return S;
1073}
1074
1075std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
1076 std::string Proto) const {
1077 std::string Ret = Name;
1078 while (Ret.find(c: '{') != std::string::npos) {
1079 size_t Pos = Ret.find(c: '{');
1080 size_t End = Ret.find(c: '}');
1081 unsigned NumChars = End - Pos + 1;
1082 assert(NumChars == 3 && "Unexpected template argument");
1083
1084 SVEType T;
1085 char C = Ret[Pos+1];
1086 switch(C) {
1087 default:
1088 llvm_unreachable("Unknown predication specifier");
1089 case 'd':
1090 T = SVEType(TS, 'd');
1091 break;
1092 case '0':
1093 case '1':
1094 case '2':
1095 case '3':
1096 // Extract the modifier before passing to SVEType to handle numeric
1097 // modifiers
1098 auto [Mod, NumVectors] = getProtoModifier(Proto, Op: (C - '0'));
1099 T = SVEType(TS, Mod);
1100 break;
1101 }
1102
1103 // Replace templated arg with the right suffix (e.g. u32)
1104 std::string TypeCode;
1105
1106 if (T.isSignedInteger())
1107 TypeCode = 's';
1108 else if (T.isUnsignedInteger())
1109 TypeCode = 'u';
1110 else if (T.isSvcount())
1111 TypeCode = 'c';
1112 else if (T.isPredicate())
1113 TypeCode = 'b';
1114 else if (T.isBFloat())
1115 TypeCode = "bf";
1116 else if (T.isMFloat())
1117 TypeCode = "mf";
1118 else
1119 TypeCode = 'f';
1120 Ret.replace(pos: Pos, n: NumChars, str: TypeCode + utostr(X: T.getElementSizeInBits()));
1121 }
1122
1123 return Ret;
1124}
1125
1126std::string Intrinsic::mangleLLVMName() const {
1127 std::string S = getLLVMName();
1128
1129 // Replace all {d} like expressions with e.g. 'u32'
1130 return replaceTemplatedArgs(Name: S, TS: getBaseTypeSpec(), Proto: getProto());
1131}
1132
1133std::string Intrinsic::mangleName(ClassKind LocalCK) const {
1134 std::string S = getName();
1135
1136 if (LocalCK == ClassG) {
1137 // Remove the square brackets and everything in between.
1138 while (S.find(c: '[') != std::string::npos) {
1139 auto Start = S.find(c: '[');
1140 auto End = S.find(c: ']');
1141 S.erase(pos: Start, n: (End-Start)+1);
1142 }
1143 } else {
1144 // Remove the square brackets.
1145 while (S.find(c: '[') != std::string::npos) {
1146 auto BrPos = S.find(c: '[');
1147 if (BrPos != std::string::npos)
1148 S.erase(pos: BrPos, n: 1);
1149 BrPos = S.find(c: ']');
1150 if (BrPos != std::string::npos)
1151 S.erase(pos: BrPos, n: 1);
1152 }
1153 }
1154
1155 // Replace all {d} like expressions with e.g. 'u32'
1156 return replaceTemplatedArgs(Name: S, TS: getBaseTypeSpec(), Proto: getProto())
1157 .append(str: getMergeSuffix())
1158 .append(svt: getFPMSuffix());
1159}
1160
1161void Intrinsic::emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter,
1162 ACLEKind Kind) const {
1163 bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1;
1164
1165 std::string FullName = mangleName(LocalCK: ClassS);
1166 std::string ProtoName = mangleName(LocalCK: getClassKind());
1167 OS << (IsOverloaded ? "__aio " : "__ai ")
1168 << "__attribute__((__clang_arm_builtin_alias(";
1169
1170 switch (Kind) {
1171 case ACLEKind::SME:
1172 OS << "__builtin_sme_" << FullName << ")";
1173 break;
1174 case ACLEKind::SVE:
1175 OS << "__builtin_sve_" << FullName << ")";
1176 break;
1177 }
1178
1179 OS << "))\n";
1180
1181 OS << getTypes()[0].str() << " " << ProtoName << "(";
1182 for (unsigned I = 0; I < getTypes().size() - 1; ++I) {
1183 if (I != 0)
1184 OS << ", ";
1185 OS << getTypes()[I + 1].str();
1186 }
1187 OS << ");\n";
1188}
1189
1190//===----------------------------------------------------------------------===//
1191// SVEEmitter implementation
1192//===----------------------------------------------------------------------===//
1193uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {
1194 if (T.isFloat()) {
1195 switch (T.getElementSizeInBits()) {
1196 case 16:
1197 return encodeEltType(EltName: "EltTyFloat16");
1198 case 32:
1199 return encodeEltType(EltName: "EltTyFloat32");
1200 case 64:
1201 return encodeEltType(EltName: "EltTyFloat64");
1202 default:
1203 llvm_unreachable("Unhandled float element bitwidth!");
1204 }
1205 }
1206
1207 if (T.isBFloat()) {
1208 assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat.");
1209 return encodeEltType(EltName: "EltTyBFloat16");
1210 }
1211
1212 if (T.isMFloat()) {
1213 assert(T.getElementSizeInBits() == 8 && "Not a valid MFloat.");
1214 return encodeEltType(EltName: "EltTyMFloat8");
1215 }
1216
1217 if (T.isPredicate() || T.isSvcount()) {
1218 switch (T.getElementSizeInBits()) {
1219 case 8:
1220 return encodeEltType(EltName: "EltTyBool8");
1221 case 16:
1222 return encodeEltType(EltName: "EltTyBool16");
1223 case 32:
1224 return encodeEltType(EltName: "EltTyBool32");
1225 case 64:
1226 return encodeEltType(EltName: "EltTyBool64");
1227 default:
1228 llvm_unreachable("Unhandled predicate element bitwidth!");
1229 }
1230 }
1231
1232 switch (T.getElementSizeInBits()) {
1233 case 8:
1234 return encodeEltType(EltName: "EltTyInt8");
1235 case 16:
1236 return encodeEltType(EltName: "EltTyInt16");
1237 case 32:
1238 return encodeEltType(EltName: "EltTyInt32");
1239 case 64:
1240 return encodeEltType(EltName: "EltTyInt64");
1241 case 128:
1242 return encodeEltType(EltName: "EltTyInt128");
1243 default:
1244 llvm_unreachable("Unhandled integer element bitwidth!");
1245 }
1246}
1247
1248void SVEEmitter::createIntrinsic(
1249 const Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) {
1250 StringRef Name = R->getValueAsString(FieldName: "Name");
1251 StringRef Proto = R->getValueAsString(FieldName: "Prototype");
1252 StringRef Types = R->getValueAsString(FieldName: "Types");
1253 StringRef SVEGuard = R->getValueAsString(FieldName: "SVETargetGuard");
1254 StringRef SMEGuard = R->getValueAsString(FieldName: "SMETargetGuard");
1255 StringRef LLVMName = R->getValueAsString(FieldName: "LLVMIntrinsic");
1256 uint64_t Merge = R->getValueAsInt(FieldName: "Merge");
1257 StringRef MergeSuffix = R->getValueAsString(FieldName: "MergeSuffix");
1258 uint64_t MemEltType = R->getValueAsInt(FieldName: "MemEltType");
1259
1260 int64_t Flags = 0;
1261 for (const Record *FlagRec : R->getValueAsListOfDefs(FieldName: "Flags"))
1262 Flags |= FlagRec->getValueAsInt(FieldName: "Value");
1263
1264 // Create a dummy TypeSpec for non-overloaded builtins.
1265 if (Types.empty()) {
1266 assert((Flags & getEnumValueForFlag("IsOverloadNone")) &&
1267 "Expect TypeSpec for overloaded builtin!");
1268 Types = "i";
1269 }
1270
1271 // Extract type specs from string
1272 SmallVector<TypeSpec, 8> TypeSpecs;
1273 TypeSpec Acc;
1274 for (char I : Types) {
1275 Acc.push_back(c: I);
1276 if (islower(I)) {
1277 TypeSpecs.push_back(Elt: TypeSpec(Acc));
1278 Acc.clear();
1279 }
1280 }
1281
1282 // Remove duplicate type specs.
1283 sort(C&: TypeSpecs);
1284 TypeSpecs.erase(CS: llvm::unique(R&: TypeSpecs), CE: TypeSpecs.end());
1285
1286 // Create an Intrinsic for each type spec.
1287 for (auto TS : TypeSpecs) {
1288 // Collate a list of range/option checks for the immediates.
1289 SmallVector<ImmCheck, 2> ImmChecks;
1290 for (const Record *ImmR : R->getValueAsListOfDefs(FieldName: "ImmChecks")) {
1291 int64_t ArgIdx = ImmR->getValueAsInt(FieldName: "ImmArgIdx");
1292 int64_t EltSizeArgIdx = ImmR->getValueAsInt(FieldName: "TypeContextArgIdx");
1293 int64_t Kind = ImmR->getValueAsDef(FieldName: "Kind")->getValueAsInt(FieldName: "Value");
1294 assert(ArgIdx >= 0 && Kind >= 0 &&
1295 "ImmArgIdx and Kind must be nonnegative");
1296
1297 unsigned ElementSizeInBits = 0;
1298 auto [Mod, NumVectors] = getProtoModifier(Proto, Op: EltSizeArgIdx + 1);
1299 if (EltSizeArgIdx >= 0)
1300 ElementSizeInBits = SVEType(TS, Mod, NumVectors).getElementSizeInBits();
1301 ImmChecks.push_back(Elt: ImmCheck(ArgIdx, Kind, ElementSizeInBits));
1302 }
1303
1304 Out.push_back(Elt: std::make_unique<Intrinsic>(
1305 args&: Name, args&: Proto, args&: Merge, args&: MergeSuffix, args&: MemEltType, args&: LLVMName, args&: Flags, args&: ImmChecks,
1306 args&: TS, args: ClassS, args&: *this, args&: SVEGuard, args&: SMEGuard));
1307
1308 // Also generate the short-form (e.g. svadd_m) for the given type-spec.
1309 if (Intrinsic::isOverloadedIntrinsic(Name))
1310 Out.push_back(Elt: std::make_unique<Intrinsic>(
1311 args&: Name, args&: Proto, args&: Merge, args&: MergeSuffix, args&: MemEltType, args&: LLVMName, args&: Flags,
1312 args&: ImmChecks, args&: TS, args: ClassG, args&: *this, args&: SVEGuard, args&: SMEGuard));
1313 }
1314}
1315
1316void SVEEmitter::createCoreHeaderIntrinsics(raw_ostream &OS,
1317 SVEEmitter &Emitter,
1318 ACLEKind Kind) {
1319 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1320 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1321 for (auto *R : RV)
1322 createIntrinsic(R, Out&: Defs);
1323
1324 // Sort intrinsics in header file by following order/priority:
1325 // - Architectural guard (i.e. does it require SVE2 or SVE2_AES)
1326 // - Class (is intrinsic overloaded or not)
1327 // - Intrinsic name
1328 llvm::stable_sort(Range&: Defs, C: [](const std::unique_ptr<Intrinsic> &A,
1329 const std::unique_ptr<Intrinsic> &B) {
1330 auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) {
1331 return std::make_tuple(args: I->getSVEGuard().str() + I->getSMEGuard().str(),
1332 args: (unsigned)I->getClassKind(), args: I->getName());
1333 };
1334 return ToTuple(A) < ToTuple(B);
1335 });
1336
1337 // Actually emit the intrinsic declarations.
1338 for (auto &I : Defs)
1339 I->emitIntrinsic(OS, Emitter, Kind);
1340}
1341
1342void SVEEmitter::createHeader(raw_ostream &OS) {
1343 OS << "/*===---- arm_sve.h - ARM SVE intrinsics "
1344 "-----------------------------------===\n"
1345 " *\n"
1346 " *\n"
1347 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1348 "Exceptions.\n"
1349 " * See https://llvm.org/LICENSE.txt for license information.\n"
1350 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1351 " *\n"
1352 " *===-----------------------------------------------------------------"
1353 "------===\n"
1354 " */\n\n";
1355
1356 OS << "#ifndef __ARM_SVE_H\n";
1357 OS << "#define __ARM_SVE_H\n\n";
1358
1359 OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1360 OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n";
1361 OS << "#endif\n";
1362
1363 OS << "#include <stdint.h>\n\n";
1364 OS << "#ifdef __cplusplus\n";
1365 OS << "extern \"C\" {\n";
1366 OS << "#else\n";
1367 OS << "#include <stdbool.h>\n";
1368 OS << "#endif\n\n";
1369
1370 OS << "typedef __fp16 float16_t;\n";
1371 OS << "typedef float float32_t;\n";
1372 OS << "typedef double float64_t;\n";
1373
1374 OS << "typedef __SVInt8_t svint8_t;\n";
1375 OS << "typedef __SVInt16_t svint16_t;\n";
1376 OS << "typedef __SVInt32_t svint32_t;\n";
1377 OS << "typedef __SVInt64_t svint64_t;\n";
1378 OS << "typedef __SVUint8_t svuint8_t;\n";
1379 OS << "typedef __SVUint16_t svuint16_t;\n";
1380 OS << "typedef __SVUint32_t svuint32_t;\n";
1381 OS << "typedef __SVUint64_t svuint64_t;\n";
1382 OS << "typedef __SVFloat16_t svfloat16_t;\n\n";
1383
1384 OS << "typedef __SVBfloat16_t svbfloat16_t;\n";
1385
1386 OS << "#include <arm_bf16.h>\n";
1387 OS << "#include <arm_vector_types.h>\n";
1388
1389 OS << "typedef __SVMfloat8_t svmfloat8_t;\n\n";
1390
1391 OS << "typedef __SVFloat32_t svfloat32_t;\n";
1392 OS << "typedef __SVFloat64_t svfloat64_t;\n";
1393 OS << "typedef __clang_svint8x2_t svint8x2_t;\n";
1394 OS << "typedef __clang_svint16x2_t svint16x2_t;\n";
1395 OS << "typedef __clang_svint32x2_t svint32x2_t;\n";
1396 OS << "typedef __clang_svint64x2_t svint64x2_t;\n";
1397 OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n";
1398 OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n";
1399 OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n";
1400 OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n";
1401 OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n";
1402 OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n";
1403 OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n";
1404 OS << "typedef __clang_svint8x3_t svint8x3_t;\n";
1405 OS << "typedef __clang_svint16x3_t svint16x3_t;\n";
1406 OS << "typedef __clang_svint32x3_t svint32x3_t;\n";
1407 OS << "typedef __clang_svint64x3_t svint64x3_t;\n";
1408 OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n";
1409 OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n";
1410 OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n";
1411 OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n";
1412 OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n";
1413 OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n";
1414 OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n";
1415 OS << "typedef __clang_svint8x4_t svint8x4_t;\n";
1416 OS << "typedef __clang_svint16x4_t svint16x4_t;\n";
1417 OS << "typedef __clang_svint32x4_t svint32x4_t;\n";
1418 OS << "typedef __clang_svint64x4_t svint64x4_t;\n";
1419 OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n";
1420 OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n";
1421 OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n";
1422 OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n";
1423 OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n";
1424 OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n";
1425 OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n";
1426 OS << "typedef __SVBool_t svbool_t;\n";
1427 OS << "typedef __clang_svboolx2_t svboolx2_t;\n";
1428 OS << "typedef __clang_svboolx4_t svboolx4_t;\n\n";
1429
1430 OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";
1431 OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";
1432 OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";
1433
1434 OS << "typedef __clang_svmfloat8x2_t svmfloat8x2_t;\n";
1435 OS << "typedef __clang_svmfloat8x3_t svmfloat8x3_t;\n";
1436 OS << "typedef __clang_svmfloat8x4_t svmfloat8x4_t;\n";
1437
1438 OS << "typedef __SVCount_t svcount_t;\n\n";
1439
1440 OS << "enum svpattern\n";
1441 OS << "{\n";
1442 OS << " SV_POW2 = 0,\n";
1443 OS << " SV_VL1 = 1,\n";
1444 OS << " SV_VL2 = 2,\n";
1445 OS << " SV_VL3 = 3,\n";
1446 OS << " SV_VL4 = 4,\n";
1447 OS << " SV_VL5 = 5,\n";
1448 OS << " SV_VL6 = 6,\n";
1449 OS << " SV_VL7 = 7,\n";
1450 OS << " SV_VL8 = 8,\n";
1451 OS << " SV_VL16 = 9,\n";
1452 OS << " SV_VL32 = 10,\n";
1453 OS << " SV_VL64 = 11,\n";
1454 OS << " SV_VL128 = 12,\n";
1455 OS << " SV_VL256 = 13,\n";
1456 OS << " SV_MUL4 = 29,\n";
1457 OS << " SV_MUL3 = 30,\n";
1458 OS << " SV_ALL = 31\n";
1459 OS << "};\n\n";
1460
1461 OS << "enum svprfop\n";
1462 OS << "{\n";
1463 OS << " SV_PLDL1KEEP = 0,\n";
1464 OS << " SV_PLDL1STRM = 1,\n";
1465 OS << " SV_PLDL2KEEP = 2,\n";
1466 OS << " SV_PLDL2STRM = 3,\n";
1467 OS << " SV_PLDL3KEEP = 4,\n";
1468 OS << " SV_PLDL3STRM = 5,\n";
1469 OS << " SV_PSTL1KEEP = 8,\n";
1470 OS << " SV_PSTL1STRM = 9,\n";
1471 OS << " SV_PSTL2KEEP = 10,\n";
1472 OS << " SV_PSTL2STRM = 11,\n";
1473 OS << " SV_PSTL3KEEP = 12,\n";
1474 OS << " SV_PSTL3STRM = 13\n";
1475 OS << "};\n\n";
1476
1477 OS << "/* Function attributes */\n";
1478 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1479 "__nodebug__))\n\n";
1480 OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1481 "__nodebug__, __overloadable__))\n\n";
1482
1483 // Add reinterpret functions.
1484 for (auto [N, Suffix] :
1485 std::initializer_list<std::pair<unsigned, const char *>>{
1486 {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1487 for (auto ShortForm : {false, true})
1488 for (const ReinterpretTypeInfo &To : Reinterprets) {
1489 SVEType ToV(To.BaseType, N);
1490 for (const ReinterpretTypeInfo &From : Reinterprets) {
1491 SVEType FromV(From.BaseType, N);
1492 OS << "__aio "
1493 "__attribute__((__clang_arm_builtin_alias(__builtin_sve_"
1494 "reinterpret_"
1495 << To.Suffix << "_" << From.Suffix << Suffix << ")))\n"
1496 << ToV.str() << " svreinterpret_" << To.Suffix;
1497 if (!ShortForm)
1498 OS << "_" << From.Suffix << Suffix;
1499 OS << "(" << FromV.str() << " op);\n";
1500 }
1501 }
1502 }
1503
1504 createCoreHeaderIntrinsics(OS, Emitter&: *this, Kind: ACLEKind::SVE);
1505
1506 OS << "#define svcvtnt_bf16_x svcvtnt_bf16_m\n";
1507 OS << "#define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m\n";
1508
1509 OS << "#define svcvtnt_f16_x svcvtnt_f16_m\n";
1510 OS << "#define svcvtnt_f16_f32_x svcvtnt_f16_f32_m\n";
1511 OS << "#define svcvtnt_f32_x svcvtnt_f32_m\n";
1512 OS << "#define svcvtnt_f32_f64_x svcvtnt_f32_f64_m\n\n";
1513
1514 OS << "#define svcvtxnt_f32_x svcvtxnt_f32_m\n";
1515 OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n";
1516
1517 OS << "#ifdef __cplusplus\n";
1518 OS << "} // extern \"C\"\n";
1519 OS << "#endif\n\n";
1520 OS << "#undef __ai\n\n";
1521 OS << "#undef __aio\n\n";
1522 OS << "#endif /* __ARM_SVE_H */\n";
1523}
1524
1525void SVEEmitter::createBuiltins(raw_ostream &OS) {
1526 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1527 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1528 for (auto *R : RV)
1529 createIntrinsic(R, Out&: Defs);
1530
1531 // The mappings must be sorted based on BuiltinID.
1532 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1533 const std::unique_ptr<Intrinsic> &B) {
1534 return A->getMangledName() < B->getMangledName();
1535 });
1536
1537 llvm::StringToOffsetTable Table;
1538 Table.GetOrAddStringOffset(Str: "");
1539 Table.GetOrAddStringOffset(Str: "n");
1540
1541 for (const auto &Def : Defs)
1542 if (Def->getClassKind() != ClassG) {
1543 Table.GetOrAddStringOffset(Str: Def->getMangledName());
1544 Table.GetOrAddStringOffset(Str: Def->getBuiltinTypeStr());
1545 Table.GetOrAddStringOffset(Str: Def->getGuard());
1546 }
1547
1548 Table.GetOrAddStringOffset(Str: "sme|sve");
1549 SmallVector<std::pair<std::string, std::string>> ReinterpretBuiltins;
1550 for (auto [N, Suffix] :
1551 std::initializer_list<std::pair<unsigned, const char *>>{
1552 {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1553 for (const ReinterpretTypeInfo &To : Reinterprets) {
1554 SVEType ToV(To.BaseType, N);
1555 for (const ReinterpretTypeInfo &From : Reinterprets) {
1556 SVEType FromV(From.BaseType, N);
1557 std::string Name =
1558 (Twine("reinterpret_") + To.Suffix + "_" + From.Suffix + Suffix)
1559 .str();
1560 std::string Type = ToV.builtin_str() + FromV.builtin_str();
1561 Table.GetOrAddStringOffset(Str: Name);
1562 Table.GetOrAddStringOffset(Str: Type);
1563 ReinterpretBuiltins.push_back(Elt: {Name, Type});
1564 }
1565 }
1566 }
1567
1568 OS << "#ifdef GET_SVE_BUILTIN_ENUMERATORS\n";
1569 for (const auto &Def : Defs)
1570 if (Def->getClassKind() != ClassG)
1571 OS << " BI__builtin_sve_" << Def->getMangledName() << ",\n";
1572 for (const auto &[Name, _] : ReinterpretBuiltins)
1573 OS << " BI__builtin_sve_" << Name << ",\n";
1574 OS << "#endif // GET_SVE_BUILTIN_ENUMERATORS\n\n";
1575
1576 OS << "#ifdef GET_SVE_BUILTIN_STR_TABLE\n";
1577 Table.EmitStringTableDef(OS, Name: "BuiltinStrings");
1578 OS << "#endif // GET_SVE_BUILTIN_STR_TABLE\n\n";
1579
1580 OS << "#ifdef GET_SVE_BUILTIN_INFOS\n";
1581 for (const auto &Def : Defs) {
1582 // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1583 // declarations only live in the header file.
1584 if (Def->getClassKind() != ClassG) {
1585 OS << " Builtin::Info{Builtin::Info::StrOffsets{"
1586 << Table.GetStringOffset(Str: Def->getMangledName()) << " /* "
1587 << Def->getMangledName() << " */, ";
1588 OS << Table.GetStringOffset(Str: Def->getBuiltinTypeStr()) << " /* "
1589 << Def->getBuiltinTypeStr() << " */, ";
1590 OS << Table.GetStringOffset(Str: "n") << " /* n */, ";
1591 OS << Table.GetStringOffset(Str: Def->getGuard()) << " /* " << Def->getGuard()
1592 << " */}, ";
1593 OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
1594 }
1595 }
1596 for (const auto &[Name, Type] : ReinterpretBuiltins) {
1597 OS << " Builtin::Info{Builtin::Info::StrOffsets{"
1598 << Table.GetStringOffset(Str: Name) << " /* " << Name << " */, ";
1599 OS << Table.GetStringOffset(Str: Type) << " /* " << Type << " */, ";
1600 OS << Table.GetStringOffset(Str: "n") << " /* n */, ";
1601 OS << Table.GetStringOffset(Str: "sme|sve") << " /* sme|sve */}, ";
1602 OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
1603 }
1604 OS << "#endif // GET_SVE_BUILTIN_INFOS\n\n";
1605}
1606
1607void SVEEmitter::createBuiltinsJSON(raw_ostream &OS) {
1608 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1609 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1610 for (auto *R : RV)
1611 createIntrinsic(R, Out&: Defs);
1612
1613 OS << "[\n";
1614 bool FirstDef = true;
1615
1616 for (auto &Def : Defs) {
1617 std::vector<std::string> Flags;
1618
1619 if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsStreaming")))
1620 Flags.push_back(x: "streaming-only");
1621 else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsStreamingCompatible")))
1622 Flags.push_back(x: "streaming-compatible");
1623 else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "VerifyRuntimeMode")))
1624 Flags.push_back(x: "feature-dependent");
1625
1626 if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInZA")) ||
1627 Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsOutZA")) ||
1628 Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInOutZA")))
1629 Flags.push_back(x: "requires-za");
1630
1631 if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInZT0")) ||
1632 Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsOutZT0")) ||
1633 Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInOutZT0")))
1634 Flags.push_back(x: "requires-zt");
1635
1636 if (!FirstDef)
1637 OS << ",\n";
1638
1639 OS << "{ ";
1640 OS << "\"guard\": \"" << Def->getSVEGuard() << "\",";
1641 OS << "\"streaming_guard\": \"" << Def->getSMEGuard() << "\",";
1642 OS << "\"flags\": \"";
1643
1644 for (size_t I = 0; I < Flags.size(); ++I) {
1645 if (I != 0)
1646 OS << ',';
1647 OS << Flags[I];
1648 }
1649
1650 OS << "\",\"builtin\": \"";
1651
1652 std::string BuiltinName = Def->getMangledName(CK: Def->getClassKind());
1653
1654 OS << Def->getReturnType().str() << " " << BuiltinName << "(";
1655 for (unsigned I = 0; I < Def->getTypes().size() - 1; ++I) {
1656 if (I != 0)
1657 OS << ", ";
1658
1659 SVEType ParamType = Def->getParamType(I);
1660
1661 // These are ImmCheck'd but their type names are sufficiently clear.
1662 if (ParamType.isPredicatePattern() || ParamType.isPrefetchOp()) {
1663 OS << ParamType.str();
1664 continue;
1665 }
1666
1667 // Pass ImmCheck information by pretending it's a type.
1668 auto Iter = llvm::find_if(Range: Def->getImmChecks(), P: [I](const auto &Chk) {
1669 return (unsigned)Chk.getImmArgIdx() == I;
1670 });
1671 if (Iter != Def->getImmChecks().end())
1672 OS << getImmCheckForEnumValue(Id: Iter->getKind());
1673 else
1674 OS << ParamType.str();
1675 }
1676 OS << ");\" }";
1677 FirstDef = false;
1678 }
1679
1680 OS << "\n]\n";
1681}
1682
1683void SVEEmitter::createCodeGenMap(raw_ostream &OS) {
1684 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1685 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1686 for (auto *R : RV)
1687 createIntrinsic(R, Out&: Defs);
1688
1689 // The mappings must be sorted based on BuiltinID.
1690 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1691 const std::unique_ptr<Intrinsic> &B) {
1692 return A->getMangledName() < B->getMangledName();
1693 });
1694
1695 OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n";
1696 for (auto &Def : Defs) {
1697 // Builtins only exist for non-overloaded intrinsics, overloaded
1698 // declarations only live in the header file.
1699 if (Def->getClassKind() == ClassG)
1700 continue;
1701
1702 uint64_t Flags = Def->getFlags();
1703 auto FlagString = std::to_string(val: Flags);
1704
1705 std::string LLVMName = Def->getMangledLLVMName();
1706 std::string Builtin = Def->getMangledName();
1707 if (!LLVMName.empty())
1708 OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1709 << "),\n";
1710 else
1711 OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n";
1712 }
1713 OS << "#endif\n\n";
1714}
1715
1716void SVEEmitter::createRangeChecks(raw_ostream &OS) {
1717 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1718 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1719 for (auto *R : RV)
1720 createIntrinsic(R, Out&: Defs);
1721
1722 // The mappings must be sorted based on BuiltinID.
1723 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1724 const std::unique_ptr<Intrinsic> &B) {
1725 return A->getMangledName() < B->getMangledName();
1726 });
1727
1728 OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n";
1729
1730 // Ensure these are only emitted once.
1731 std::set<std::string> Emitted;
1732
1733 for (auto &Def : Defs) {
1734 if (Emitted.find(x: Def->getMangledName()) != Emitted.end() ||
1735 Def->getImmChecks().empty())
1736 continue;
1737
1738 OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n";
1739 for (auto &Check : Def->getImmChecks())
1740 OS << "ImmChecks.emplace_back(" << Check.getImmArgIdx() << ", "
1741 << Check.getKind() << ", " << Check.getElementSizeInBits() << ");\n";
1742 OS << " break;\n";
1743
1744 Emitted.insert(x: Def->getMangledName());
1745 }
1746
1747 OS << "#endif\n\n";
1748}
1749
1750/// Create the SVETypeFlags used in CGBuiltins
1751void SVEEmitter::createTypeFlags(raw_ostream &OS) {
1752 OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n";
1753 for (auto &KV : FlagTypes)
1754 OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n";
1755 OS << "#endif\n\n";
1756
1757 OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n";
1758 for (auto &KV : EltTypes)
1759 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1760 OS << "#endif\n\n";
1761
1762 OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n";
1763 for (auto &KV : MemEltTypes)
1764 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1765 OS << "#endif\n\n";
1766
1767 OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n";
1768 for (auto &KV : MergeTypes)
1769 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1770 OS << "#endif\n\n";
1771}
1772
1773void SVEEmitter::createImmCheckTypes(raw_ostream &OS) {
1774 OS << "#ifdef LLVM_GET_ARM_INTRIN_IMMCHECKTYPES\n";
1775 for (auto &KV : ImmCheckTypes)
1776 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1777 OS << "#endif\n\n";
1778}
1779
1780void SVEEmitter::createSMEHeader(raw_ostream &OS) {
1781 OS << "/*===---- arm_sme.h - ARM SME intrinsics "
1782 "------===\n"
1783 " *\n"
1784 " *\n"
1785 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1786 "Exceptions.\n"
1787 " * See https://llvm.org/LICENSE.txt for license information.\n"
1788 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1789 " *\n"
1790 " *===-----------------------------------------------------------------"
1791 "------===\n"
1792 " */\n\n";
1793
1794 OS << "#ifndef __ARM_SME_H\n";
1795 OS << "#define __ARM_SME_H\n\n";
1796
1797 OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1798 OS << "#error \"Big endian is currently not supported for arm_sme.h\"\n";
1799 OS << "#endif\n";
1800
1801 OS << "#include <arm_sve.h>\n\n";
1802 OS << "#include <stddef.h>\n\n";
1803
1804 OS << "/* Function attributes */\n";
1805 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1806 "__nodebug__))\n\n";
1807 OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1808 "__nodebug__, __overloadable__))\n\n";
1809
1810 OS << "#ifdef __cplusplus\n";
1811 OS << "extern \"C\" {\n";
1812 OS << "#endif\n\n";
1813
1814 OS << "void __arm_za_disable(void) __arm_streaming_compatible;\n\n";
1815
1816 OS << "__ai bool __arm_has_sme(void) __arm_streaming_compatible {\n";
1817 OS << " uint64_t x0, x1;\n";
1818 OS << " __builtin_arm_get_sme_state(&x0, &x1);\n";
1819 OS << " return x0 & (1ULL << 63);\n";
1820 OS << "}\n\n";
1821
1822 OS << "void *__arm_sc_memcpy(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
1823 OS << "void *__arm_sc_memmove(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
1824 OS << "void *__arm_sc_memset(void *s, int c, size_t n) __arm_streaming_compatible;\n";
1825 OS << "void *__arm_sc_memchr(void *s, int c, size_t n) __arm_streaming_compatible;\n\n";
1826
1827 OS << "__ai __attribute__((target(\"sme\"))) void svundef_za(void) "
1828 "__arm_streaming_compatible __arm_out(\"za\") "
1829 "{ }\n\n";
1830
1831 createCoreHeaderIntrinsics(OS, Emitter&: *this, Kind: ACLEKind::SME);
1832
1833 OS << "#ifdef __cplusplus\n";
1834 OS << "} // extern \"C\"\n";
1835 OS << "#endif\n\n";
1836 OS << "#undef __ai\n\n";
1837 OS << "#endif /* __ARM_SME_H */\n";
1838}
1839
1840void SVEEmitter::createSMEBuiltins(raw_ostream &OS) {
1841 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1842 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1843 for (auto *R : RV) {
1844 createIntrinsic(R, Out&: Defs);
1845 }
1846
1847 // The mappings must be sorted based on BuiltinID.
1848 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1849 const std::unique_ptr<Intrinsic> &B) {
1850 return A->getMangledName() < B->getMangledName();
1851 });
1852
1853 llvm::StringToOffsetTable Table;
1854 Table.GetOrAddStringOffset(Str: "");
1855 Table.GetOrAddStringOffset(Str: "n");
1856
1857 for (const auto &Def : Defs)
1858 if (Def->getClassKind() != ClassG) {
1859 Table.GetOrAddStringOffset(Str: Def->getMangledName());
1860 Table.GetOrAddStringOffset(Str: Def->getBuiltinTypeStr());
1861 Table.GetOrAddStringOffset(Str: Def->getGuard());
1862 }
1863
1864 OS << "#ifdef GET_SME_BUILTIN_ENUMERATORS\n";
1865 for (const auto &Def : Defs)
1866 if (Def->getClassKind() != ClassG)
1867 OS << " BI__builtin_sme_" << Def->getMangledName() << ",\n";
1868 OS << "#endif // GET_SME_BUILTIN_ENUMERATORS\n\n";
1869
1870 OS << "#ifdef GET_SME_BUILTIN_STR_TABLE\n";
1871 Table.EmitStringTableDef(OS, Name: "BuiltinStrings");
1872 OS << "#endif // GET_SME_BUILTIN_STR_TABLE\n\n";
1873
1874 OS << "#ifdef GET_SME_BUILTIN_INFOS\n";
1875 for (const auto &Def : Defs) {
1876 // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1877 // declarations only live in the header file.
1878 if (Def->getClassKind() != ClassG) {
1879 OS << " Builtin::Info{Builtin::Info::StrOffsets{"
1880 << Table.GetStringOffset(Str: Def->getMangledName()) << " /* "
1881 << Def->getMangledName() << " */, ";
1882 OS << Table.GetStringOffset(Str: Def->getBuiltinTypeStr()) << " /* "
1883 << Def->getBuiltinTypeStr() << " */, ";
1884 OS << Table.GetStringOffset(Str: "n") << " /* n */, ";
1885 OS << Table.GetStringOffset(Str: Def->getGuard()) << " /* " << Def->getGuard()
1886 << " */}, ";
1887 OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
1888 }
1889 }
1890 OS << "#endif // GET_SME_BUILTIN_INFOS\n\n";
1891}
1892
1893void SVEEmitter::createSMECodeGenMap(raw_ostream &OS) {
1894 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1895 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1896 for (auto *R : RV) {
1897 createIntrinsic(R, Out&: Defs);
1898 }
1899
1900 // The mappings must be sorted based on BuiltinID.
1901 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1902 const std::unique_ptr<Intrinsic> &B) {
1903 return A->getMangledName() < B->getMangledName();
1904 });
1905
1906 OS << "#ifdef GET_SME_LLVM_INTRINSIC_MAP\n";
1907 for (auto &Def : Defs) {
1908 // Builtins only exist for non-overloaded intrinsics, overloaded
1909 // declarations only live in the header file.
1910 if (Def->getClassKind() == ClassG)
1911 continue;
1912
1913 uint64_t Flags = Def->getFlags();
1914 auto FlagString = std::to_string(val: Flags);
1915
1916 std::string LLVMName = Def->getLLVMName();
1917 std::string Builtin = Def->getMangledName();
1918 if (!LLVMName.empty())
1919 OS << "SMEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1920 << "),\n";
1921 else
1922 OS << "SMEMAP2(" << Builtin << ", " << FlagString << "),\n";
1923 }
1924 OS << "#endif\n\n";
1925}
1926
1927void SVEEmitter::createSMERangeChecks(raw_ostream &OS) {
1928 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1929 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1930 for (auto *R : RV) {
1931 createIntrinsic(R, Out&: Defs);
1932 }
1933
1934 // The mappings must be sorted based on BuiltinID.
1935 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1936 const std::unique_ptr<Intrinsic> &B) {
1937 return A->getMangledName() < B->getMangledName();
1938 });
1939
1940 OS << "#ifdef GET_SME_IMMEDIATE_CHECK\n";
1941
1942 // Ensure these are only emitted once.
1943 std::set<std::string> Emitted;
1944
1945 for (auto &Def : Defs) {
1946 if (Emitted.find(x: Def->getMangledName()) != Emitted.end() ||
1947 Def->getImmChecks().empty())
1948 continue;
1949
1950 OS << "case SME::BI__builtin_sme_" << Def->getMangledName() << ":\n";
1951 for (auto &Check : Def->getImmChecks())
1952 OS << "ImmChecks.push_back(std::make_tuple(" << Check.getImmArgIdx()
1953 << ", " << Check.getKind() << ", " << Check.getElementSizeInBits()
1954 << "));\n";
1955 OS << " break;\n";
1956
1957 Emitted.insert(x: Def->getMangledName());
1958 }
1959
1960 OS << "#endif\n\n";
1961}
1962
1963void SVEEmitter::createBuiltinZAState(raw_ostream &OS) {
1964 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1965 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1966 for (auto *R : RV)
1967 createIntrinsic(R, Out&: Defs);
1968
1969 std::map<std::string, std::set<std::string>> IntrinsicsPerState;
1970 for (auto &Def : Defs) {
1971 std::string Key;
1972 auto AddToKey = [&Key](const std::string &S) -> void {
1973 Key = Key.empty() ? S : (Key + " | " + S);
1974 };
1975
1976 if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInZA")))
1977 AddToKey("ArmInZA");
1978 else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsOutZA")))
1979 AddToKey("ArmOutZA");
1980 else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInOutZA")))
1981 AddToKey("ArmInOutZA");
1982
1983 if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInZT0")))
1984 AddToKey("ArmInZT0");
1985 else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsOutZT0")))
1986 AddToKey("ArmOutZT0");
1987 else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInOutZT0")))
1988 AddToKey("ArmInOutZT0");
1989
1990 if (!Key.empty())
1991 IntrinsicsPerState[Key].insert(x: Def->getMangledName());
1992 }
1993
1994 OS << "#ifdef GET_SME_BUILTIN_GET_STATE\n";
1995 for (auto &KV : IntrinsicsPerState) {
1996 for (StringRef Name : KV.second)
1997 OS << "case SME::BI__builtin_sme_" << Name << ":\n";
1998 OS << " return " << KV.first << ";\n";
1999 }
2000 OS << "#endif\n\n";
2001}
2002
2003void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) {
2004 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
2005 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
2006 for (auto *R : RV)
2007 createIntrinsic(R, Out&: Defs);
2008
2009 StringRef ExtensionKind;
2010 switch (Kind) {
2011 case ACLEKind::SME:
2012 ExtensionKind = "SME";
2013 break;
2014 case ACLEKind::SVE:
2015 ExtensionKind = "SVE";
2016 break;
2017 }
2018
2019 OS << "#ifdef GET_" << ExtensionKind << "_STREAMING_ATTRS\n";
2020
2021 StringMap<std::set<std::string>> StreamingMap;
2022
2023 uint64_t IsStreamingFlag = getEnumValueForFlag(C: "IsStreaming");
2024 uint64_t VerifyRuntimeMode = getEnumValueForFlag(C: "VerifyRuntimeMode");
2025 uint64_t IsStreamingCompatibleFlag =
2026 getEnumValueForFlag(C: "IsStreamingCompatible");
2027
2028 for (auto &Def : Defs) {
2029 if (!Def->isFlagSet(Flag: VerifyRuntimeMode) && !Def->getSVEGuard().empty() &&
2030 !Def->getSMEGuard().empty())
2031 report_fatal_error(reason: "Missing VerifyRuntimeMode flag");
2032 if (Def->isFlagSet(Flag: VerifyRuntimeMode) &&
2033 (Def->getSVEGuard().empty() || Def->getSMEGuard().empty()))
2034 report_fatal_error(reason: "VerifyRuntimeMode requires SVE and SME guards");
2035
2036 if (Def->isFlagSet(Flag: IsStreamingFlag))
2037 StreamingMap["ArmStreaming"].insert(x: Def->getMangledName());
2038 else if (Def->isFlagSet(Flag: VerifyRuntimeMode))
2039 StreamingMap["VerifyRuntimeMode"].insert(x: Def->getMangledName());
2040 else if (Def->isFlagSet(Flag: IsStreamingCompatibleFlag))
2041 StreamingMap["ArmStreamingCompatible"].insert(x: Def->getMangledName());
2042 else
2043 StreamingMap["ArmNonStreaming"].insert(x: Def->getMangledName());
2044 }
2045
2046 for (auto BuiltinType : StreamingMap.keys()) {
2047 for (auto Name : StreamingMap[BuiltinType]) {
2048 OS << "case " << ExtensionKind << "::BI__builtin_"
2049 << ExtensionKind.lower() << "_";
2050 OS << Name << ":\n";
2051 }
2052 OS << " BuiltinType = " << BuiltinType << ";\n";
2053 OS << " break;\n";
2054 }
2055
2056 OS << "#endif\n\n";
2057}
2058
2059namespace clang {
2060void EmitSveHeader(const RecordKeeper &Records, raw_ostream &OS) {
2061 SVEEmitter(Records).createHeader(OS);
2062}
2063
2064void EmitSveBuiltins(const RecordKeeper &Records, raw_ostream &OS) {
2065 SVEEmitter(Records).createBuiltins(OS);
2066}
2067
2068void EmitSveBuiltinsJSON(const RecordKeeper &Records, raw_ostream &OS) {
2069 SVEEmitter(Records).createBuiltinsJSON(OS);
2070}
2071
2072void EmitSveBuiltinCG(const RecordKeeper &Records, raw_ostream &OS) {
2073 SVEEmitter(Records).createCodeGenMap(OS);
2074}
2075
2076void EmitSveRangeChecks(const RecordKeeper &Records, raw_ostream &OS) {
2077 SVEEmitter(Records).createRangeChecks(OS);
2078}
2079
2080void EmitSveTypeFlags(const RecordKeeper &Records, raw_ostream &OS) {
2081 SVEEmitter(Records).createTypeFlags(OS);
2082}
2083
2084void EmitImmCheckTypes(const RecordKeeper &Records, raw_ostream &OS) {
2085 SVEEmitter(Records).createImmCheckTypes(OS);
2086}
2087
2088void EmitSveStreamingAttrs(const RecordKeeper &Records, raw_ostream &OS) {
2089 SVEEmitter(Records).createStreamingAttrs(OS, Kind: ACLEKind::SVE);
2090}
2091
2092void EmitSmeHeader(const RecordKeeper &Records, raw_ostream &OS) {
2093 SVEEmitter(Records).createSMEHeader(OS);
2094}
2095
2096void EmitSmeBuiltins(const RecordKeeper &Records, raw_ostream &OS) {
2097 SVEEmitter(Records).createSMEBuiltins(OS);
2098}
2099
2100void EmitSmeBuiltinsJSON(const RecordKeeper &Records, raw_ostream &OS) {
2101 SVEEmitter(Records).createBuiltinsJSON(OS);
2102}
2103
2104void EmitSmeBuiltinCG(const RecordKeeper &Records, raw_ostream &OS) {
2105 SVEEmitter(Records).createSMECodeGenMap(OS);
2106}
2107
2108void EmitSmeRangeChecks(const RecordKeeper &Records, raw_ostream &OS) {
2109 SVEEmitter(Records).createSMERangeChecks(OS);
2110}
2111
2112void EmitSmeStreamingAttrs(const RecordKeeper &Records, raw_ostream &OS) {
2113 SVEEmitter(Records).createStreamingAttrs(OS, Kind: ACLEKind::SME);
2114}
2115
2116void EmitSmeBuiltinZAState(const RecordKeeper &Records, raw_ostream &OS) {
2117 SVEEmitter(Records).createBuiltinZAState(OS);
2118}
2119} // End namespace clang
2120