1//===-- SveEmitter.cpp - Generate arm_sve.h for use with clang ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This tablegen backend is responsible for emitting arm_sve.h, which includes
10// a declaration and definition of each function specified by the ARM C/C++
11// Language Extensions (ACLE).
12//
13// For details, visit:
14// https://developer.arm.com/architectures/system-architectures/software-standards/acle
15//
16// Each SVE instruction is implemented in terms of 1 or more functions which
17// are suffixed with the element type of the input vectors. Functions may be
18// implemented in terms of generic vector operations such as +, *, -, etc. or
19// by calling a __builtin_-prefixed function which will be handled by clang's
20// CodeGen library.
21//
22// See also the documentation in include/clang/Basic/arm_sve.td.
23//
24//===----------------------------------------------------------------------===//
25
26#include "llvm/ADT/ArrayRef.h"
27#include "llvm/ADT/STLExtras.h"
28#include "llvm/ADT/StringExtras.h"
29#include "llvm/ADT/StringMap.h"
30#include "llvm/Support/raw_ostream.h"
31#include "llvm/TableGen/AArch64ImmCheck.h"
32#include "llvm/TableGen/Error.h"
33#include "llvm/TableGen/Record.h"
34#include "llvm/TableGen/StringToOffsetTable.h"
35#include <array>
36#include <cctype>
37#include <set>
38#include <string>
39#include <tuple>
40
41using namespace llvm;
42
43enum ClassKind {
44 ClassNone,
45 ClassS, // signed/unsigned, e.g., "_s8", "_u8" suffix
46 ClassG, // Overloaded name without type suffix
47};
48
49enum class ACLEKind { SVE, SME };
50
51using TypeSpec = std::string;
52
53namespace {
54class SVEType {
55
56 enum TypeKind {
57 Invalid,
58 Void,
59 Float,
60 SInt,
61 UInt,
62 BFloat16,
63 MFloat8,
64 Svcount,
65 PrefetchOp,
66 PredicatePattern,
67 Predicate,
68 Fpm
69 };
70 TypeKind Kind;
71 bool Immediate, Constant, Pointer, DefaultType, IsScalable;
72 unsigned Bitwidth, ElementBitwidth, NumVectors;
73
74public:
75 SVEType() : SVEType("", 'v') {}
76
77 SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1)
78 : Kind(Invalid), Immediate(false), Constant(false), Pointer(false),
79 DefaultType(false), IsScalable(true), Bitwidth(128),
80 ElementBitwidth(~0U), NumVectors(NumVectors) {
81 if (!TS.empty())
82 applyTypespec(TS);
83 applyModifier(Mod: CharMod);
84 }
85
86 SVEType(const SVEType &Base, unsigned NumV) : SVEType(Base) {
87 NumVectors = NumV;
88 }
89
90 bool isPointer() const { return Pointer; }
91 bool isConstant() const { return Constant; }
92 bool isImmediate() const { return Immediate; }
93 bool isScalar() const { return NumVectors == 0; }
94 bool isVector() const { return NumVectors > 0; }
95 bool isScalableVector() const { return isVector() && IsScalable; }
96 bool isFixedLengthVector() const { return isVector() && !IsScalable; }
97 bool isChar() const { return ElementBitwidth == 8 && isInteger(); }
98 bool isVoid() const { return Kind == Void; }
99 bool isDefault() const { return DefaultType; }
100 bool isFloat() const { return Kind == Float; }
101 bool isBFloat() const { return Kind == BFloat16; }
102 bool isMFloat() const { return Kind == MFloat8; }
103 bool isFloatingPoint() const {
104 return Kind == Float || Kind == BFloat16 || Kind == MFloat8;
105 }
106 bool isInteger() const { return Kind == SInt || Kind == UInt; }
107 bool isSignedInteger() const { return Kind == SInt; }
108 bool isUnsignedInteger() const { return Kind == UInt; }
109 bool isScalarPredicate() const {
110 return Kind == Predicate && NumVectors == 0;
111 }
112 bool isPredicate() const { return Kind == Predicate; }
113 bool isPredicatePattern() const { return Kind == PredicatePattern; }
114 bool isPrefetchOp() const { return Kind == PrefetchOp; }
115 bool isSvcount() const { return Kind == Svcount; }
116 bool isFpm() const { return Kind == Fpm; }
117 bool isInvalid() const { return Kind == Invalid; }
118 unsigned getElementSizeInBits() const { return ElementBitwidth; }
119 unsigned getNumVectors() const { return NumVectors; }
120
121 unsigned getNumElements() const {
122 assert(ElementBitwidth != ~0U);
123 return isPredicate() ? 16 : (Bitwidth / ElementBitwidth);
124 }
125 unsigned getSizeInBits() const {
126 return Bitwidth;
127 }
128
129 /// Return the string representation of a type, which is an encoded
130 /// string for passing to the BUILTIN() macro in Builtins.def.
131 std::string builtin_str() const;
132
133 /// Return the C/C++ string representation of a type for use in the
134 /// arm_sve.h header file.
135 std::string str() const;
136
137private:
138 /// Creates the type based on the typespec string in TS.
139 void applyTypespec(StringRef TS);
140
141 /// Applies a prototype modifier to the type.
142 void applyModifier(char Mod);
143
144 /// Get the builtin base for this SVEType, e.g. 'Wi' for svint64_t.
145 std::string builtinBaseType() const;
146};
147
148class SVEEmitter;
149
150/// The main grunt class. This represents an instantiation of an intrinsic with
151/// a particular typespec and prototype.
152class Intrinsic {
153 /// The unmangled name.
154 std::string Name;
155
156 /// The name of the corresponding LLVM IR intrinsic.
157 std::string LLVMName;
158
159 /// Intrinsic prototype.
160 std::string Proto;
161
162 /// The base type spec for this intrinsic.
163 TypeSpec BaseTypeSpec;
164
165 /// The base class kind. Most intrinsics use ClassS, which has full type
166 /// info for integers (_s32/_u32), or ClassG which is used for overloaded
167 /// intrinsics.
168 ClassKind Class;
169
170 /// The architectural #ifdef guard.
171 std::string SVEGuard, SMEGuard;
172
173 // The merge suffix such as _m, _x or _z.
174 std::string MergeSuffix;
175
176 /// The types of return value [0] and parameters [1..].
177 std::vector<SVEType> Types;
178
179 /// The "base type", which is VarType('d', BaseTypeSpec).
180 SVEType BaseType;
181
182 uint64_t Flags;
183
184 SmallVector<ImmCheck, 2> ImmChecks;
185
186 bool SetsFPMR;
187
188public:
189 Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
190 StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName,
191 uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT,
192 ClassKind Class, SVEEmitter &Emitter, StringRef SVEGuard,
193 StringRef SMEGuard);
194
195 ~Intrinsic()=default;
196
197 std::string getName() const { return Name; }
198 std::string getLLVMName() const { return LLVMName; }
199 std::string getProto() const { return Proto; }
200 TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; }
201 SVEType getBaseType() const { return BaseType; }
202
203 StringRef getSVEGuard() const { return SVEGuard; }
204 StringRef getSMEGuard() const { return SMEGuard; }
205 std::string getGuard() const {
206 std::string Guard;
207 llvm::raw_string_ostream OS(Guard);
208 if (!SVEGuard.empty() && SMEGuard.empty())
209 OS << SVEGuard;
210 else if (SVEGuard.empty() && !SMEGuard.empty())
211 OS << SMEGuard;
212 else {
213 if (SVEGuard.find(s: ",") != std::string::npos ||
214 SVEGuard.find(s: "|") != std::string::npos)
215 OS << "(" << SVEGuard << ")";
216 else
217 OS << SVEGuard;
218 OS << "|";
219 if (SMEGuard.find(s: ",") != std::string::npos ||
220 SMEGuard.find(s: "|") != std::string::npos)
221 OS << "(" << SMEGuard << ")";
222 else
223 OS << SMEGuard;
224 }
225 return Guard;
226 }
227 ClassKind getClassKind() const { return Class; }
228
229 SVEType getReturnType() const { return Types[0]; }
230 ArrayRef<SVEType> getTypes() const { return Types; }
231 SVEType getParamType(unsigned I) const { return Types[I + 1]; }
232 unsigned getNumParams() const {
233 return Proto.size() - (2 * count(Range: Proto, Element: '.')) - 1;
234 }
235
236 uint64_t getFlags() const { return Flags; }
237 bool isFlagSet(uint64_t Flag) const { return Flags & Flag;}
238
239 ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; }
240
241 /// Return the type string for a BUILTIN() macro in Builtins.def.
242 std::string getBuiltinTypeStr();
243
244 /// Return the name, mangled with type information. The name is mangled for
245 /// ClassS, so will add type suffixes such as _u32/_s32.
246 std::string getMangledName() const { return mangleName(LocalCK: ClassS); }
247
248 /// As above, but mangles the LLVM name instead.
249 std::string getMangledLLVMName() const { return mangleLLVMName(); }
250
251 /// Returns true if the intrinsic is overloaded, in that it should also generate
252 /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of
253 /// 'svld1_u32(..)'.
254 static bool isOverloadedIntrinsic(StringRef Name) {
255 return Name.contains(C: '[') && Name.contains(C: ']');
256 }
257
258 /// Return true if the intrinsic takes a splat operand.
259 bool hasSplat() const {
260 // These prototype modifiers are described in arm_sve.td.
261 return Proto.find_first_of(s: "ajfrKLR@!") != std::string::npos;
262 }
263
264 /// Return the parameter index of the splat operand.
265 unsigned getSplatIdx() const {
266 unsigned I = 1, Param = 0;
267 for (; I < Proto.size(); ++I, ++Param) {
268 if (Proto[I] == 'a' || Proto[I] == 'j' || Proto[I] == 'f' ||
269 Proto[I] == 'r' || Proto[I] == 'K' || Proto[I] == 'L' ||
270 Proto[I] == 'R' || Proto[I] == '@' || Proto[I] == '!')
271 break;
272
273 // Multivector modifier can be skipped
274 if (Proto[I] == '.')
275 I += 2;
276 }
277 assert(I != Proto.size() && "Prototype has no splat operand");
278 return Param;
279 }
280
281 /// Emits the intrinsic declaration to the ostream.
282 void emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter, ACLEKind Kind) const;
283
284private:
285 std::string getMergeSuffix() const { return MergeSuffix; }
286 StringRef getFPMSuffix() const { return SetsFPMR ? "_fpm" : ""; }
287 std::string mangleName(ClassKind LocalCK) const;
288 std::string mangleLLVMName() const;
289 std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,
290 std::string Proto) const;
291};
292
293class SVEEmitter {
294private:
295 // The reinterpret builtins are generated separately because they
296 // need the cross product of all types (121 functions in total),
297 // which is inconvenient to specify in the arm_sve.td file or
298 // generate in CGBuiltin.cpp.
299 struct ReinterpretTypeInfo {
300 SVEType BaseType;
301 const char *Suffix;
302 };
303
304 static const std::array<ReinterpretTypeInfo, 13> Reinterprets;
305
306 const RecordKeeper &Records;
307 StringMap<uint64_t> EltTypes;
308 StringMap<uint64_t> MemEltTypes;
309 StringMap<uint64_t> FlagTypes;
310 StringMap<uint64_t> MergeTypes;
311 StringMap<uint64_t> ImmCheckTypes;
312
313public:
314 SVEEmitter(const RecordKeeper &R) : Records(R) {
315 for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "EltType"))
316 EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
317 for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "MemEltType"))
318 MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
319 for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "FlagType"))
320 FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
321 for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "MergeType"))
322 MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
323 for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "ImmCheckType"))
324 ImmCheckTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
325 }
326
327 /// Returns the enum value for the immcheck type
328 unsigned getEnumValueForImmCheck(StringRef C) const {
329 auto It = ImmCheckTypes.find(Key: C);
330 if (It != ImmCheckTypes.end())
331 return It->getValue();
332 llvm_unreachable("Unsupported imm check");
333 }
334
335 /// Returns the enum value for the flag type
336 uint64_t getEnumValueForFlag(StringRef C) const {
337 auto Res = FlagTypes.find(Key: C);
338 if (Res != FlagTypes.end())
339 return Res->getValue();
340 llvm_unreachable("Unsupported flag");
341 }
342
343 // Returns the SVETypeFlags for a given value and mask.
344 uint64_t encodeFlag(uint64_t V, StringRef MaskName) const {
345 auto It = FlagTypes.find(Key: MaskName);
346 if (It != FlagTypes.end()) {
347 uint64_t Mask = It->getValue();
348 unsigned Shift = countr_zero(Val: Mask);
349 assert(Shift < 64 && "Mask value produced an invalid shift value");
350 return (V << Shift) & Mask;
351 }
352 llvm_unreachable("Unsupported flag");
353 }
354
355 // Returns the SVETypeFlags for the given element type.
356 uint64_t encodeEltType(StringRef EltName) {
357 auto It = EltTypes.find(Key: EltName);
358 if (It != EltTypes.end())
359 return encodeFlag(V: It->getValue(), MaskName: "EltTypeMask");
360 llvm_unreachable("Unsupported EltType");
361 }
362
363 // Returns the SVETypeFlags for the given memory element type.
364 uint64_t encodeMemoryElementType(uint64_t MT) {
365 return encodeFlag(V: MT, MaskName: "MemEltTypeMask");
366 }
367
368 // Returns the SVETypeFlags for the given merge type.
369 uint64_t encodeMergeType(uint64_t MT) {
370 return encodeFlag(V: MT, MaskName: "MergeTypeMask");
371 }
372
373 // Returns the SVETypeFlags for the given splat operand.
374 unsigned encodeSplatOperand(unsigned SplatIdx) {
375 assert(SplatIdx < 7 && "SplatIdx out of encodable range");
376 return encodeFlag(V: SplatIdx + 1, MaskName: "SplatOperandMask");
377 }
378
379 // Returns the SVETypeFlags value for the given SVEType.
380 uint64_t encodeTypeFlags(const SVEType &T);
381
382 /// Emit arm_sve.h.
383 void createHeader(raw_ostream &o);
384
385 // Emits core intrinsics in both arm_sme.h and arm_sve.h
386 void createCoreHeaderIntrinsics(raw_ostream &o, SVEEmitter &Emitter,
387 ACLEKind Kind);
388
389 /// Emit all the __builtin prototypes and code needed by Sema.
390 void createBuiltins(raw_ostream &o);
391
392 /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
393 void createCodeGenMap(raw_ostream &o);
394
395 /// Emit all the range checks for the immediates.
396 void createRangeChecks(raw_ostream &o);
397
398 // Emit all the ImmCheckTypes to arm_immcheck_types.inc
399 void createImmCheckTypes(raw_ostream &OS);
400
401 /// Create the SVETypeFlags used in CGBuiltins
402 void createTypeFlags(raw_ostream &o);
403
404 /// Emit arm_sme.h.
405 void createSMEHeader(raw_ostream &o);
406
407 /// Emit all the SME __builtin prototypes and code needed by Sema.
408 void createSMEBuiltins(raw_ostream &o);
409
410 /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
411 void createSMECodeGenMap(raw_ostream &o);
412
413 /// Create a table for a builtin's requirement for PSTATE.SM.
414 void createStreamingAttrs(raw_ostream &o, ACLEKind Kind);
415
416 /// Emit all the range checks for the immediates.
417 void createSMERangeChecks(raw_ostream &o);
418
419 /// Create a table for a builtin's requirement for PSTATE.ZA.
420 void createBuiltinZAState(raw_ostream &OS);
421
422 /// Create intrinsic and add it to \p Out
423 void createIntrinsic(const Record *R,
424 SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out);
425};
426
427const std::array<SVEEmitter::ReinterpretTypeInfo, 13> SVEEmitter::Reinterprets =
428 {._M_elems: {{.BaseType: SVEType("c", 'd'), .Suffix: "s8"},
429 {.BaseType: SVEType("Uc", 'd'), .Suffix: "u8"},
430 {.BaseType: SVEType("m", 'd'), .Suffix: "mf8"},
431 {.BaseType: SVEType("s", 'd'), .Suffix: "s16"},
432 {.BaseType: SVEType("Us", 'd'), .Suffix: "u16"},
433 {.BaseType: SVEType("i", 'd'), .Suffix: "s32"},
434 {.BaseType: SVEType("Ui", 'd'), .Suffix: "u32"},
435 {.BaseType: SVEType("l", 'd'), .Suffix: "s64"},
436 {.BaseType: SVEType("Ul", 'd'), .Suffix: "u64"},
437 {.BaseType: SVEType("h", 'd'), .Suffix: "f16"},
438 {.BaseType: SVEType("b", 'd'), .Suffix: "bf16"},
439 {.BaseType: SVEType("f", 'd'), .Suffix: "f32"},
440 {.BaseType: SVEType("d", 'd'), .Suffix: "f64"}}};
441
442} // end anonymous namespace
443
444//===----------------------------------------------------------------------===//
445// Type implementation
446//===----------------------------------------------------------------------===//
447
448std::string SVEType::builtinBaseType() const {
449 switch (Kind) {
450 case TypeKind::Void:
451 return "v";
452 case TypeKind::Svcount:
453 return "Qa";
454 case TypeKind::PrefetchOp:
455 case TypeKind::PredicatePattern:
456 return "i";
457 case TypeKind::Fpm:
458 return "UWi";
459 case TypeKind::Predicate:
460 return "b";
461 case TypeKind::BFloat16:
462 assert(ElementBitwidth == 16 && "Invalid BFloat16!");
463 return "y";
464 case TypeKind::MFloat8:
465 assert(ElementBitwidth == 8 && "Invalid MFloat8!");
466 return "m";
467 case TypeKind::Float:
468 switch (ElementBitwidth) {
469 case 16:
470 return "h";
471 case 32:
472 return "f";
473 case 64:
474 return "d";
475 default:
476 llvm_unreachable("Unhandled float width!");
477 }
478 case TypeKind::SInt:
479 case TypeKind::UInt:
480 switch (ElementBitwidth) {
481 case 1:
482 return "b";
483 case 8:
484 return "c";
485 case 16:
486 return "s";
487 case 32:
488 return "i";
489 case 64:
490 return "Wi";
491 case 128:
492 return "LLLi";
493 default:
494 llvm_unreachable("Unhandled bitwidth!");
495 }
496 case TypeKind::Invalid:
497 llvm_unreachable("Attempting to resolve builtin string from Invalid type!");
498 }
499 llvm_unreachable("Unhandled TypeKind!");
500}
501
502std::string SVEType::builtin_str() const {
503 std::string Prefix;
504
505 if (isScalableVector())
506 Prefix = "q" + llvm::utostr(X: getNumElements() * NumVectors);
507 else if (isFixedLengthVector())
508 Prefix = "V" + llvm::utostr(X: getNumElements() * NumVectors);
509 else if (isImmediate()) {
510 assert(!isFloatingPoint() && "fp immediates are not supported");
511 Prefix = "I";
512 }
513
514 // Make chars and integer pointers explicitly signed.
515 if ((ElementBitwidth == 8 || isPointer()) && isSignedInteger())
516 Prefix += "S";
517 else if (isUnsignedInteger())
518 Prefix += "U";
519
520 std::string BuiltinStr = Prefix + builtinBaseType();
521 if (isConstant())
522 BuiltinStr += "C";
523 if (isPointer())
524 BuiltinStr += "*";
525
526 return BuiltinStr;
527}
528
529std::string SVEType::str() const {
530 std::string TypeStr;
531
532 switch (Kind) {
533 case TypeKind::PrefetchOp:
534 return "enum svprfop";
535 case TypeKind::PredicatePattern:
536 return "enum svpattern";
537 case TypeKind::Fpm:
538 TypeStr += "fpm";
539 break;
540 case TypeKind::Void:
541 TypeStr += "void";
542 break;
543 case TypeKind::Float:
544 TypeStr += "float" + llvm::utostr(X: ElementBitwidth);
545 break;
546 case TypeKind::Svcount:
547 TypeStr += "svcount";
548 break;
549 case TypeKind::Predicate:
550 TypeStr += "bool";
551 break;
552 case TypeKind::BFloat16:
553 TypeStr += "bfloat16";
554 break;
555 case TypeKind::MFloat8:
556 TypeStr += "mfloat8";
557 break;
558 case TypeKind::SInt:
559 TypeStr += "int" + llvm::utostr(X: ElementBitwidth);
560 break;
561 case TypeKind::UInt:
562 TypeStr += "uint" + llvm::utostr(X: ElementBitwidth);
563 break;
564 case TypeKind::Invalid:
565 llvm_unreachable("Attempting to resolve type name from Invalid type!");
566 }
567
568 if (isFixedLengthVector())
569 TypeStr += "x" + llvm::utostr(X: getNumElements());
570 else if (isScalableVector())
571 TypeStr = "sv" + TypeStr;
572
573 if (NumVectors > 1)
574 TypeStr += "x" + llvm::utostr(X: NumVectors);
575 if (!isScalarPredicate() && !isVoid())
576 TypeStr += "_t";
577 if (isConstant())
578 TypeStr += " const";
579 if (isPointer())
580 TypeStr += " *";
581
582 return TypeStr;
583}
584
585void SVEType::applyTypespec(StringRef TS) {
586 for (char I : TS) {
587 switch (I) {
588 case 'Q':
589 assert(isInvalid() && "Unexpected use of typespec modifier");
590 Kind = Svcount;
591 break;
592 case 'P':
593 assert(isInvalid() && "Unexpected use of typespec modifier");
594 Kind = Predicate;
595 break;
596 case 'U':
597 assert(isInvalid() && "Unexpected use of typespec modifier");
598 Kind = UInt;
599 break;
600 case 'c':
601 Kind = isInvalid() ? SInt : Kind;
602 ElementBitwidth = 8;
603 break;
604 case 's':
605 Kind = isInvalid() ? SInt : Kind;
606 ElementBitwidth = 16;
607 break;
608 case 'i':
609 Kind = isInvalid() ? SInt : Kind;
610 ElementBitwidth = 32;
611 break;
612 case 'l':
613 Kind = isInvalid() ? SInt : Kind;
614 ElementBitwidth = 64;
615 break;
616 case 'q':
617 Kind = isInvalid() ? SInt : Kind;
618 ElementBitwidth = 128;
619 break;
620 case 'h':
621 assert(isInvalid() && "Unexpected use of typespec modifier");
622 Kind = Float;
623 ElementBitwidth = 16;
624 break;
625 case 'f':
626 assert(isInvalid() && "Unexpected use of typespec modifier");
627 Kind = Float;
628 ElementBitwidth = 32;
629 break;
630 case 'd':
631 assert(isInvalid() && "Unexpected use of typespec modifier");
632 Kind = Float;
633 ElementBitwidth = 64;
634 break;
635 case 'b':
636 assert(isInvalid() && "Unexpected use of typespec modifier");
637 Kind = BFloat16;
638 ElementBitwidth = 16;
639 break;
640 case 'm':
641 assert(isInvalid() && "Unexpected use of typespec modifier");
642 Kind = MFloat8;
643 ElementBitwidth = 8;
644 break;
645 default:
646 llvm_unreachable("Unhandled type code!");
647 }
648 }
649 assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
650}
651
652void SVEType::applyModifier(char Mod) {
653 switch (Mod) {
654 case 'v':
655 Kind = Void;
656 NumVectors = 0;
657 break;
658 case 'd':
659 DefaultType = true;
660 break;
661 case 'c':
662 Constant = true;
663 [[fallthrough]];
664 case 'p':
665 Pointer = true;
666 Bitwidth = ElementBitwidth;
667 NumVectors = 0;
668 break;
669 case 'e':
670 Kind = UInt;
671 ElementBitwidth /= 2;
672 break;
673 case 'h':
674 ElementBitwidth /= 2;
675 break;
676 case 'q':
677 ElementBitwidth /= 4;
678 break;
679 case 'b':
680 Kind = UInt;
681 ElementBitwidth /= 4;
682 break;
683 case 'o':
684 ElementBitwidth *= 4;
685 break;
686 case 'P':
687 Kind = Predicate;
688 Bitwidth = 16;
689 ElementBitwidth = 1;
690 break;
691 case '{':
692 IsScalable = false;
693 Bitwidth = 128;
694 NumVectors = 1;
695 break;
696 case 's':
697 case 'a':
698 Bitwidth = ElementBitwidth;
699 NumVectors = 0;
700 break;
701 case 'R':
702 ElementBitwidth /= 2;
703 NumVectors = 0;
704 break;
705 case 'r':
706 ElementBitwidth /= 4;
707 NumVectors = 0;
708 break;
709 case '@':
710 Kind = UInt;
711 ElementBitwidth /= 4;
712 NumVectors = 0;
713 break;
714 case 'K':
715 Kind = SInt;
716 Bitwidth = ElementBitwidth;
717 NumVectors = 0;
718 break;
719 case 'L':
720 Kind = UInt;
721 Bitwidth = ElementBitwidth;
722 NumVectors = 0;
723 break;
724 case 'u':
725 Kind = UInt;
726 break;
727 case 'x':
728 Kind = SInt;
729 break;
730 case 'i':
731 Kind = UInt;
732 ElementBitwidth = Bitwidth = 64;
733 NumVectors = 0;
734 Immediate = true;
735 break;
736 case 'I':
737 Kind = PredicatePattern;
738 ElementBitwidth = Bitwidth = 32;
739 NumVectors = 0;
740 Immediate = true;
741 break;
742 case 'J':
743 Kind = PrefetchOp;
744 ElementBitwidth = Bitwidth = 32;
745 NumVectors = 0;
746 Immediate = true;
747 break;
748 case 'k':
749 Kind = SInt;
750 ElementBitwidth = Bitwidth = 32;
751 NumVectors = 0;
752 break;
753 case 'l':
754 Kind = SInt;
755 ElementBitwidth = Bitwidth = 64;
756 NumVectors = 0;
757 break;
758 case 'm':
759 Kind = UInt;
760 ElementBitwidth = Bitwidth = 32;
761 NumVectors = 0;
762 break;
763 case '>':
764 Kind = Fpm;
765 ElementBitwidth = Bitwidth = 64;
766 NumVectors = 0;
767 break;
768 case 'n':
769 Kind = UInt;
770 ElementBitwidth = Bitwidth = 64;
771 NumVectors = 0;
772 break;
773 case 'w':
774 ElementBitwidth = 64;
775 break;
776 case 'j':
777 ElementBitwidth = Bitwidth = 64;
778 NumVectors = 0;
779 break;
780 case 'f':
781 Kind = UInt;
782 ElementBitwidth = Bitwidth = 64;
783 NumVectors = 0;
784 break;
785 case 'g':
786 Kind = UInt;
787 ElementBitwidth = 64;
788 break;
789 case '#':
790 Kind = SInt;
791 ElementBitwidth = 64;
792 break;
793 case '[':
794 Kind = UInt;
795 ElementBitwidth = 8;
796 break;
797 case 't':
798 Kind = SInt;
799 ElementBitwidth = 32;
800 break;
801 case 'z':
802 Kind = UInt;
803 ElementBitwidth = 32;
804 break;
805 case 'O':
806 Kind = Float;
807 ElementBitwidth = 16;
808 break;
809 case 'M':
810 Kind = Float;
811 ElementBitwidth = 32;
812 break;
813 case 'N':
814 Kind = Float;
815 ElementBitwidth = 64;
816 break;
817 case 'Q':
818 Kind = Void;
819 Constant = true;
820 Pointer = true;
821 NumVectors = 0;
822 break;
823 case 'S':
824 Kind = SInt;
825 Constant = true;
826 Pointer = true;
827 ElementBitwidth = Bitwidth = 8;
828 NumVectors = 0;
829 break;
830 case 'W':
831 Kind = UInt;
832 Constant = true;
833 Pointer = true;
834 ElementBitwidth = Bitwidth = 8;
835 NumVectors = 0;
836 break;
837 case 'T':
838 Kind = SInt;
839 Constant = true;
840 Pointer = true;
841 ElementBitwidth = Bitwidth = 16;
842 NumVectors = 0;
843 break;
844 case 'X':
845 Kind = UInt;
846 Constant = true;
847 Pointer = true;
848 ElementBitwidth = Bitwidth = 16;
849 NumVectors = 0;
850 break;
851 case 'Y':
852 Kind = UInt;
853 Constant = true;
854 Pointer = true;
855 ElementBitwidth = Bitwidth = 32;
856 NumVectors = 0;
857 break;
858 case 'U':
859 Kind = SInt;
860 Constant = true;
861 Pointer = true;
862 ElementBitwidth = Bitwidth = 32;
863 NumVectors = 0;
864 break;
865 case '%':
866 Kind = Void;
867 Pointer = true;
868 NumVectors = 0;
869 break;
870 case 'A':
871 Kind = SInt;
872 Pointer = true;
873 ElementBitwidth = Bitwidth = 8;
874 NumVectors = 0;
875 break;
876 case 'B':
877 Kind = SInt;
878 Pointer = true;
879 ElementBitwidth = Bitwidth = 16;
880 NumVectors = 0;
881 break;
882 case 'C':
883 Kind = SInt;
884 Pointer = true;
885 ElementBitwidth = Bitwidth = 32;
886 NumVectors = 0;
887 break;
888 case 'D':
889 Kind = SInt;
890 Pointer = true;
891 ElementBitwidth = Bitwidth = 64;
892 NumVectors = 0;
893 break;
894 case 'E':
895 Kind = UInt;
896 Pointer = true;
897 ElementBitwidth = Bitwidth = 8;
898 NumVectors = 0;
899 break;
900 case 'F':
901 Kind = UInt;
902 Pointer = true;
903 ElementBitwidth = Bitwidth = 16;
904 NumVectors = 0;
905 break;
906 case 'G':
907 Kind = UInt;
908 Pointer = true;
909 ElementBitwidth = Bitwidth = 32;
910 NumVectors = 0;
911 break;
912 case '$':
913 Kind = BFloat16;
914 ElementBitwidth = 16;
915 break;
916 case '}':
917 Kind = Svcount;
918 NumVectors = 0;
919 break;
920 case '~':
921 Kind = MFloat8;
922 ElementBitwidth = 8;
923 break;
924 case '!':
925 Kind = MFloat8;
926 Bitwidth = ElementBitwidth = 8;
927 NumVectors = 0;
928 break;
929 case '.':
930 llvm_unreachable(". is never a type in itself");
931 break;
932 default:
933 llvm_unreachable("Unhandled character!");
934 }
935}
936
937/// Returns the modifier and number of vectors for the given operand \p Op.
938std::pair<char, unsigned> getProtoModifier(StringRef Proto, unsigned Op) {
939 for (unsigned P = 0; !Proto.empty(); ++P) {
940 unsigned NumVectors = 1;
941 unsigned CharsToSkip = 1;
942 char Mod = Proto[0];
943 if (Mod == '2' || Mod == '3' || Mod == '4') {
944 NumVectors = Mod - '0';
945 Mod = 'd';
946 if (Proto.size() > 1 && Proto[1] == '.') {
947 Mod = Proto[2];
948 CharsToSkip = 3;
949 }
950 }
951
952 if (P == Op)
953 return {Mod, NumVectors};
954
955 Proto = Proto.drop_front(N: CharsToSkip);
956 }
957 llvm_unreachable("Unexpected Op");
958}
959
960//===----------------------------------------------------------------------===//
961// Intrinsic implementation
962//===----------------------------------------------------------------------===//
963
964Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
965 StringRef MergeSuffix, uint64_t MemoryElementTy,
966 StringRef LLVMName, uint64_t Flags,
967 ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class,
968 SVEEmitter &Emitter, StringRef SVEGuard,
969 StringRef SMEGuard)
970 : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()),
971 BaseTypeSpec(BT), Class(Class), MergeSuffix(MergeSuffix.str()),
972 BaseType(BT, 'd'), Flags(Flags), ImmChecks(Checks) {
973
974 auto FormatGuard = [](StringRef Guard, StringRef Base) -> std::string {
975 if (Guard.contains(C: '|'))
976 return Base.str() + ",(" + Guard.str() + ")";
977 if (Guard.empty() || Guard == Base || Guard.starts_with(Prefix: Base.str() + ","))
978 return Guard.str();
979 return Base.str() + "," + Guard.str();
980 };
981
982 this->SVEGuard = FormatGuard(SVEGuard, "sve");
983 this->SMEGuard = FormatGuard(SMEGuard, "sme");
984
985 // Types[0] is the return value.
986 for (unsigned I = 0; I < (getNumParams() + 1); ++I) {
987 char Mod;
988 unsigned NumVectors;
989 std::tie(args&: Mod, args&: NumVectors) = getProtoModifier(Proto, Op: I);
990 SVEType T(BaseTypeSpec, Mod, NumVectors);
991 Types.push_back(x: T);
992 SetsFPMR = T.isFpm();
993
994 // Add range checks for immediates
995 if (I > 0) {
996 if (T.isPredicatePattern())
997 ImmChecks.emplace_back(
998 Args: I - 1, Args: Emitter.getEnumValueForImmCheck(C: "ImmCheck0_31"));
999 else if (T.isPrefetchOp())
1000 ImmChecks.emplace_back(
1001 Args: I - 1, Args: Emitter.getEnumValueForImmCheck(C: "ImmCheck0_13"));
1002 }
1003 }
1004
1005 // Set flags based on properties
1006 this->Flags |= Emitter.encodeTypeFlags(T: BaseType);
1007 this->Flags |= Emitter.encodeMemoryElementType(MT: MemoryElementTy);
1008 this->Flags |= Emitter.encodeMergeType(MT: MergeTy);
1009 if (hasSplat())
1010 this->Flags |= Emitter.encodeSplatOperand(SplatIdx: getSplatIdx());
1011 if (SetsFPMR)
1012 this->Flags |= Emitter.getEnumValueForFlag(C: "SetsFPMR");
1013}
1014
1015std::string Intrinsic::getBuiltinTypeStr() {
1016 std::string S = getReturnType().builtin_str();
1017 for (unsigned I = 0; I < getNumParams(); ++I)
1018 S += getParamType(I).builtin_str();
1019
1020 return S;
1021}
1022
1023std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
1024 std::string Proto) const {
1025 std::string Ret = Name;
1026 while (Ret.find(c: '{') != std::string::npos) {
1027 size_t Pos = Ret.find(c: '{');
1028 size_t End = Ret.find(c: '}');
1029 unsigned NumChars = End - Pos + 1;
1030 assert(NumChars == 3 && "Unexpected template argument");
1031
1032 SVEType T;
1033 char C = Ret[Pos+1];
1034 switch(C) {
1035 default:
1036 llvm_unreachable("Unknown predication specifier");
1037 case 'd':
1038 T = SVEType(TS, 'd');
1039 break;
1040 case '0':
1041 case '1':
1042 case '2':
1043 case '3':
1044 // Extract the modifier before passing to SVEType to handle numeric
1045 // modifiers
1046 auto [Mod, NumVectors] = getProtoModifier(Proto, Op: (C - '0'));
1047 T = SVEType(TS, Mod);
1048 break;
1049 }
1050
1051 // Replace templated arg with the right suffix (e.g. u32)
1052 std::string TypeCode;
1053
1054 if (T.isSignedInteger())
1055 TypeCode = 's';
1056 else if (T.isUnsignedInteger())
1057 TypeCode = 'u';
1058 else if (T.isSvcount())
1059 TypeCode = 'c';
1060 else if (T.isPredicate())
1061 TypeCode = 'b';
1062 else if (T.isBFloat())
1063 TypeCode = "bf";
1064 else if (T.isMFloat())
1065 TypeCode = "mf";
1066 else
1067 TypeCode = 'f';
1068 Ret.replace(pos: Pos, n: NumChars, str: TypeCode + utostr(X: T.getElementSizeInBits()));
1069 }
1070
1071 return Ret;
1072}
1073
1074std::string Intrinsic::mangleLLVMName() const {
1075 std::string S = getLLVMName();
1076
1077 // Replace all {d} like expressions with e.g. 'u32'
1078 return replaceTemplatedArgs(Name: S, TS: getBaseTypeSpec(), Proto: getProto());
1079}
1080
1081std::string Intrinsic::mangleName(ClassKind LocalCK) const {
1082 std::string S = getName();
1083
1084 if (LocalCK == ClassG) {
1085 // Remove the square brackets and everything in between.
1086 while (S.find(c: '[') != std::string::npos) {
1087 auto Start = S.find(c: '[');
1088 auto End = S.find(c: ']');
1089 S.erase(pos: Start, n: (End-Start)+1);
1090 }
1091 } else {
1092 // Remove the square brackets.
1093 while (S.find(c: '[') != std::string::npos) {
1094 auto BrPos = S.find(c: '[');
1095 if (BrPos != std::string::npos)
1096 S.erase(pos: BrPos, n: 1);
1097 BrPos = S.find(c: ']');
1098 if (BrPos != std::string::npos)
1099 S.erase(pos: BrPos, n: 1);
1100 }
1101 }
1102
1103 // Replace all {d} like expressions with e.g. 'u32'
1104 return replaceTemplatedArgs(Name: S, TS: getBaseTypeSpec(), Proto: getProto())
1105 .append(str: getMergeSuffix())
1106 .append(svt: getFPMSuffix());
1107}
1108
1109void Intrinsic::emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter,
1110 ACLEKind Kind) const {
1111 bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1;
1112
1113 std::string FullName = mangleName(LocalCK: ClassS);
1114 std::string ProtoName = mangleName(LocalCK: getClassKind());
1115 OS << (IsOverloaded ? "__aio " : "__ai ")
1116 << "__attribute__((__clang_arm_builtin_alias(";
1117
1118 switch (Kind) {
1119 case ACLEKind::SME:
1120 OS << "__builtin_sme_" << FullName << ")";
1121 break;
1122 case ACLEKind::SVE:
1123 OS << "__builtin_sve_" << FullName << ")";
1124 break;
1125 }
1126
1127 OS << "))\n";
1128
1129 OS << getTypes()[0].str() << " " << ProtoName << "(";
1130 for (unsigned I = 0; I < getTypes().size() - 1; ++I) {
1131 if (I != 0)
1132 OS << ", ";
1133 OS << getTypes()[I + 1].str();
1134 }
1135 OS << ");\n";
1136}
1137
1138//===----------------------------------------------------------------------===//
1139// SVEEmitter implementation
1140//===----------------------------------------------------------------------===//
1141uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {
1142 if (T.isFloat()) {
1143 switch (T.getElementSizeInBits()) {
1144 case 16:
1145 return encodeEltType(EltName: "EltTyFloat16");
1146 case 32:
1147 return encodeEltType(EltName: "EltTyFloat32");
1148 case 64:
1149 return encodeEltType(EltName: "EltTyFloat64");
1150 default:
1151 llvm_unreachable("Unhandled float element bitwidth!");
1152 }
1153 }
1154
1155 if (T.isBFloat()) {
1156 assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat.");
1157 return encodeEltType(EltName: "EltTyBFloat16");
1158 }
1159
1160 if (T.isMFloat()) {
1161 assert(T.getElementSizeInBits() == 8 && "Not a valid MFloat.");
1162 return encodeEltType(EltName: "EltTyMFloat8");
1163 }
1164
1165 if (T.isPredicate() || T.isSvcount()) {
1166 switch (T.getElementSizeInBits()) {
1167 case 8:
1168 return encodeEltType(EltName: "EltTyBool8");
1169 case 16:
1170 return encodeEltType(EltName: "EltTyBool16");
1171 case 32:
1172 return encodeEltType(EltName: "EltTyBool32");
1173 case 64:
1174 return encodeEltType(EltName: "EltTyBool64");
1175 default:
1176 llvm_unreachable("Unhandled predicate element bitwidth!");
1177 }
1178 }
1179
1180 switch (T.getElementSizeInBits()) {
1181 case 8:
1182 return encodeEltType(EltName: "EltTyInt8");
1183 case 16:
1184 return encodeEltType(EltName: "EltTyInt16");
1185 case 32:
1186 return encodeEltType(EltName: "EltTyInt32");
1187 case 64:
1188 return encodeEltType(EltName: "EltTyInt64");
1189 case 128:
1190 return encodeEltType(EltName: "EltTyInt128");
1191 default:
1192 llvm_unreachable("Unhandled integer element bitwidth!");
1193 }
1194}
1195
1196void SVEEmitter::createIntrinsic(
1197 const Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) {
1198 StringRef Name = R->getValueAsString(FieldName: "Name");
1199 StringRef Proto = R->getValueAsString(FieldName: "Prototype");
1200 StringRef Types = R->getValueAsString(FieldName: "Types");
1201 StringRef SVEGuard = R->getValueAsString(FieldName: "SVETargetGuard");
1202 StringRef SMEGuard = R->getValueAsString(FieldName: "SMETargetGuard");
1203 StringRef LLVMName = R->getValueAsString(FieldName: "LLVMIntrinsic");
1204 uint64_t Merge = R->getValueAsInt(FieldName: "Merge");
1205 StringRef MergeSuffix = R->getValueAsString(FieldName: "MergeSuffix");
1206 uint64_t MemEltType = R->getValueAsInt(FieldName: "MemEltType");
1207
1208 int64_t Flags = 0;
1209 for (const Record *FlagRec : R->getValueAsListOfDefs(FieldName: "Flags"))
1210 Flags |= FlagRec->getValueAsInt(FieldName: "Value");
1211
1212 // Create a dummy TypeSpec for non-overloaded builtins.
1213 if (Types.empty()) {
1214 assert((Flags & getEnumValueForFlag("IsOverloadNone")) &&
1215 "Expect TypeSpec for overloaded builtin!");
1216 Types = "i";
1217 }
1218
1219 // Extract type specs from string
1220 SmallVector<TypeSpec, 8> TypeSpecs;
1221 TypeSpec Acc;
1222 for (char I : Types) {
1223 Acc.push_back(c: I);
1224 if (islower(I)) {
1225 TypeSpecs.push_back(Elt: TypeSpec(Acc));
1226 Acc.clear();
1227 }
1228 }
1229
1230 // Remove duplicate type specs.
1231 sort(C&: TypeSpecs);
1232 TypeSpecs.erase(CS: llvm::unique(R&: TypeSpecs), CE: TypeSpecs.end());
1233
1234 // Create an Intrinsic for each type spec.
1235 for (auto TS : TypeSpecs) {
1236 // Collate a list of range/option checks for the immediates.
1237 SmallVector<ImmCheck, 2> ImmChecks;
1238 for (const Record *ImmR : R->getValueAsListOfDefs(FieldName: "ImmChecks")) {
1239 int64_t ArgIdx = ImmR->getValueAsInt(FieldName: "ImmArgIdx");
1240 int64_t EltSizeArgIdx = ImmR->getValueAsInt(FieldName: "TypeContextArgIdx");
1241 int64_t Kind = ImmR->getValueAsDef(FieldName: "Kind")->getValueAsInt(FieldName: "Value");
1242 assert(ArgIdx >= 0 && Kind >= 0 &&
1243 "ImmArgIdx and Kind must be nonnegative");
1244
1245 unsigned ElementSizeInBits = 0;
1246 auto [Mod, NumVectors] = getProtoModifier(Proto, Op: EltSizeArgIdx + 1);
1247 if (EltSizeArgIdx >= 0)
1248 ElementSizeInBits = SVEType(TS, Mod, NumVectors).getElementSizeInBits();
1249 ImmChecks.push_back(Elt: ImmCheck(ArgIdx, Kind, ElementSizeInBits));
1250 }
1251
1252 Out.push_back(Elt: std::make_unique<Intrinsic>(
1253 args&: Name, args&: Proto, args&: Merge, args&: MergeSuffix, args&: MemEltType, args&: LLVMName, args&: Flags, args&: ImmChecks,
1254 args&: TS, args: ClassS, args&: *this, args&: SVEGuard, args&: SMEGuard));
1255
1256 // Also generate the short-form (e.g. svadd_m) for the given type-spec.
1257 if (Intrinsic::isOverloadedIntrinsic(Name))
1258 Out.push_back(Elt: std::make_unique<Intrinsic>(
1259 args&: Name, args&: Proto, args&: Merge, args&: MergeSuffix, args&: MemEltType, args&: LLVMName, args&: Flags,
1260 args&: ImmChecks, args&: TS, args: ClassG, args&: *this, args&: SVEGuard, args&: SMEGuard));
1261 }
1262}
1263
1264void SVEEmitter::createCoreHeaderIntrinsics(raw_ostream &OS,
1265 SVEEmitter &Emitter,
1266 ACLEKind Kind) {
1267 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1268 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1269 for (auto *R : RV)
1270 createIntrinsic(R, Out&: Defs);
1271
1272 // Sort intrinsics in header file by following order/priority:
1273 // - Architectural guard (i.e. does it require SVE2 or SVE2_AES)
1274 // - Class (is intrinsic overloaded or not)
1275 // - Intrinsic name
1276 llvm::stable_sort(Range&: Defs, C: [](const std::unique_ptr<Intrinsic> &A,
1277 const std::unique_ptr<Intrinsic> &B) {
1278 auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) {
1279 return std::make_tuple(args: I->getSVEGuard().str() + I->getSMEGuard().str(),
1280 args: (unsigned)I->getClassKind(), args: I->getName());
1281 };
1282 return ToTuple(A) < ToTuple(B);
1283 });
1284
1285 // Actually emit the intrinsic declarations.
1286 for (auto &I : Defs)
1287 I->emitIntrinsic(OS, Emitter, Kind);
1288}
1289
1290void SVEEmitter::createHeader(raw_ostream &OS) {
1291 OS << "/*===---- arm_sve.h - ARM SVE intrinsics "
1292 "-----------------------------------===\n"
1293 " *\n"
1294 " *\n"
1295 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1296 "Exceptions.\n"
1297 " * See https://llvm.org/LICENSE.txt for license information.\n"
1298 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1299 " *\n"
1300 " *===-----------------------------------------------------------------"
1301 "------===\n"
1302 " */\n\n";
1303
1304 OS << "#ifndef __ARM_SVE_H\n";
1305 OS << "#define __ARM_SVE_H\n\n";
1306
1307 OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1308 OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n";
1309 OS << "#endif\n";
1310
1311 OS << "#include <stdint.h>\n\n";
1312 OS << "#ifdef __cplusplus\n";
1313 OS << "extern \"C\" {\n";
1314 OS << "#else\n";
1315 OS << "#include <stdbool.h>\n";
1316 OS << "#endif\n\n";
1317
1318 OS << "typedef __fp16 float16_t;\n";
1319 OS << "typedef float float32_t;\n";
1320 OS << "typedef double float64_t;\n";
1321
1322 OS << "typedef __SVInt8_t svint8_t;\n";
1323 OS << "typedef __SVInt16_t svint16_t;\n";
1324 OS << "typedef __SVInt32_t svint32_t;\n";
1325 OS << "typedef __SVInt64_t svint64_t;\n";
1326 OS << "typedef __SVUint8_t svuint8_t;\n";
1327 OS << "typedef __SVUint16_t svuint16_t;\n";
1328 OS << "typedef __SVUint32_t svuint32_t;\n";
1329 OS << "typedef __SVUint64_t svuint64_t;\n";
1330 OS << "typedef __SVFloat16_t svfloat16_t;\n\n";
1331
1332 OS << "typedef __SVBfloat16_t svbfloat16_t;\n";
1333
1334 OS << "#include <arm_bf16.h>\n";
1335 OS << "#include <arm_vector_types.h>\n";
1336
1337 OS << "typedef __SVMfloat8_t svmfloat8_t;\n\n";
1338
1339 OS << "typedef __SVFloat32_t svfloat32_t;\n";
1340 OS << "typedef __SVFloat64_t svfloat64_t;\n";
1341 OS << "typedef __clang_svint8x2_t svint8x2_t;\n";
1342 OS << "typedef __clang_svint16x2_t svint16x2_t;\n";
1343 OS << "typedef __clang_svint32x2_t svint32x2_t;\n";
1344 OS << "typedef __clang_svint64x2_t svint64x2_t;\n";
1345 OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n";
1346 OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n";
1347 OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n";
1348 OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n";
1349 OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n";
1350 OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n";
1351 OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n";
1352 OS << "typedef __clang_svint8x3_t svint8x3_t;\n";
1353 OS << "typedef __clang_svint16x3_t svint16x3_t;\n";
1354 OS << "typedef __clang_svint32x3_t svint32x3_t;\n";
1355 OS << "typedef __clang_svint64x3_t svint64x3_t;\n";
1356 OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n";
1357 OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n";
1358 OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n";
1359 OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n";
1360 OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n";
1361 OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n";
1362 OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n";
1363 OS << "typedef __clang_svint8x4_t svint8x4_t;\n";
1364 OS << "typedef __clang_svint16x4_t svint16x4_t;\n";
1365 OS << "typedef __clang_svint32x4_t svint32x4_t;\n";
1366 OS << "typedef __clang_svint64x4_t svint64x4_t;\n";
1367 OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n";
1368 OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n";
1369 OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n";
1370 OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n";
1371 OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n";
1372 OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n";
1373 OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n";
1374 OS << "typedef __SVBool_t svbool_t;\n";
1375 OS << "typedef __clang_svboolx2_t svboolx2_t;\n";
1376 OS << "typedef __clang_svboolx4_t svboolx4_t;\n\n";
1377
1378 OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";
1379 OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";
1380 OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";
1381
1382 OS << "typedef __clang_svmfloat8x2_t svmfloat8x2_t;\n";
1383 OS << "typedef __clang_svmfloat8x3_t svmfloat8x3_t;\n";
1384 OS << "typedef __clang_svmfloat8x4_t svmfloat8x4_t;\n";
1385
1386 OS << "typedef __SVCount_t svcount_t;\n\n";
1387
1388 OS << "enum svpattern\n";
1389 OS << "{\n";
1390 OS << " SV_POW2 = 0,\n";
1391 OS << " SV_VL1 = 1,\n";
1392 OS << " SV_VL2 = 2,\n";
1393 OS << " SV_VL3 = 3,\n";
1394 OS << " SV_VL4 = 4,\n";
1395 OS << " SV_VL5 = 5,\n";
1396 OS << " SV_VL6 = 6,\n";
1397 OS << " SV_VL7 = 7,\n";
1398 OS << " SV_VL8 = 8,\n";
1399 OS << " SV_VL16 = 9,\n";
1400 OS << " SV_VL32 = 10,\n";
1401 OS << " SV_VL64 = 11,\n";
1402 OS << " SV_VL128 = 12,\n";
1403 OS << " SV_VL256 = 13,\n";
1404 OS << " SV_MUL4 = 29,\n";
1405 OS << " SV_MUL3 = 30,\n";
1406 OS << " SV_ALL = 31\n";
1407 OS << "};\n\n";
1408
1409 OS << "enum svprfop\n";
1410 OS << "{\n";
1411 OS << " SV_PLDL1KEEP = 0,\n";
1412 OS << " SV_PLDL1STRM = 1,\n";
1413 OS << " SV_PLDL2KEEP = 2,\n";
1414 OS << " SV_PLDL2STRM = 3,\n";
1415 OS << " SV_PLDL3KEEP = 4,\n";
1416 OS << " SV_PLDL3STRM = 5,\n";
1417 OS << " SV_PSTL1KEEP = 8,\n";
1418 OS << " SV_PSTL1STRM = 9,\n";
1419 OS << " SV_PSTL2KEEP = 10,\n";
1420 OS << " SV_PSTL2STRM = 11,\n";
1421 OS << " SV_PSTL3KEEP = 12,\n";
1422 OS << " SV_PSTL3STRM = 13\n";
1423 OS << "};\n\n";
1424
1425 OS << "/* Function attributes */\n";
1426 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1427 "__nodebug__))\n\n";
1428 OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1429 "__nodebug__, __overloadable__))\n\n";
1430
1431 // Add reinterpret functions.
1432 for (auto [N, Suffix] :
1433 std::initializer_list<std::pair<unsigned, const char *>>{
1434 {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1435 for (auto ShortForm : {false, true})
1436 for (const ReinterpretTypeInfo &To : Reinterprets) {
1437 SVEType ToV(To.BaseType, N);
1438 for (const ReinterpretTypeInfo &From : Reinterprets) {
1439 SVEType FromV(From.BaseType, N);
1440 OS << "__aio "
1441 "__attribute__((__clang_arm_builtin_alias(__builtin_sve_"
1442 "reinterpret_"
1443 << To.Suffix << "_" << From.Suffix << Suffix << ")))\n"
1444 << ToV.str() << " svreinterpret_" << To.Suffix;
1445 if (!ShortForm)
1446 OS << "_" << From.Suffix << Suffix;
1447 OS << "(" << FromV.str() << " op);\n";
1448 }
1449 }
1450 }
1451
1452 createCoreHeaderIntrinsics(OS, Emitter&: *this, Kind: ACLEKind::SVE);
1453
1454 OS << "#define svcvtnt_bf16_x svcvtnt_bf16_m\n";
1455 OS << "#define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m\n";
1456
1457 OS << "#define svcvtnt_f16_x svcvtnt_f16_m\n";
1458 OS << "#define svcvtnt_f16_f32_x svcvtnt_f16_f32_m\n";
1459 OS << "#define svcvtnt_f32_x svcvtnt_f32_m\n";
1460 OS << "#define svcvtnt_f32_f64_x svcvtnt_f32_f64_m\n\n";
1461
1462 OS << "#define svcvtxnt_f32_x svcvtxnt_f32_m\n";
1463 OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n";
1464
1465 OS << "#ifdef __cplusplus\n";
1466 OS << "} // extern \"C\"\n";
1467 OS << "#endif\n\n";
1468 OS << "#undef __ai\n\n";
1469 OS << "#undef __aio\n\n";
1470 OS << "#endif /* __ARM_SVE_H */\n";
1471}
1472
1473void SVEEmitter::createBuiltins(raw_ostream &OS) {
1474 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1475 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1476 for (auto *R : RV)
1477 createIntrinsic(R, Out&: Defs);
1478
1479 // The mappings must be sorted based on BuiltinID.
1480 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1481 const std::unique_ptr<Intrinsic> &B) {
1482 return A->getMangledName() < B->getMangledName();
1483 });
1484
1485 llvm::StringToOffsetTable Table;
1486 Table.GetOrAddStringOffset(Str: "");
1487 Table.GetOrAddStringOffset(Str: "n");
1488
1489 for (const auto &Def : Defs)
1490 if (Def->getClassKind() != ClassG) {
1491 Table.GetOrAddStringOffset(Str: Def->getMangledName());
1492 Table.GetOrAddStringOffset(Str: Def->getBuiltinTypeStr());
1493 Table.GetOrAddStringOffset(Str: Def->getGuard());
1494 }
1495
1496 Table.GetOrAddStringOffset(Str: "sme|sve");
1497 SmallVector<std::pair<std::string, std::string>> ReinterpretBuiltins;
1498 for (auto [N, Suffix] :
1499 std::initializer_list<std::pair<unsigned, const char *>>{
1500 {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1501 for (const ReinterpretTypeInfo &To : Reinterprets) {
1502 SVEType ToV(To.BaseType, N);
1503 for (const ReinterpretTypeInfo &From : Reinterprets) {
1504 SVEType FromV(From.BaseType, N);
1505 std::string Name =
1506 (Twine("reinterpret_") + To.Suffix + "_" + From.Suffix + Suffix)
1507 .str();
1508 std::string Type = ToV.builtin_str() + FromV.builtin_str();
1509 Table.GetOrAddStringOffset(Str: Name);
1510 Table.GetOrAddStringOffset(Str: Type);
1511 ReinterpretBuiltins.push_back(Elt: {Name, Type});
1512 }
1513 }
1514 }
1515
1516 OS << "#ifdef GET_SVE_BUILTIN_ENUMERATORS\n";
1517 for (const auto &Def : Defs)
1518 if (Def->getClassKind() != ClassG)
1519 OS << " BI__builtin_sve_" << Def->getMangledName() << ",\n";
1520 for (const auto &[Name, _] : ReinterpretBuiltins)
1521 OS << " BI__builtin_sve_" << Name << ",\n";
1522 OS << "#endif // GET_SVE_BUILTIN_ENUMERATORS\n\n";
1523
1524 OS << "#ifdef GET_SVE_BUILTIN_STR_TABLE\n";
1525 Table.EmitStringTableDef(OS, Name: "BuiltinStrings");
1526 OS << "#endif // GET_SVE_BUILTIN_STR_TABLE\n\n";
1527
1528 OS << "#ifdef GET_SVE_BUILTIN_INFOS\n";
1529 for (const auto &Def : Defs) {
1530 // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1531 // declarations only live in the header file.
1532 if (Def->getClassKind() != ClassG) {
1533 OS << " Builtin::Info{Builtin::Info::StrOffsets{"
1534 << Table.GetStringOffset(Str: Def->getMangledName()) << " /* "
1535 << Def->getMangledName() << " */, ";
1536 OS << Table.GetStringOffset(Str: Def->getBuiltinTypeStr()) << " /* "
1537 << Def->getBuiltinTypeStr() << " */, ";
1538 OS << Table.GetStringOffset(Str: "n") << " /* n */, ";
1539 OS << Table.GetStringOffset(Str: Def->getGuard()) << " /* " << Def->getGuard()
1540 << " */}, ";
1541 OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
1542 }
1543 }
1544 for (const auto &[Name, Type] : ReinterpretBuiltins) {
1545 OS << " Builtin::Info{Builtin::Info::StrOffsets{"
1546 << Table.GetStringOffset(Str: Name) << " /* " << Name << " */, ";
1547 OS << Table.GetStringOffset(Str: Type) << " /* " << Type << " */, ";
1548 OS << Table.GetStringOffset(Str: "n") << " /* n */, ";
1549 OS << Table.GetStringOffset(Str: "sme|sve") << " /* sme|sve */}, ";
1550 OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
1551 }
1552 OS << "#endif // GET_SVE_BUILTIN_INFOS\n\n";
1553}
1554
1555void SVEEmitter::createCodeGenMap(raw_ostream &OS) {
1556 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1557 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1558 for (auto *R : RV)
1559 createIntrinsic(R, Out&: Defs);
1560
1561 // The mappings must be sorted based on BuiltinID.
1562 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1563 const std::unique_ptr<Intrinsic> &B) {
1564 return A->getMangledName() < B->getMangledName();
1565 });
1566
1567 OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n";
1568 for (auto &Def : Defs) {
1569 // Builtins only exist for non-overloaded intrinsics, overloaded
1570 // declarations only live in the header file.
1571 if (Def->getClassKind() == ClassG)
1572 continue;
1573
1574 uint64_t Flags = Def->getFlags();
1575 auto FlagString = std::to_string(val: Flags);
1576
1577 std::string LLVMName = Def->getMangledLLVMName();
1578 std::string Builtin = Def->getMangledName();
1579 if (!LLVMName.empty())
1580 OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1581 << "),\n";
1582 else
1583 OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n";
1584 }
1585 OS << "#endif\n\n";
1586}
1587
1588void SVEEmitter::createRangeChecks(raw_ostream &OS) {
1589 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1590 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1591 for (auto *R : RV)
1592 createIntrinsic(R, Out&: Defs);
1593
1594 // The mappings must be sorted based on BuiltinID.
1595 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1596 const std::unique_ptr<Intrinsic> &B) {
1597 return A->getMangledName() < B->getMangledName();
1598 });
1599
1600 OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n";
1601
1602 // Ensure these are only emitted once.
1603 std::set<std::string> Emitted;
1604
1605 for (auto &Def : Defs) {
1606 if (Emitted.find(x: Def->getMangledName()) != Emitted.end() ||
1607 Def->getImmChecks().empty())
1608 continue;
1609
1610 OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n";
1611 for (auto &Check : Def->getImmChecks())
1612 OS << "ImmChecks.emplace_back(" << Check.getImmArgIdx() << ", "
1613 << Check.getKind() << ", " << Check.getElementSizeInBits() << ");\n";
1614 OS << " break;\n";
1615
1616 Emitted.insert(x: Def->getMangledName());
1617 }
1618
1619 OS << "#endif\n\n";
1620}
1621
1622/// Create the SVETypeFlags used in CGBuiltins
1623void SVEEmitter::createTypeFlags(raw_ostream &OS) {
1624 OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n";
1625 for (auto &KV : FlagTypes)
1626 OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n";
1627 OS << "#endif\n\n";
1628
1629 OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n";
1630 for (auto &KV : EltTypes)
1631 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1632 OS << "#endif\n\n";
1633
1634 OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n";
1635 for (auto &KV : MemEltTypes)
1636 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1637 OS << "#endif\n\n";
1638
1639 OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n";
1640 for (auto &KV : MergeTypes)
1641 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1642 OS << "#endif\n\n";
1643}
1644
1645void SVEEmitter::createImmCheckTypes(raw_ostream &OS) {
1646 OS << "#ifdef LLVM_GET_ARM_INTRIN_IMMCHECKTYPES\n";
1647 for (auto &KV : ImmCheckTypes)
1648 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1649 OS << "#endif\n\n";
1650}
1651
1652void SVEEmitter::createSMEHeader(raw_ostream &OS) {
1653 OS << "/*===---- arm_sme.h - ARM SME intrinsics "
1654 "------===\n"
1655 " *\n"
1656 " *\n"
1657 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1658 "Exceptions.\n"
1659 " * See https://llvm.org/LICENSE.txt for license information.\n"
1660 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1661 " *\n"
1662 " *===-----------------------------------------------------------------"
1663 "------===\n"
1664 " */\n\n";
1665
1666 OS << "#ifndef __ARM_SME_H\n";
1667 OS << "#define __ARM_SME_H\n\n";
1668
1669 OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1670 OS << "#error \"Big endian is currently not supported for arm_sme.h\"\n";
1671 OS << "#endif\n";
1672
1673 OS << "#include <arm_sve.h>\n\n";
1674 OS << "#include <stddef.h>\n\n";
1675
1676 OS << "/* Function attributes */\n";
1677 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1678 "__nodebug__))\n\n";
1679 OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1680 "__nodebug__, __overloadable__))\n\n";
1681
1682 OS << "#ifdef __cplusplus\n";
1683 OS << "extern \"C\" {\n";
1684 OS << "#endif\n\n";
1685
1686 OS << "void __arm_za_disable(void) __arm_streaming_compatible;\n\n";
1687
1688 OS << "__ai bool __arm_has_sme(void) __arm_streaming_compatible {\n";
1689 OS << " uint64_t x0, x1;\n";
1690 OS << " __builtin_arm_get_sme_state(&x0, &x1);\n";
1691 OS << " return x0 & (1ULL << 63);\n";
1692 OS << "}\n\n";
1693
1694 OS << "void *__arm_sc_memcpy(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
1695 OS << "void *__arm_sc_memmove(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
1696 OS << "void *__arm_sc_memset(void *s, int c, size_t n) __arm_streaming_compatible;\n";
1697 OS << "void *__arm_sc_memchr(void *s, int c, size_t n) __arm_streaming_compatible;\n\n";
1698
1699 OS << "__ai __attribute__((target(\"sme\"))) void svundef_za(void) "
1700 "__arm_streaming_compatible __arm_out(\"za\") "
1701 "{ }\n\n";
1702
1703 createCoreHeaderIntrinsics(OS, Emitter&: *this, Kind: ACLEKind::SME);
1704
1705 OS << "#ifdef __cplusplus\n";
1706 OS << "} // extern \"C\"\n";
1707 OS << "#endif\n\n";
1708 OS << "#undef __ai\n\n";
1709 OS << "#endif /* __ARM_SME_H */\n";
1710}
1711
1712void SVEEmitter::createSMEBuiltins(raw_ostream &OS) {
1713 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1714 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1715 for (auto *R : RV) {
1716 createIntrinsic(R, Out&: Defs);
1717 }
1718
1719 // The mappings must be sorted based on BuiltinID.
1720 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1721 const std::unique_ptr<Intrinsic> &B) {
1722 return A->getMangledName() < B->getMangledName();
1723 });
1724
1725 llvm::StringToOffsetTable Table;
1726 Table.GetOrAddStringOffset(Str: "");
1727 Table.GetOrAddStringOffset(Str: "n");
1728
1729 for (const auto &Def : Defs)
1730 if (Def->getClassKind() != ClassG) {
1731 Table.GetOrAddStringOffset(Str: Def->getMangledName());
1732 Table.GetOrAddStringOffset(Str: Def->getBuiltinTypeStr());
1733 Table.GetOrAddStringOffset(Str: Def->getGuard());
1734 }
1735
1736 OS << "#ifdef GET_SME_BUILTIN_ENUMERATORS\n";
1737 for (const auto &Def : Defs)
1738 if (Def->getClassKind() != ClassG)
1739 OS << " BI__builtin_sme_" << Def->getMangledName() << ",\n";
1740 OS << "#endif // GET_SME_BUILTIN_ENUMERATORS\n\n";
1741
1742 OS << "#ifdef GET_SME_BUILTIN_STR_TABLE\n";
1743 Table.EmitStringTableDef(OS, Name: "BuiltinStrings");
1744 OS << "#endif // GET_SME_BUILTIN_STR_TABLE\n\n";
1745
1746 OS << "#ifdef GET_SME_BUILTIN_INFOS\n";
1747 for (const auto &Def : Defs) {
1748 // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1749 // declarations only live in the header file.
1750 if (Def->getClassKind() != ClassG) {
1751 OS << " Builtin::Info{Builtin::Info::StrOffsets{"
1752 << Table.GetStringOffset(Str: Def->getMangledName()) << " /* "
1753 << Def->getMangledName() << " */, ";
1754 OS << Table.GetStringOffset(Str: Def->getBuiltinTypeStr()) << " /* "
1755 << Def->getBuiltinTypeStr() << " */, ";
1756 OS << Table.GetStringOffset(Str: "n") << " /* n */, ";
1757 OS << Table.GetStringOffset(Str: Def->getGuard()) << " /* " << Def->getGuard()
1758 << " */}, ";
1759 OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
1760 }
1761 }
1762 OS << "#endif // GET_SME_BUILTIN_INFOS\n\n";
1763}
1764
1765void SVEEmitter::createSMECodeGenMap(raw_ostream &OS) {
1766 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1767 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1768 for (auto *R : RV) {
1769 createIntrinsic(R, Out&: Defs);
1770 }
1771
1772 // The mappings must be sorted based on BuiltinID.
1773 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1774 const std::unique_ptr<Intrinsic> &B) {
1775 return A->getMangledName() < B->getMangledName();
1776 });
1777
1778 OS << "#ifdef GET_SME_LLVM_INTRINSIC_MAP\n";
1779 for (auto &Def : Defs) {
1780 // Builtins only exist for non-overloaded intrinsics, overloaded
1781 // declarations only live in the header file.
1782 if (Def->getClassKind() == ClassG)
1783 continue;
1784
1785 uint64_t Flags = Def->getFlags();
1786 auto FlagString = std::to_string(val: Flags);
1787
1788 std::string LLVMName = Def->getLLVMName();
1789 std::string Builtin = Def->getMangledName();
1790 if (!LLVMName.empty())
1791 OS << "SMEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1792 << "),\n";
1793 else
1794 OS << "SMEMAP2(" << Builtin << ", " << FlagString << "),\n";
1795 }
1796 OS << "#endif\n\n";
1797}
1798
1799void SVEEmitter::createSMERangeChecks(raw_ostream &OS) {
1800 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1801 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1802 for (auto *R : RV) {
1803 createIntrinsic(R, Out&: Defs);
1804 }
1805
1806 // The mappings must be sorted based on BuiltinID.
1807 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1808 const std::unique_ptr<Intrinsic> &B) {
1809 return A->getMangledName() < B->getMangledName();
1810 });
1811
1812 OS << "#ifdef GET_SME_IMMEDIATE_CHECK\n";
1813
1814 // Ensure these are only emitted once.
1815 std::set<std::string> Emitted;
1816
1817 for (auto &Def : Defs) {
1818 if (Emitted.find(x: Def->getMangledName()) != Emitted.end() ||
1819 Def->getImmChecks().empty())
1820 continue;
1821
1822 OS << "case SME::BI__builtin_sme_" << Def->getMangledName() << ":\n";
1823 for (auto &Check : Def->getImmChecks())
1824 OS << "ImmChecks.push_back(std::make_tuple(" << Check.getImmArgIdx()
1825 << ", " << Check.getKind() << ", " << Check.getElementSizeInBits()
1826 << "));\n";
1827 OS << " break;\n";
1828
1829 Emitted.insert(x: Def->getMangledName());
1830 }
1831
1832 OS << "#endif\n\n";
1833}
1834
1835void SVEEmitter::createBuiltinZAState(raw_ostream &OS) {
1836 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1837 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1838 for (auto *R : RV)
1839 createIntrinsic(R, Out&: Defs);
1840
1841 std::map<std::string, std::set<std::string>> IntrinsicsPerState;
1842 for (auto &Def : Defs) {
1843 std::string Key;
1844 auto AddToKey = [&Key](const std::string &S) -> void {
1845 Key = Key.empty() ? S : (Key + " | " + S);
1846 };
1847
1848 if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInZA")))
1849 AddToKey("ArmInZA");
1850 else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsOutZA")))
1851 AddToKey("ArmOutZA");
1852 else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInOutZA")))
1853 AddToKey("ArmInOutZA");
1854
1855 if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInZT0")))
1856 AddToKey("ArmInZT0");
1857 else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsOutZT0")))
1858 AddToKey("ArmOutZT0");
1859 else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInOutZT0")))
1860 AddToKey("ArmInOutZT0");
1861
1862 if (!Key.empty())
1863 IntrinsicsPerState[Key].insert(x: Def->getMangledName());
1864 }
1865
1866 OS << "#ifdef GET_SME_BUILTIN_GET_STATE\n";
1867 for (auto &KV : IntrinsicsPerState) {
1868 for (StringRef Name : KV.second)
1869 OS << "case SME::BI__builtin_sme_" << Name << ":\n";
1870 OS << " return " << KV.first << ";\n";
1871 }
1872 OS << "#endif\n\n";
1873}
1874
1875void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) {
1876 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1877 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1878 for (auto *R : RV)
1879 createIntrinsic(R, Out&: Defs);
1880
1881 StringRef ExtensionKind;
1882 switch (Kind) {
1883 case ACLEKind::SME:
1884 ExtensionKind = "SME";
1885 break;
1886 case ACLEKind::SVE:
1887 ExtensionKind = "SVE";
1888 break;
1889 }
1890
1891 OS << "#ifdef GET_" << ExtensionKind << "_STREAMING_ATTRS\n";
1892
1893 StringMap<std::set<std::string>> StreamingMap;
1894
1895 uint64_t IsStreamingFlag = getEnumValueForFlag(C: "IsStreaming");
1896 uint64_t VerifyRuntimeMode = getEnumValueForFlag(C: "VerifyRuntimeMode");
1897 uint64_t IsStreamingCompatibleFlag =
1898 getEnumValueForFlag(C: "IsStreamingCompatible");
1899
1900 for (auto &Def : Defs) {
1901 if (!Def->isFlagSet(Flag: VerifyRuntimeMode) && !Def->getSVEGuard().empty() &&
1902 !Def->getSMEGuard().empty())
1903 report_fatal_error(reason: "Missing VerifyRuntimeMode flag");
1904
1905 if (Def->isFlagSet(Flag: IsStreamingFlag))
1906 StreamingMap["ArmStreaming"].insert(x: Def->getMangledName());
1907 else if (Def->isFlagSet(Flag: VerifyRuntimeMode))
1908 StreamingMap["VerifyRuntimeMode"].insert(x: Def->getMangledName());
1909 else if (Def->isFlagSet(Flag: IsStreamingCompatibleFlag))
1910 StreamingMap["ArmStreamingCompatible"].insert(x: Def->getMangledName());
1911 else
1912 StreamingMap["ArmNonStreaming"].insert(x: Def->getMangledName());
1913 }
1914
1915 for (auto BuiltinType : StreamingMap.keys()) {
1916 for (auto Name : StreamingMap[BuiltinType]) {
1917 OS << "case " << ExtensionKind << "::BI__builtin_"
1918 << ExtensionKind.lower() << "_";
1919 OS << Name << ":\n";
1920 }
1921 OS << " BuiltinType = " << BuiltinType << ";\n";
1922 OS << " break;\n";
1923 }
1924
1925 OS << "#endif\n\n";
1926}
1927
1928namespace clang {
1929void EmitSveHeader(const RecordKeeper &Records, raw_ostream &OS) {
1930 SVEEmitter(Records).createHeader(OS);
1931}
1932
1933void EmitSveBuiltins(const RecordKeeper &Records, raw_ostream &OS) {
1934 SVEEmitter(Records).createBuiltins(OS);
1935}
1936
1937void EmitSveBuiltinCG(const RecordKeeper &Records, raw_ostream &OS) {
1938 SVEEmitter(Records).createCodeGenMap(OS);
1939}
1940
1941void EmitSveRangeChecks(const RecordKeeper &Records, raw_ostream &OS) {
1942 SVEEmitter(Records).createRangeChecks(OS);
1943}
1944
1945void EmitSveTypeFlags(const RecordKeeper &Records, raw_ostream &OS) {
1946 SVEEmitter(Records).createTypeFlags(OS);
1947}
1948
1949void EmitImmCheckTypes(const RecordKeeper &Records, raw_ostream &OS) {
1950 SVEEmitter(Records).createImmCheckTypes(OS);
1951}
1952
1953void EmitSveStreamingAttrs(const RecordKeeper &Records, raw_ostream &OS) {
1954 SVEEmitter(Records).createStreamingAttrs(OS, Kind: ACLEKind::SVE);
1955}
1956
1957void EmitSmeHeader(const RecordKeeper &Records, raw_ostream &OS) {
1958 SVEEmitter(Records).createSMEHeader(OS);
1959}
1960
1961void EmitSmeBuiltins(const RecordKeeper &Records, raw_ostream &OS) {
1962 SVEEmitter(Records).createSMEBuiltins(OS);
1963}
1964
1965void EmitSmeBuiltinCG(const RecordKeeper &Records, raw_ostream &OS) {
1966 SVEEmitter(Records).createSMECodeGenMap(OS);
1967}
1968
1969void EmitSmeRangeChecks(const RecordKeeper &Records, raw_ostream &OS) {
1970 SVEEmitter(Records).createSMERangeChecks(OS);
1971}
1972
1973void EmitSmeStreamingAttrs(const RecordKeeper &Records, raw_ostream &OS) {
1974 SVEEmitter(Records).createStreamingAttrs(OS, Kind: ACLEKind::SME);
1975}
1976
1977void EmitSmeBuiltinZAState(const RecordKeeper &Records, raw_ostream &OS) {
1978 SVEEmitter(Records).createBuiltinZAState(OS);
1979}
1980} // End namespace clang
1981