1//===-- SveEmitter.cpp - Generate arm_sve.h for use with clang ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This tablegen backend is responsible for emitting arm_sve.h, which includes
10// a declaration and definition of each function specified by the ARM C/C++
11// Language Extensions (ACLE).
12//
13// For details, visit:
14// https://developer.arm.com/architectures/system-architectures/software-standards/acle
15//
16// Each SVE instruction is implemented in terms of 1 or more functions which
17// are suffixed with the element type of the input vectors. Functions may be
18// implemented in terms of generic vector operations such as +, *, -, etc. or
19// by calling a __builtin_-prefixed function which will be handled by clang's
20// CodeGen library.
21//
22// See also the documentation in include/clang/Basic/arm_sve.td.
23//
24//===----------------------------------------------------------------------===//
25
26#include "llvm/ADT/ArrayRef.h"
27#include "llvm/ADT/STLExtras.h"
28#include "llvm/ADT/StringExtras.h"
29#include "llvm/ADT/StringMap.h"
30#include "llvm/Support/raw_ostream.h"
31#include "llvm/TableGen/AArch64ImmCheck.h"
32#include "llvm/TableGen/Error.h"
33#include "llvm/TableGen/Record.h"
34#include "llvm/TableGen/StringToOffsetTable.h"
35#include <array>
36#include <cctype>
37#include <set>
38#include <string>
39#include <tuple>
40
41using namespace llvm;
42
43enum ClassKind {
44 ClassNone,
45 ClassS, // signed/unsigned, e.g., "_s8", "_u8" suffix
46 ClassG, // Overloaded name without type suffix
47};
48
49enum class ACLEKind { SVE, SME };
50
51using TypeSpec = std::string;
52
53namespace {
54class SVEType {
55
56 enum TypeKind {
57 Invalid,
58 Void,
59 Float,
60 SInt,
61 UInt,
62 BFloat16,
63 MFloat8,
64 Svcount,
65 PrefetchOp,
66 PredicatePattern,
67 Predicate,
68 Fpm
69 };
70 TypeKind Kind;
71 bool Immediate, Constant, Pointer, DefaultType, IsScalable;
72 unsigned Bitwidth, ElementBitwidth, NumVectors;
73
74public:
75 SVEType() : SVEType("", 'v') {}
76
77 SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1)
78 : Kind(Invalid), Immediate(false), Constant(false), Pointer(false),
79 DefaultType(false), IsScalable(true), Bitwidth(128),
80 ElementBitwidth(~0U), NumVectors(NumVectors) {
81 if (!TS.empty())
82 applyTypespec(TS);
83 applyModifier(Mod: CharMod);
84 }
85
86 SVEType(const SVEType &Base, unsigned NumV) : SVEType(Base) {
87 NumVectors = NumV;
88 }
89
90 bool isPointer() const { return Pointer; }
91 bool isConstant() const { return Constant; }
92 bool isImmediate() const { return Immediate; }
93 bool isScalar() const { return NumVectors == 0; }
94 bool isVector() const { return NumVectors > 0; }
95 bool isScalableVector() const { return isVector() && IsScalable; }
96 bool isFixedLengthVector() const { return isVector() && !IsScalable; }
97 bool isChar() const { return ElementBitwidth == 8 && isInteger(); }
98 bool isVoid() const { return Kind == Void; }
99 bool isDefault() const { return DefaultType; }
100 bool isFloat() const { return Kind == Float; }
101 bool isBFloat() const { return Kind == BFloat16; }
102 bool isMFloat() const { return Kind == MFloat8; }
103 bool isFloatingPoint() const {
104 return Kind == Float || Kind == BFloat16 || Kind == MFloat8;
105 }
106 bool isInteger() const { return Kind == SInt || Kind == UInt; }
107 bool isSignedInteger() const { return Kind == SInt; }
108 bool isUnsignedInteger() const { return Kind == UInt; }
109 bool isScalarPredicate() const {
110 return Kind == Predicate && NumVectors == 0;
111 }
112 bool isPredicate() const { return Kind == Predicate; }
113 bool isPredicatePattern() const { return Kind == PredicatePattern; }
114 bool isPrefetchOp() const { return Kind == PrefetchOp; }
115 bool isSvcount() const { return Kind == Svcount; }
116 bool isFpm() const { return Kind == Fpm; }
117 bool isInvalid() const { return Kind == Invalid; }
118 unsigned getElementSizeInBits() const { return ElementBitwidth; }
119 unsigned getNumVectors() const { return NumVectors; }
120
121 unsigned getNumElements() const {
122 assert(ElementBitwidth != ~0U);
123 return isPredicate() ? 16 : (Bitwidth / ElementBitwidth);
124 }
125 unsigned getSizeInBits() const {
126 return Bitwidth;
127 }
128
129 /// Return the string representation of a type, which is an encoded
130 /// string for passing to the BUILTIN() macro in Builtins.def.
131 std::string builtin_str() const;
132
133 /// Return the C/C++ string representation of a type for use in the
134 /// arm_sve.h header file.
135 std::string str() const;
136
137private:
138 /// Creates the type based on the typespec string in TS.
139 void applyTypespec(StringRef TS);
140
141 /// Applies a prototype modifier to the type.
142 void applyModifier(char Mod);
143
144 /// Get the builtin base for this SVEType, e.g. 'Wi' for svint64_t.
145 std::string builtinBaseType() const;
146};
147
148class SVEEmitter;
149
150/// The main grunt class. This represents an instantiation of an intrinsic with
151/// a particular typespec and prototype.
152class Intrinsic {
153 /// The unmangled name.
154 std::string Name;
155
156 /// The name of the corresponding LLVM IR intrinsic.
157 std::string LLVMName;
158
159 /// Intrinsic prototype.
160 std::string Proto;
161
162 /// The base type spec for this intrinsic.
163 TypeSpec BaseTypeSpec;
164
165 /// The base class kind. Most intrinsics use ClassS, which has full type
166 /// info for integers (_s32/_u32), or ClassG which is used for overloaded
167 /// intrinsics.
168 ClassKind Class;
169
170 /// The architectural #ifdef guard.
171 std::string SVEGuard, SMEGuard;
172
173 // The merge suffix such as _m, _x or _z.
174 std::string MergeSuffix;
175
176 /// The types of return value [0] and parameters [1..].
177 std::vector<SVEType> Types;
178
179 /// The "base type", which is VarType('d', BaseTypeSpec).
180 SVEType BaseType;
181
182 uint64_t Flags;
183
184 SmallVector<ImmCheck, 2> ImmChecks;
185
186 bool SetsFPMR;
187
188public:
189 Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
190 StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName,
191 uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT,
192 ClassKind Class, SVEEmitter &Emitter, StringRef SVEGuard,
193 StringRef SMEGuard);
194
195 ~Intrinsic()=default;
196
197 std::string getName() const { return Name; }
198 std::string getLLVMName() const { return LLVMName; }
199 std::string getProto() const { return Proto; }
200 TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; }
201 SVEType getBaseType() const { return BaseType; }
202
203 StringRef getSVEGuard() const { return SVEGuard; }
204 StringRef getSMEGuard() const { return SMEGuard; }
205 std::string getGuard() const {
206 std::string Guard;
207 llvm::raw_string_ostream OS(Guard);
208 if (!SVEGuard.empty() && SMEGuard.empty())
209 OS << SVEGuard;
210 else if (SVEGuard.empty() && !SMEGuard.empty())
211 OS << SMEGuard;
212 else {
213 if (SVEGuard.find(s: ",") != std::string::npos ||
214 SVEGuard.find(s: "|") != std::string::npos)
215 OS << "(" << SVEGuard << ")";
216 else
217 OS << SVEGuard;
218 OS << "|";
219 if (SMEGuard.find(s: ",") != std::string::npos ||
220 SMEGuard.find(s: "|") != std::string::npos)
221 OS << "(" << SMEGuard << ")";
222 else
223 OS << SMEGuard;
224 }
225 return Guard;
226 }
227 ClassKind getClassKind() const { return Class; }
228
229 SVEType getReturnType() const { return Types[0]; }
230 ArrayRef<SVEType> getTypes() const { return Types; }
231 SVEType getParamType(unsigned I) const { return Types[I + 1]; }
232 unsigned getNumParams() const {
233 return Proto.size() - (2 * count(Range: Proto, Element: '.')) - 1;
234 }
235
236 uint64_t getFlags() const { return Flags; }
237 bool isFlagSet(uint64_t Flag) const { return Flags & Flag;}
238
239 ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; }
240
241 /// Return the type string for a BUILTIN() macro in Builtins.def.
242 std::string getBuiltinTypeStr();
243
244 /// Return the name, mangled with type information. The name is mangled for
245 /// ClassS, so will add type suffixes such as _u32/_s32.
246 std::string getMangledName(ClassKind CK = ClassS) const {
247 return mangleName(LocalCK: CK);
248 }
249
250 /// As above, but mangles the LLVM name instead.
251 std::string getMangledLLVMName() const { return mangleLLVMName(); }
252
253 /// Returns true if the intrinsic is overloaded, in that it should also generate
254 /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of
255 /// 'svld1_u32(..)'.
256 static bool isOverloadedIntrinsic(StringRef Name) {
257 return Name.contains(C: '[') && Name.contains(C: ']');
258 }
259
260 /// Return true if the intrinsic takes a splat operand.
261 bool hasSplat() const {
262 // These prototype modifiers are described in arm_sve.td.
263 return Proto.find_first_of(s: "ajfrKLR@!") != std::string::npos;
264 }
265
266 /// Return the parameter index of the splat operand.
267 unsigned getSplatIdx() const {
268 unsigned I = 1, Param = 0;
269 for (; I < Proto.size(); ++I, ++Param) {
270 assert(Proto[I] != '4' &&
271 "Handling for '4' prototype modifier not implemented");
272 if (Proto[I] == 'a' || Proto[I] == 'j' || Proto[I] == 'f' ||
273 Proto[I] == 'r' || Proto[I] == 'K' || Proto[I] == 'L' ||
274 Proto[I] == 'R' || Proto[I] == '@' || Proto[I] == '!')
275 break;
276
277 if (Proto[I] == '2')
278 Param += 1;
279
280 // Multivector modifier can be skipped
281 if (Proto[I] == '.') {
282 Param -= 1; // Adjust for the increment at the top of the loop
283 I += 2;
284 }
285 }
286 assert(I != Proto.size() && "Prototype has no splat operand");
287 return Param;
288 }
289
290 /// Emits the intrinsic declaration to the ostream.
291 void emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter, ACLEKind Kind) const;
292
293private:
294 std::string getMergeSuffix() const { return MergeSuffix; }
295 StringRef getFPMSuffix() const { return SetsFPMR ? "_fpm" : ""; }
296 std::string mangleName(ClassKind LocalCK) const;
297 std::string mangleLLVMName() const;
298 std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,
299 std::string Proto) const;
300};
301
302class SVEEmitter {
303private:
304 // The reinterpret builtins are generated separately because they
305 // need the cross product of all types (121 functions in total),
306 // which is inconvenient to specify in the arm_sve.td file or
307 // generate in CGBuiltin.cpp.
308 struct ReinterpretTypeInfo {
309 SVEType BaseType;
310 const char *Suffix;
311 };
312
313 static const std::array<ReinterpretTypeInfo, 13> Reinterprets;
314
315 const RecordKeeper &Records;
316 StringMap<uint64_t> EltTypes;
317 StringMap<uint64_t> MemEltTypes;
318 StringMap<uint64_t> FlagTypes;
319 StringMap<uint64_t> MergeTypes;
320 StringMap<uint64_t> ImmCheckTypes;
321 std::vector<llvm::StringRef> ImmCheckTypeNames;
322
323public:
324 SVEEmitter(const RecordKeeper &R) : Records(R) {
325 for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "EltType"))
326 EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
327 for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "MemEltType"))
328 MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
329 for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "FlagType"))
330 FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
331 for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "MergeType"))
332 MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt(FieldName: "Value");
333 for (auto *RV : Records.getAllDerivedDefinitions(ClassName: "ImmCheckType")) {
334 auto [it, inserted] = ImmCheckTypes.try_emplace(
335 Key: RV->getNameInitAsString(), Args: RV->getValueAsInt(FieldName: "Value"));
336 if (!inserted)
337 llvm_unreachable("Duplicate imm check");
338 if ((size_t)it->second >= ImmCheckTypeNames.size())
339 ImmCheckTypeNames.resize(new_size: (size_t)it->second + 1);
340 ImmCheckTypeNames[it->second] = it->first();
341 }
342 }
343
344 /// Returns the enum value for the immcheck type
345 unsigned getEnumValueForImmCheck(StringRef C) const {
346 auto It = ImmCheckTypes.find(Key: C);
347 if (It != ImmCheckTypes.end())
348 return It->getValue();
349 llvm_unreachable("Unsupported imm check");
350 }
351
352 /// Returns the enum value for the flag type
353 uint64_t getEnumValueForFlag(StringRef C) const {
354 auto Res = FlagTypes.find(Key: C);
355 if (Res != FlagTypes.end())
356 return Res->getValue();
357 llvm_unreachable("Unsupported flag");
358 }
359
360 /// Returns the name for the immcheck type
361 StringRef getImmCheckForEnumValue(unsigned Id) {
362 if ((size_t)Id < ImmCheckTypeNames.size())
363 return ImmCheckTypeNames[Id];
364 llvm_unreachable("Unsupported imm check");
365 }
366
367 // Returns the SVETypeFlags for a given value and mask.
368 uint64_t encodeFlag(uint64_t V, StringRef MaskName) const {
369 auto It = FlagTypes.find(Key: MaskName);
370 if (It != FlagTypes.end()) {
371 uint64_t Mask = It->getValue();
372 unsigned Shift = countr_zero(Val: Mask);
373 assert(Shift < 64 && "Mask value produced an invalid shift value");
374 return (V << Shift) & Mask;
375 }
376 llvm_unreachable("Unsupported flag");
377 }
378
379 // Returns the SVETypeFlags for the given element type.
380 uint64_t encodeEltType(StringRef EltName) {
381 auto It = EltTypes.find(Key: EltName);
382 if (It != EltTypes.end())
383 return encodeFlag(V: It->getValue(), MaskName: "EltTypeMask");
384 llvm_unreachable("Unsupported EltType");
385 }
386
387 // Returns the SVETypeFlags for the given memory element type.
388 uint64_t encodeMemoryElementType(uint64_t MT) {
389 return encodeFlag(V: MT, MaskName: "MemEltTypeMask");
390 }
391
392 // Returns the SVETypeFlags for the given merge type.
393 uint64_t encodeMergeType(uint64_t MT) {
394 return encodeFlag(V: MT, MaskName: "MergeTypeMask");
395 }
396
397 // Returns the SVETypeFlags for the given splat operand.
398 unsigned encodeSplatOperand(unsigned SplatIdx) {
399 assert(SplatIdx < 7 && "SplatIdx out of encodable range");
400 return encodeFlag(V: SplatIdx + 1, MaskName: "SplatOperandMask");
401 }
402
403 // Returns the SVETypeFlags value for the given SVEType.
404 uint64_t encodeTypeFlags(const SVEType &T);
405
406 /// Emit arm_sve.h.
407 void createHeader(raw_ostream &o);
408
409 // Emits core intrinsics in both arm_sme.h and arm_sve.h
410 void createCoreHeaderIntrinsics(raw_ostream &o, SVEEmitter &Emitter,
411 ACLEKind Kind);
412
413 /// Emit all the __builtin prototypes and code needed by Sema.
414 void createBuiltins(raw_ostream &o);
415
416 /// Emit all the __builtin prototypes in JSON format.
417 void createBuiltinsJSON(raw_ostream &o);
418
419 /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
420 void createCodeGenMap(raw_ostream &o);
421
422 /// Emit all the range checks for the immediates.
423 void createRangeChecks(raw_ostream &o);
424
425 // Emit all the ImmCheckTypes to arm_immcheck_types.inc
426 void createImmCheckTypes(raw_ostream &OS);
427
428 /// Create the SVETypeFlags used in CGBuiltins
429 void createTypeFlags(raw_ostream &o);
430
431 /// Emit arm_sme.h.
432 void createSMEHeader(raw_ostream &o);
433
434 /// Emit all the SME __builtin prototypes and code needed by Sema.
435 void createSMEBuiltins(raw_ostream &o);
436
437 /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
438 void createSMECodeGenMap(raw_ostream &o);
439
440 /// Create a table for a builtin's requirement for PSTATE.SM.
441 void createStreamingAttrs(raw_ostream &o, ACLEKind Kind);
442
443 /// Emit all the range checks for the immediates.
444 void createSMERangeChecks(raw_ostream &o);
445
446 /// Create a table for a builtin's requirement for PSTATE.ZA.
447 void createBuiltinZAState(raw_ostream &OS);
448
449 /// Create intrinsic and add it to \p Out
450 void createIntrinsic(const Record *R,
451 SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out);
452};
453
454const std::array<SVEEmitter::ReinterpretTypeInfo, 13> SVEEmitter::Reinterprets =
455 {._M_elems: {{.BaseType: SVEType("c", 'd'), .Suffix: "s8"},
456 {.BaseType: SVEType("Uc", 'd'), .Suffix: "u8"},
457 {.BaseType: SVEType("m", 'd'), .Suffix: "mf8"},
458 {.BaseType: SVEType("s", 'd'), .Suffix: "s16"},
459 {.BaseType: SVEType("Us", 'd'), .Suffix: "u16"},
460 {.BaseType: SVEType("i", 'd'), .Suffix: "s32"},
461 {.BaseType: SVEType("Ui", 'd'), .Suffix: "u32"},
462 {.BaseType: SVEType("l", 'd'), .Suffix: "s64"},
463 {.BaseType: SVEType("Ul", 'd'), .Suffix: "u64"},
464 {.BaseType: SVEType("h", 'd'), .Suffix: "f16"},
465 {.BaseType: SVEType("b", 'd'), .Suffix: "bf16"},
466 {.BaseType: SVEType("f", 'd'), .Suffix: "f32"},
467 {.BaseType: SVEType("d", 'd'), .Suffix: "f64"}}};
468
469} // end anonymous namespace
470
471//===----------------------------------------------------------------------===//
472// Type implementation
473//===----------------------------------------------------------------------===//
474
475std::string SVEType::builtinBaseType() const {
476 switch (Kind) {
477 case TypeKind::Void:
478 return "v";
479 case TypeKind::Svcount:
480 return "Qa";
481 case TypeKind::PrefetchOp:
482 case TypeKind::PredicatePattern:
483 return "i";
484 case TypeKind::Fpm:
485 return "UWi";
486 case TypeKind::Predicate:
487 return "b";
488 case TypeKind::BFloat16:
489 assert(ElementBitwidth == 16 && "Invalid BFloat16!");
490 return "y";
491 case TypeKind::MFloat8:
492 assert(ElementBitwidth == 8 && "Invalid MFloat8!");
493 return "m";
494 case TypeKind::Float:
495 switch (ElementBitwidth) {
496 case 16:
497 return "h";
498 case 32:
499 return "f";
500 case 64:
501 return "d";
502 default:
503 llvm_unreachable("Unhandled float width!");
504 }
505 case TypeKind::SInt:
506 case TypeKind::UInt:
507 switch (ElementBitwidth) {
508 case 1:
509 return "b";
510 case 8:
511 return "c";
512 case 16:
513 return "s";
514 case 32:
515 return "i";
516 case 64:
517 return "Wi";
518 case 128:
519 return "LLLi";
520 default:
521 llvm_unreachable("Unhandled bitwidth!");
522 }
523 case TypeKind::Invalid:
524 llvm_unreachable("Attempting to resolve builtin string from Invalid type!");
525 }
526 llvm_unreachable("Unhandled TypeKind!");
527}
528
529std::string SVEType::builtin_str() const {
530 std::string Prefix;
531
532 if (isScalableVector())
533 Prefix = "q" + llvm::utostr(X: getNumElements() * NumVectors);
534 else if (isFixedLengthVector())
535 Prefix = "V" + llvm::utostr(X: getNumElements() * NumVectors);
536 else if (isImmediate()) {
537 assert(!isFloatingPoint() && "fp immediates are not supported");
538 Prefix = "I";
539 }
540
541 // Make chars and integer pointers explicitly signed.
542 if ((ElementBitwidth == 8 || isPointer()) && isSignedInteger())
543 Prefix += "S";
544 else if (isUnsignedInteger())
545 Prefix += "U";
546
547 std::string BuiltinStr = Prefix + builtinBaseType();
548 if (isConstant())
549 BuiltinStr += "C";
550 if (isPointer())
551 BuiltinStr += "*";
552
553 return BuiltinStr;
554}
555
556std::string SVEType::str() const {
557 std::string TypeStr;
558
559 switch (Kind) {
560 case TypeKind::PrefetchOp:
561 return "enum svprfop";
562 case TypeKind::PredicatePattern:
563 return "enum svpattern";
564 case TypeKind::Fpm:
565 TypeStr += "fpm";
566 break;
567 case TypeKind::Void:
568 TypeStr += "void";
569 break;
570 case TypeKind::Float:
571 TypeStr += "float" + llvm::utostr(X: ElementBitwidth);
572 break;
573 case TypeKind::Svcount:
574 TypeStr += "svcount";
575 break;
576 case TypeKind::Predicate:
577 TypeStr += "bool";
578 break;
579 case TypeKind::BFloat16:
580 TypeStr += "bfloat16";
581 break;
582 case TypeKind::MFloat8:
583 TypeStr += "mfloat8";
584 break;
585 case TypeKind::SInt:
586 TypeStr += "int" + llvm::utostr(X: ElementBitwidth);
587 break;
588 case TypeKind::UInt:
589 TypeStr += "uint" + llvm::utostr(X: ElementBitwidth);
590 break;
591 case TypeKind::Invalid:
592 llvm_unreachable("Attempting to resolve type name from Invalid type!");
593 }
594
595 if (isFixedLengthVector())
596 TypeStr += "x" + llvm::utostr(X: getNumElements());
597 else if (isScalableVector())
598 TypeStr = "sv" + TypeStr;
599
600 if (NumVectors > 1)
601 TypeStr += "x" + llvm::utostr(X: NumVectors);
602 if (!isScalarPredicate() && !isVoid())
603 TypeStr += "_t";
604 if (isConstant())
605 TypeStr += " const";
606 if (isPointer())
607 TypeStr += " *";
608
609 return TypeStr;
610}
611
612void SVEType::applyTypespec(StringRef TS) {
613 for (char I : TS) {
614 switch (I) {
615 case 'Q':
616 assert(isInvalid() && "Unexpected use of typespec modifier");
617 Kind = Svcount;
618 break;
619 case 'P':
620 assert(isInvalid() && "Unexpected use of typespec modifier");
621 Kind = Predicate;
622 break;
623 case 'U':
624 assert(isInvalid() && "Unexpected use of typespec modifier");
625 Kind = UInt;
626 break;
627 case 'c':
628 Kind = isInvalid() ? SInt : Kind;
629 ElementBitwidth = 8;
630 break;
631 case 's':
632 Kind = isInvalid() ? SInt : Kind;
633 ElementBitwidth = 16;
634 break;
635 case 'i':
636 Kind = isInvalid() ? SInt : Kind;
637 ElementBitwidth = 32;
638 break;
639 case 'l':
640 Kind = isInvalid() ? SInt : Kind;
641 ElementBitwidth = 64;
642 break;
643 case 'q':
644 Kind = isInvalid() ? SInt : Kind;
645 ElementBitwidth = 128;
646 break;
647 case 'h':
648 assert(isInvalid() && "Unexpected use of typespec modifier");
649 Kind = Float;
650 ElementBitwidth = 16;
651 break;
652 case 'f':
653 assert(isInvalid() && "Unexpected use of typespec modifier");
654 Kind = Float;
655 ElementBitwidth = 32;
656 break;
657 case 'd':
658 assert(isInvalid() && "Unexpected use of typespec modifier");
659 Kind = Float;
660 ElementBitwidth = 64;
661 break;
662 case 'b':
663 assert(isInvalid() && "Unexpected use of typespec modifier");
664 Kind = BFloat16;
665 ElementBitwidth = 16;
666 break;
667 case 'm':
668 assert(isInvalid() && "Unexpected use of typespec modifier");
669 Kind = MFloat8;
670 ElementBitwidth = 8;
671 break;
672 default:
673 llvm_unreachable("Unhandled type code!");
674 }
675 }
676 assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
677}
678
679void SVEType::applyModifier(char Mod) {
680 switch (Mod) {
681 case 'v':
682 Kind = Void;
683 NumVectors = 0;
684 break;
685 case 'd':
686 DefaultType = true;
687 break;
688 case 'c':
689 Constant = true;
690 [[fallthrough]];
691 case 'p':
692 Pointer = true;
693 Bitwidth = ElementBitwidth;
694 NumVectors = 0;
695 break;
696 case 'e':
697 Kind = UInt;
698 ElementBitwidth /= 2;
699 break;
700 case 'h':
701 ElementBitwidth /= 2;
702 break;
703 case 'q':
704 ElementBitwidth /= 4;
705 break;
706 case 'b':
707 Kind = UInt;
708 ElementBitwidth /= 4;
709 break;
710 case 'o':
711 ElementBitwidth *= 4;
712 break;
713 case 'P':
714 Kind = Predicate;
715 Bitwidth = 16;
716 ElementBitwidth = 1;
717 break;
718 case '{':
719 IsScalable = false;
720 Bitwidth = 128;
721 NumVectors = 1;
722 break;
723 case 's':
724 case 'a':
725 Bitwidth = ElementBitwidth;
726 NumVectors = 0;
727 break;
728 case 'R':
729 ElementBitwidth /= 2;
730 NumVectors = 0;
731 break;
732 case 'r':
733 ElementBitwidth /= 4;
734 NumVectors = 0;
735 break;
736 case '@':
737 Kind = UInt;
738 ElementBitwidth /= 4;
739 NumVectors = 0;
740 break;
741 case 'K':
742 Kind = SInt;
743 Bitwidth = ElementBitwidth;
744 NumVectors = 0;
745 break;
746 case 'L':
747 Kind = UInt;
748 Bitwidth = ElementBitwidth;
749 NumVectors = 0;
750 break;
751 case 'u':
752 Kind = UInt;
753 break;
754 case 'x':
755 Kind = SInt;
756 break;
757 case 'i':
758 Kind = UInt;
759 ElementBitwidth = Bitwidth = 64;
760 NumVectors = 0;
761 Immediate = true;
762 break;
763 case 'I':
764 Kind = PredicatePattern;
765 ElementBitwidth = Bitwidth = 32;
766 NumVectors = 0;
767 Immediate = true;
768 break;
769 case 'J':
770 Kind = PrefetchOp;
771 ElementBitwidth = Bitwidth = 32;
772 NumVectors = 0;
773 Immediate = true;
774 break;
775 case 'k':
776 Kind = SInt;
777 ElementBitwidth = Bitwidth = 32;
778 NumVectors = 0;
779 break;
780 case 'l':
781 Kind = SInt;
782 ElementBitwidth = Bitwidth = 64;
783 NumVectors = 0;
784 break;
785 case 'm':
786 Kind = UInt;
787 ElementBitwidth = Bitwidth = 32;
788 NumVectors = 0;
789 break;
790 case '>':
791 Kind = Fpm;
792 ElementBitwidth = Bitwidth = 64;
793 NumVectors = 0;
794 break;
795 case 'n':
796 Kind = UInt;
797 ElementBitwidth = Bitwidth = 64;
798 NumVectors = 0;
799 break;
800 case 'w':
801 ElementBitwidth = 64;
802 break;
803 case 'j':
804 ElementBitwidth = Bitwidth = 64;
805 NumVectors = 0;
806 break;
807 case 'f':
808 Kind = UInt;
809 ElementBitwidth = Bitwidth = 64;
810 NumVectors = 0;
811 break;
812 case 'g':
813 Kind = UInt;
814 ElementBitwidth = 64;
815 break;
816 case '#':
817 Kind = SInt;
818 ElementBitwidth = 64;
819 break;
820 case '[':
821 Kind = UInt;
822 ElementBitwidth = 8;
823 break;
824 case 't':
825 Kind = SInt;
826 ElementBitwidth = 32;
827 break;
828 case 'z':
829 Kind = UInt;
830 ElementBitwidth = 32;
831 break;
832 case 'O':
833 Kind = Float;
834 ElementBitwidth = 16;
835 break;
836 case 'M':
837 Kind = Float;
838 ElementBitwidth = 32;
839 break;
840 case 'N':
841 Kind = Float;
842 ElementBitwidth = 64;
843 break;
844 case 'Q':
845 Kind = Void;
846 Constant = true;
847 Pointer = true;
848 NumVectors = 0;
849 break;
850 case 'S':
851 Kind = SInt;
852 Constant = true;
853 Pointer = true;
854 ElementBitwidth = Bitwidth = 8;
855 NumVectors = 0;
856 break;
857 case 'W':
858 Kind = UInt;
859 Constant = true;
860 Pointer = true;
861 ElementBitwidth = Bitwidth = 8;
862 NumVectors = 0;
863 break;
864 case 'T':
865 Kind = SInt;
866 Constant = true;
867 Pointer = true;
868 ElementBitwidth = Bitwidth = 16;
869 NumVectors = 0;
870 break;
871 case 'X':
872 Kind = UInt;
873 Constant = true;
874 Pointer = true;
875 ElementBitwidth = Bitwidth = 16;
876 NumVectors = 0;
877 break;
878 case 'Y':
879 Kind = UInt;
880 Constant = true;
881 Pointer = true;
882 ElementBitwidth = Bitwidth = 32;
883 NumVectors = 0;
884 break;
885 case 'U':
886 Kind = SInt;
887 Constant = true;
888 Pointer = true;
889 ElementBitwidth = Bitwidth = 32;
890 NumVectors = 0;
891 break;
892 case '%':
893 Kind = Void;
894 Pointer = true;
895 NumVectors = 0;
896 break;
897 case 'A':
898 Kind = SInt;
899 Pointer = true;
900 ElementBitwidth = Bitwidth = 8;
901 NumVectors = 0;
902 break;
903 case 'B':
904 Kind = SInt;
905 Pointer = true;
906 ElementBitwidth = Bitwidth = 16;
907 NumVectors = 0;
908 break;
909 case 'C':
910 Kind = SInt;
911 Pointer = true;
912 ElementBitwidth = Bitwidth = 32;
913 NumVectors = 0;
914 break;
915 case 'D':
916 Kind = SInt;
917 Pointer = true;
918 ElementBitwidth = Bitwidth = 64;
919 NumVectors = 0;
920 break;
921 case 'E':
922 Kind = UInt;
923 Pointer = true;
924 ElementBitwidth = Bitwidth = 8;
925 NumVectors = 0;
926 break;
927 case 'F':
928 Kind = UInt;
929 Pointer = true;
930 ElementBitwidth = Bitwidth = 16;
931 NumVectors = 0;
932 break;
933 case 'G':
934 Kind = UInt;
935 Pointer = true;
936 ElementBitwidth = Bitwidth = 32;
937 NumVectors = 0;
938 break;
939 case '$':
940 Kind = BFloat16;
941 ElementBitwidth = 16;
942 break;
943 case '}':
944 Kind = Svcount;
945 NumVectors = 0;
946 break;
947 case '~':
948 Kind = MFloat8;
949 ElementBitwidth = 8;
950 break;
951 case '!':
952 Kind = MFloat8;
953 Bitwidth = ElementBitwidth = 8;
954 NumVectors = 0;
955 break;
956 case '.':
957 llvm_unreachable(". is never a type in itself");
958 break;
959 default:
960 llvm_unreachable("Unhandled character!");
961 }
962}
963
964/// Returns the modifier and number of vectors for the given operand \p Op.
965std::pair<char, unsigned> getProtoModifier(StringRef Proto, unsigned Op) {
966 for (unsigned P = 0; !Proto.empty(); ++P) {
967 unsigned NumVectors = 1;
968 unsigned CharsToSkip = 1;
969 char Mod = Proto[0];
970 if (Mod == '2' || Mod == '3' || Mod == '4') {
971 NumVectors = Mod - '0';
972 Mod = 'd';
973 if (Proto.size() > 1 && Proto[1] == '.') {
974 Mod = Proto[2];
975 CharsToSkip = 3;
976 }
977 }
978
979 if (P == Op)
980 return {Mod, NumVectors};
981
982 Proto = Proto.drop_front(N: CharsToSkip);
983 }
984 llvm_unreachable("Unexpected Op");
985}
986
987//===----------------------------------------------------------------------===//
988// Intrinsic implementation
989//===----------------------------------------------------------------------===//
990
991Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
992 StringRef MergeSuffix, uint64_t MemoryElementTy,
993 StringRef LLVMName, uint64_t Flags,
994 ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class,
995 SVEEmitter &Emitter, StringRef SVEGuard,
996 StringRef SMEGuard)
997 : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()),
998 BaseTypeSpec(BT), Class(Class), MergeSuffix(MergeSuffix.str()),
999 BaseType(BT, 'd'), Flags(Flags), ImmChecks(Checks) {
1000
1001 auto FormatGuard = [](StringRef Guard, StringRef Base) -> std::string {
1002 if (Guard.empty() || Guard == Base)
1003 return Guard.str();
1004
1005 unsigned Depth = 0;
1006 for (auto &C : Guard) {
1007 switch (C) {
1008 default:
1009 break;
1010 case '|':
1011 if (Depth == 0)
1012 // Group top-level ORs before ANDing with the base feature.
1013 return Base.str() + ",(" + Guard.str() + ")";
1014 break;
1015 case '(':
1016 ++Depth;
1017 break;
1018 case ')':
1019 if (Depth == 0)
1020 llvm_unreachable("Mismatched parentheses!");
1021
1022 --Depth;
1023 break;
1024 }
1025 }
1026
1027 return Base.str() + "," + Guard.str();
1028 };
1029
1030 this->SVEGuard = FormatGuard(SVEGuard, "sve");
1031 this->SMEGuard = FormatGuard(SMEGuard, "sme");
1032
1033 // Types[0] is the return value.
1034 for (unsigned I = 0; I < (getNumParams() + 1); ++I) {
1035 char Mod;
1036 unsigned NumVectors;
1037 std::tie(args&: Mod, args&: NumVectors) = getProtoModifier(Proto, Op: I);
1038 SVEType T(BaseTypeSpec, Mod, NumVectors);
1039 Types.push_back(x: T);
1040 SetsFPMR = T.isFpm();
1041
1042 // Add range checks for immediates
1043 if (I > 0) {
1044 if (T.isPredicatePattern())
1045 ImmChecks.emplace_back(
1046 Args: I - 1, Args: Emitter.getEnumValueForImmCheck(C: "ImmCheck0_31"));
1047 else if (T.isPrefetchOp())
1048 ImmChecks.emplace_back(
1049 Args: I - 1, Args: Emitter.getEnumValueForImmCheck(C: "ImmCheck0_13"));
1050 }
1051 }
1052
1053 // Set flags based on properties
1054 this->Flags |= Emitter.encodeTypeFlags(T: BaseType);
1055 this->Flags |= Emitter.encodeMemoryElementType(MT: MemoryElementTy);
1056 this->Flags |= Emitter.encodeMergeType(MT: MergeTy);
1057 if (hasSplat())
1058 this->Flags |= Emitter.encodeSplatOperand(SplatIdx: getSplatIdx());
1059 if (SetsFPMR)
1060 this->Flags |= Emitter.getEnumValueForFlag(C: "SetsFPMR");
1061}
1062
1063std::string Intrinsic::getBuiltinTypeStr() {
1064 std::string S = getReturnType().builtin_str();
1065 for (unsigned I = 0; I < getNumParams(); ++I)
1066 S += getParamType(I).builtin_str();
1067
1068 return S;
1069}
1070
1071std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
1072 std::string Proto) const {
1073 std::string Ret = Name;
1074 while (Ret.find(c: '{') != std::string::npos) {
1075 size_t Pos = Ret.find(c: '{');
1076 size_t End = Ret.find(c: '}');
1077 unsigned NumChars = End - Pos + 1;
1078 assert(NumChars == 3 && "Unexpected template argument");
1079
1080 SVEType T;
1081 char C = Ret[Pos+1];
1082 switch(C) {
1083 default:
1084 llvm_unreachable("Unknown predication specifier");
1085 case 'd':
1086 T = SVEType(TS, 'd');
1087 break;
1088 case '0':
1089 case '1':
1090 case '2':
1091 case '3':
1092 // Extract the modifier before passing to SVEType to handle numeric
1093 // modifiers
1094 auto [Mod, NumVectors] = getProtoModifier(Proto, Op: (C - '0'));
1095 T = SVEType(TS, Mod);
1096 break;
1097 }
1098
1099 // Replace templated arg with the right suffix (e.g. u32)
1100 std::string TypeCode;
1101
1102 if (T.isSignedInteger())
1103 TypeCode = 's';
1104 else if (T.isUnsignedInteger())
1105 TypeCode = 'u';
1106 else if (T.isSvcount())
1107 TypeCode = 'c';
1108 else if (T.isPredicate())
1109 TypeCode = 'b';
1110 else if (T.isBFloat())
1111 TypeCode = "bf";
1112 else if (T.isMFloat())
1113 TypeCode = "mf";
1114 else
1115 TypeCode = 'f';
1116 Ret.replace(pos: Pos, n: NumChars, str: TypeCode + utostr(X: T.getElementSizeInBits()));
1117 }
1118
1119 return Ret;
1120}
1121
1122std::string Intrinsic::mangleLLVMName() const {
1123 std::string S = getLLVMName();
1124
1125 // Replace all {d} like expressions with e.g. 'u32'
1126 return replaceTemplatedArgs(Name: S, TS: getBaseTypeSpec(), Proto: getProto());
1127}
1128
1129std::string Intrinsic::mangleName(ClassKind LocalCK) const {
1130 std::string S = getName();
1131
1132 if (LocalCK == ClassG) {
1133 // Remove the square brackets and everything in between.
1134 while (S.find(c: '[') != std::string::npos) {
1135 auto Start = S.find(c: '[');
1136 auto End = S.find(c: ']');
1137 S.erase(pos: Start, n: (End-Start)+1);
1138 }
1139 } else {
1140 // Remove the square brackets.
1141 while (S.find(c: '[') != std::string::npos) {
1142 auto BrPos = S.find(c: '[');
1143 if (BrPos != std::string::npos)
1144 S.erase(pos: BrPos, n: 1);
1145 BrPos = S.find(c: ']');
1146 if (BrPos != std::string::npos)
1147 S.erase(pos: BrPos, n: 1);
1148 }
1149 }
1150
1151 // Replace all {d} like expressions with e.g. 'u32'
1152 return replaceTemplatedArgs(Name: S, TS: getBaseTypeSpec(), Proto: getProto())
1153 .append(str: getMergeSuffix())
1154 .append(svt: getFPMSuffix());
1155}
1156
1157void Intrinsic::emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter,
1158 ACLEKind Kind) const {
1159 bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1;
1160
1161 std::string FullName = mangleName(LocalCK: ClassS);
1162 std::string ProtoName = mangleName(LocalCK: getClassKind());
1163 OS << (IsOverloaded ? "__aio " : "__ai ")
1164 << "__attribute__((__clang_arm_builtin_alias(";
1165
1166 switch (Kind) {
1167 case ACLEKind::SME:
1168 OS << "__builtin_sme_" << FullName << ")";
1169 break;
1170 case ACLEKind::SVE:
1171 OS << "__builtin_sve_" << FullName << ")";
1172 break;
1173 }
1174
1175 OS << "))\n";
1176
1177 OS << getTypes()[0].str() << " " << ProtoName << "(";
1178 for (unsigned I = 0; I < getTypes().size() - 1; ++I) {
1179 if (I != 0)
1180 OS << ", ";
1181 OS << getTypes()[I + 1].str();
1182 }
1183 OS << ");\n";
1184}
1185
1186//===----------------------------------------------------------------------===//
1187// SVEEmitter implementation
1188//===----------------------------------------------------------------------===//
1189uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {
1190 if (T.isFloat()) {
1191 switch (T.getElementSizeInBits()) {
1192 case 16:
1193 return encodeEltType(EltName: "EltTyFloat16");
1194 case 32:
1195 return encodeEltType(EltName: "EltTyFloat32");
1196 case 64:
1197 return encodeEltType(EltName: "EltTyFloat64");
1198 default:
1199 llvm_unreachable("Unhandled float element bitwidth!");
1200 }
1201 }
1202
1203 if (T.isBFloat()) {
1204 assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat.");
1205 return encodeEltType(EltName: "EltTyBFloat16");
1206 }
1207
1208 if (T.isMFloat()) {
1209 assert(T.getElementSizeInBits() == 8 && "Not a valid MFloat.");
1210 return encodeEltType(EltName: "EltTyMFloat8");
1211 }
1212
1213 if (T.isPredicate() || T.isSvcount()) {
1214 switch (T.getElementSizeInBits()) {
1215 case 8:
1216 return encodeEltType(EltName: "EltTyBool8");
1217 case 16:
1218 return encodeEltType(EltName: "EltTyBool16");
1219 case 32:
1220 return encodeEltType(EltName: "EltTyBool32");
1221 case 64:
1222 return encodeEltType(EltName: "EltTyBool64");
1223 default:
1224 llvm_unreachable("Unhandled predicate element bitwidth!");
1225 }
1226 }
1227
1228 switch (T.getElementSizeInBits()) {
1229 case 8:
1230 return encodeEltType(EltName: "EltTyInt8");
1231 case 16:
1232 return encodeEltType(EltName: "EltTyInt16");
1233 case 32:
1234 return encodeEltType(EltName: "EltTyInt32");
1235 case 64:
1236 return encodeEltType(EltName: "EltTyInt64");
1237 case 128:
1238 return encodeEltType(EltName: "EltTyInt128");
1239 default:
1240 llvm_unreachable("Unhandled integer element bitwidth!");
1241 }
1242}
1243
1244void SVEEmitter::createIntrinsic(
1245 const Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) {
1246 StringRef Name = R->getValueAsString(FieldName: "Name");
1247 StringRef Proto = R->getValueAsString(FieldName: "Prototype");
1248 StringRef Types = R->getValueAsString(FieldName: "Types");
1249 StringRef SVEGuard = R->getValueAsString(FieldName: "SVETargetGuard");
1250 StringRef SMEGuard = R->getValueAsString(FieldName: "SMETargetGuard");
1251 StringRef LLVMName = R->getValueAsString(FieldName: "LLVMIntrinsic");
1252 uint64_t Merge = R->getValueAsInt(FieldName: "Merge");
1253 StringRef MergeSuffix = R->getValueAsString(FieldName: "MergeSuffix");
1254 uint64_t MemEltType = R->getValueAsInt(FieldName: "MemEltType");
1255
1256 int64_t Flags = 0;
1257 for (const Record *FlagRec : R->getValueAsListOfDefs(FieldName: "Flags"))
1258 Flags |= FlagRec->getValueAsInt(FieldName: "Value");
1259
1260 // Create a dummy TypeSpec for non-overloaded builtins.
1261 if (Types.empty()) {
1262 assert((Flags & getEnumValueForFlag("IsOverloadNone")) &&
1263 "Expect TypeSpec for overloaded builtin!");
1264 Types = "i";
1265 }
1266
1267 // Extract type specs from string
1268 SmallVector<TypeSpec, 8> TypeSpecs;
1269 TypeSpec Acc;
1270 for (char I : Types) {
1271 Acc.push_back(c: I);
1272 if (islower(I)) {
1273 TypeSpecs.push_back(Elt: TypeSpec(Acc));
1274 Acc.clear();
1275 }
1276 }
1277
1278 // Remove duplicate type specs.
1279 sort(C&: TypeSpecs);
1280 TypeSpecs.erase(CS: llvm::unique(R&: TypeSpecs), CE: TypeSpecs.end());
1281
1282 // Create an Intrinsic for each type spec.
1283 for (auto TS : TypeSpecs) {
1284 // Collate a list of range/option checks for the immediates.
1285 SmallVector<ImmCheck, 2> ImmChecks;
1286 for (const Record *ImmR : R->getValueAsListOfDefs(FieldName: "ImmChecks")) {
1287 int64_t ArgIdx = ImmR->getValueAsInt(FieldName: "ImmArgIdx");
1288 int64_t EltSizeArgIdx = ImmR->getValueAsInt(FieldName: "TypeContextArgIdx");
1289 int64_t Kind = ImmR->getValueAsDef(FieldName: "Kind")->getValueAsInt(FieldName: "Value");
1290 assert(ArgIdx >= 0 && Kind >= 0 &&
1291 "ImmArgIdx and Kind must be nonnegative");
1292
1293 unsigned ElementSizeInBits = 0;
1294 auto [Mod, NumVectors] = getProtoModifier(Proto, Op: EltSizeArgIdx + 1);
1295 if (EltSizeArgIdx >= 0)
1296 ElementSizeInBits = SVEType(TS, Mod, NumVectors).getElementSizeInBits();
1297 ImmChecks.push_back(Elt: ImmCheck(ArgIdx, Kind, ElementSizeInBits));
1298 }
1299
1300 Out.push_back(Elt: std::make_unique<Intrinsic>(
1301 args&: Name, args&: Proto, args&: Merge, args&: MergeSuffix, args&: MemEltType, args&: LLVMName, args&: Flags, args&: ImmChecks,
1302 args&: TS, args: ClassS, args&: *this, args&: SVEGuard, args&: SMEGuard));
1303
1304 // Also generate the short-form (e.g. svadd_m) for the given type-spec.
1305 if (Intrinsic::isOverloadedIntrinsic(Name))
1306 Out.push_back(Elt: std::make_unique<Intrinsic>(
1307 args&: Name, args&: Proto, args&: Merge, args&: MergeSuffix, args&: MemEltType, args&: LLVMName, args&: Flags,
1308 args&: ImmChecks, args&: TS, args: ClassG, args&: *this, args&: SVEGuard, args&: SMEGuard));
1309 }
1310}
1311
1312void SVEEmitter::createCoreHeaderIntrinsics(raw_ostream &OS,
1313 SVEEmitter &Emitter,
1314 ACLEKind Kind) {
1315 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1316 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1317 for (auto *R : RV)
1318 createIntrinsic(R, Out&: Defs);
1319
1320 // Sort intrinsics in header file by following order/priority:
1321 // - Architectural guard (i.e. does it require SVE2 or SVE2_AES)
1322 // - Class (is intrinsic overloaded or not)
1323 // - Intrinsic name
1324 llvm::stable_sort(Range&: Defs, C: [](const std::unique_ptr<Intrinsic> &A,
1325 const std::unique_ptr<Intrinsic> &B) {
1326 auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) {
1327 return std::make_tuple(args: I->getSVEGuard().str() + I->getSMEGuard().str(),
1328 args: (unsigned)I->getClassKind(), args: I->getName());
1329 };
1330 return ToTuple(A) < ToTuple(B);
1331 });
1332
1333 // Actually emit the intrinsic declarations.
1334 for (auto &I : Defs)
1335 I->emitIntrinsic(OS, Emitter, Kind);
1336}
1337
1338void SVEEmitter::createHeader(raw_ostream &OS) {
1339 OS << "/*===---- arm_sve.h - ARM SVE intrinsics "
1340 "-----------------------------------===\n"
1341 " *\n"
1342 " *\n"
1343 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1344 "Exceptions.\n"
1345 " * See https://llvm.org/LICENSE.txt for license information.\n"
1346 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1347 " *\n"
1348 " *===-----------------------------------------------------------------"
1349 "------===\n"
1350 " */\n\n";
1351
1352 OS << "#ifndef __ARM_SVE_H\n";
1353 OS << "#define __ARM_SVE_H\n\n";
1354
1355 OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1356 OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n";
1357 OS << "#endif\n";
1358
1359 OS << "#include <stdint.h>\n\n";
1360 OS << "#ifdef __cplusplus\n";
1361 OS << "extern \"C\" {\n";
1362 OS << "#else\n";
1363 OS << "#include <stdbool.h>\n";
1364 OS << "#endif\n\n";
1365
1366 OS << "typedef __fp16 float16_t;\n";
1367 OS << "typedef float float32_t;\n";
1368 OS << "typedef double float64_t;\n";
1369
1370 OS << "typedef __SVInt8_t svint8_t;\n";
1371 OS << "typedef __SVInt16_t svint16_t;\n";
1372 OS << "typedef __SVInt32_t svint32_t;\n";
1373 OS << "typedef __SVInt64_t svint64_t;\n";
1374 OS << "typedef __SVUint8_t svuint8_t;\n";
1375 OS << "typedef __SVUint16_t svuint16_t;\n";
1376 OS << "typedef __SVUint32_t svuint32_t;\n";
1377 OS << "typedef __SVUint64_t svuint64_t;\n";
1378 OS << "typedef __SVFloat16_t svfloat16_t;\n\n";
1379
1380 OS << "typedef __SVBfloat16_t svbfloat16_t;\n";
1381
1382 OS << "#include <arm_bf16.h>\n";
1383 OS << "#include <arm_vector_types.h>\n";
1384
1385 OS << "typedef __SVMfloat8_t svmfloat8_t;\n\n";
1386
1387 OS << "typedef __SVFloat32_t svfloat32_t;\n";
1388 OS << "typedef __SVFloat64_t svfloat64_t;\n";
1389 OS << "typedef __clang_svint8x2_t svint8x2_t;\n";
1390 OS << "typedef __clang_svint16x2_t svint16x2_t;\n";
1391 OS << "typedef __clang_svint32x2_t svint32x2_t;\n";
1392 OS << "typedef __clang_svint64x2_t svint64x2_t;\n";
1393 OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n";
1394 OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n";
1395 OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n";
1396 OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n";
1397 OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n";
1398 OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n";
1399 OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n";
1400 OS << "typedef __clang_svint8x3_t svint8x3_t;\n";
1401 OS << "typedef __clang_svint16x3_t svint16x3_t;\n";
1402 OS << "typedef __clang_svint32x3_t svint32x3_t;\n";
1403 OS << "typedef __clang_svint64x3_t svint64x3_t;\n";
1404 OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n";
1405 OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n";
1406 OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n";
1407 OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n";
1408 OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n";
1409 OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n";
1410 OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n";
1411 OS << "typedef __clang_svint8x4_t svint8x4_t;\n";
1412 OS << "typedef __clang_svint16x4_t svint16x4_t;\n";
1413 OS << "typedef __clang_svint32x4_t svint32x4_t;\n";
1414 OS << "typedef __clang_svint64x4_t svint64x4_t;\n";
1415 OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n";
1416 OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n";
1417 OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n";
1418 OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n";
1419 OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n";
1420 OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n";
1421 OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n";
1422 OS << "typedef __SVBool_t svbool_t;\n";
1423 OS << "typedef __clang_svboolx2_t svboolx2_t;\n";
1424 OS << "typedef __clang_svboolx4_t svboolx4_t;\n\n";
1425
1426 OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";
1427 OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";
1428 OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";
1429
1430 OS << "typedef __clang_svmfloat8x2_t svmfloat8x2_t;\n";
1431 OS << "typedef __clang_svmfloat8x3_t svmfloat8x3_t;\n";
1432 OS << "typedef __clang_svmfloat8x4_t svmfloat8x4_t;\n";
1433
1434 OS << "typedef __SVCount_t svcount_t;\n\n";
1435
1436 OS << "enum svpattern\n";
1437 OS << "{\n";
1438 OS << " SV_POW2 = 0,\n";
1439 OS << " SV_VL1 = 1,\n";
1440 OS << " SV_VL2 = 2,\n";
1441 OS << " SV_VL3 = 3,\n";
1442 OS << " SV_VL4 = 4,\n";
1443 OS << " SV_VL5 = 5,\n";
1444 OS << " SV_VL6 = 6,\n";
1445 OS << " SV_VL7 = 7,\n";
1446 OS << " SV_VL8 = 8,\n";
1447 OS << " SV_VL16 = 9,\n";
1448 OS << " SV_VL32 = 10,\n";
1449 OS << " SV_VL64 = 11,\n";
1450 OS << " SV_VL128 = 12,\n";
1451 OS << " SV_VL256 = 13,\n";
1452 OS << " SV_MUL4 = 29,\n";
1453 OS << " SV_MUL3 = 30,\n";
1454 OS << " SV_ALL = 31\n";
1455 OS << "};\n\n";
1456
1457 OS << "enum svprfop\n";
1458 OS << "{\n";
1459 OS << " SV_PLDL1KEEP = 0,\n";
1460 OS << " SV_PLDL1STRM = 1,\n";
1461 OS << " SV_PLDL2KEEP = 2,\n";
1462 OS << " SV_PLDL2STRM = 3,\n";
1463 OS << " SV_PLDL3KEEP = 4,\n";
1464 OS << " SV_PLDL3STRM = 5,\n";
1465 OS << " SV_PSTL1KEEP = 8,\n";
1466 OS << " SV_PSTL1STRM = 9,\n";
1467 OS << " SV_PSTL2KEEP = 10,\n";
1468 OS << " SV_PSTL2STRM = 11,\n";
1469 OS << " SV_PSTL3KEEP = 12,\n";
1470 OS << " SV_PSTL3STRM = 13\n";
1471 OS << "};\n\n";
1472
1473 OS << "/* Function attributes */\n";
1474 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1475 "__nodebug__))\n\n";
1476 OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1477 "__nodebug__, __overloadable__))\n\n";
1478
1479 // Add reinterpret functions.
1480 for (auto [N, Suffix] :
1481 std::initializer_list<std::pair<unsigned, const char *>>{
1482 {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1483 for (auto ShortForm : {false, true})
1484 for (const ReinterpretTypeInfo &To : Reinterprets) {
1485 SVEType ToV(To.BaseType, N);
1486 for (const ReinterpretTypeInfo &From : Reinterprets) {
1487 SVEType FromV(From.BaseType, N);
1488 OS << "__aio "
1489 "__attribute__((__clang_arm_builtin_alias(__builtin_sve_"
1490 "reinterpret_"
1491 << To.Suffix << "_" << From.Suffix << Suffix << ")))\n"
1492 << ToV.str() << " svreinterpret_" << To.Suffix;
1493 if (!ShortForm)
1494 OS << "_" << From.Suffix << Suffix;
1495 OS << "(" << FromV.str() << " op);\n";
1496 }
1497 }
1498 }
1499
1500 createCoreHeaderIntrinsics(OS, Emitter&: *this, Kind: ACLEKind::SVE);
1501
1502 OS << "#define svcvtnt_bf16_x svcvtnt_bf16_m\n";
1503 OS << "#define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m\n";
1504
1505 OS << "#define svcvtnt_f16_x svcvtnt_f16_m\n";
1506 OS << "#define svcvtnt_f16_f32_x svcvtnt_f16_f32_m\n";
1507 OS << "#define svcvtnt_f32_x svcvtnt_f32_m\n";
1508 OS << "#define svcvtnt_f32_f64_x svcvtnt_f32_f64_m\n\n";
1509
1510 OS << "#define svcvtxnt_f32_x svcvtxnt_f32_m\n";
1511 OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n";
1512
1513 OS << "#ifdef __cplusplus\n";
1514 OS << "} // extern \"C\"\n";
1515 OS << "#endif\n\n";
1516 OS << "#undef __ai\n\n";
1517 OS << "#undef __aio\n\n";
1518 OS << "#endif /* __ARM_SVE_H */\n";
1519}
1520
1521void SVEEmitter::createBuiltins(raw_ostream &OS) {
1522 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1523 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1524 for (auto *R : RV)
1525 createIntrinsic(R, Out&: Defs);
1526
1527 // The mappings must be sorted based on BuiltinID.
1528 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1529 const std::unique_ptr<Intrinsic> &B) {
1530 return A->getMangledName() < B->getMangledName();
1531 });
1532
1533 llvm::StringToOffsetTable Table;
1534 Table.GetOrAddStringOffset(Str: "");
1535 Table.GetOrAddStringOffset(Str: "n");
1536
1537 for (const auto &Def : Defs)
1538 if (Def->getClassKind() != ClassG) {
1539 Table.GetOrAddStringOffset(Str: Def->getMangledName());
1540 Table.GetOrAddStringOffset(Str: Def->getBuiltinTypeStr());
1541 Table.GetOrAddStringOffset(Str: Def->getGuard());
1542 }
1543
1544 Table.GetOrAddStringOffset(Str: "sme|sve");
1545 SmallVector<std::pair<std::string, std::string>> ReinterpretBuiltins;
1546 for (auto [N, Suffix] :
1547 std::initializer_list<std::pair<unsigned, const char *>>{
1548 {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
1549 for (const ReinterpretTypeInfo &To : Reinterprets) {
1550 SVEType ToV(To.BaseType, N);
1551 for (const ReinterpretTypeInfo &From : Reinterprets) {
1552 SVEType FromV(From.BaseType, N);
1553 std::string Name =
1554 (Twine("reinterpret_") + To.Suffix + "_" + From.Suffix + Suffix)
1555 .str();
1556 std::string Type = ToV.builtin_str() + FromV.builtin_str();
1557 Table.GetOrAddStringOffset(Str: Name);
1558 Table.GetOrAddStringOffset(Str: Type);
1559 ReinterpretBuiltins.push_back(Elt: {Name, Type});
1560 }
1561 }
1562 }
1563
1564 OS << "#ifdef GET_SVE_BUILTIN_ENUMERATORS\n";
1565 for (const auto &Def : Defs)
1566 if (Def->getClassKind() != ClassG)
1567 OS << " BI__builtin_sve_" << Def->getMangledName() << ",\n";
1568 for (const auto &[Name, _] : ReinterpretBuiltins)
1569 OS << " BI__builtin_sve_" << Name << ",\n";
1570 OS << "#endif // GET_SVE_BUILTIN_ENUMERATORS\n\n";
1571
1572 OS << "#ifdef GET_SVE_BUILTIN_STR_TABLE\n";
1573 Table.EmitStringTableDef(OS, Name: "BuiltinStrings");
1574 OS << "#endif // GET_SVE_BUILTIN_STR_TABLE\n\n";
1575
1576 OS << "#ifdef GET_SVE_BUILTIN_INFOS\n";
1577 for (const auto &Def : Defs) {
1578 // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1579 // declarations only live in the header file.
1580 if (Def->getClassKind() != ClassG) {
1581 OS << " Builtin::Info{Builtin::Info::StrOffsets{"
1582 << Table.GetStringOffset(Str: Def->getMangledName()) << " /* "
1583 << Def->getMangledName() << " */, ";
1584 OS << Table.GetStringOffset(Str: Def->getBuiltinTypeStr()) << " /* "
1585 << Def->getBuiltinTypeStr() << " */, ";
1586 OS << Table.GetStringOffset(Str: "n") << " /* n */, ";
1587 OS << Table.GetStringOffset(Str: Def->getGuard()) << " /* " << Def->getGuard()
1588 << " */}, ";
1589 OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
1590 }
1591 }
1592 for (const auto &[Name, Type] : ReinterpretBuiltins) {
1593 OS << " Builtin::Info{Builtin::Info::StrOffsets{"
1594 << Table.GetStringOffset(Str: Name) << " /* " << Name << " */, ";
1595 OS << Table.GetStringOffset(Str: Type) << " /* " << Type << " */, ";
1596 OS << Table.GetStringOffset(Str: "n") << " /* n */, ";
1597 OS << Table.GetStringOffset(Str: "sme|sve") << " /* sme|sve */}, ";
1598 OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
1599 }
1600 OS << "#endif // GET_SVE_BUILTIN_INFOS\n\n";
1601}
1602
1603void SVEEmitter::createBuiltinsJSON(raw_ostream &OS) {
1604 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1605 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1606 for (auto *R : RV)
1607 createIntrinsic(R, Out&: Defs);
1608
1609 OS << "[\n";
1610 bool FirstDef = true;
1611
1612 for (auto &Def : Defs) {
1613 std::vector<std::string> Flags;
1614
1615 if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsStreaming")))
1616 Flags.push_back(x: "streaming-only");
1617 else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsStreamingCompatible")))
1618 Flags.push_back(x: "streaming-compatible");
1619 else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "VerifyRuntimeMode")))
1620 Flags.push_back(x: "feature-dependent");
1621
1622 if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInZA")) ||
1623 Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsOutZA")) ||
1624 Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInOutZA")))
1625 Flags.push_back(x: "requires-za");
1626
1627 if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInZT0")) ||
1628 Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsOutZT0")) ||
1629 Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInOutZT0")))
1630 Flags.push_back(x: "requires-zt");
1631
1632 if (!FirstDef)
1633 OS << ",\n";
1634
1635 OS << "{ ";
1636 OS << "\"guard\": \"" << Def->getSVEGuard() << "\",";
1637 OS << "\"streaming_guard\": \"" << Def->getSMEGuard() << "\",";
1638 OS << "\"flags\": \"";
1639
1640 for (size_t I = 0; I < Flags.size(); ++I) {
1641 if (I != 0)
1642 OS << ',';
1643 OS << Flags[I];
1644 }
1645
1646 OS << "\",\"builtin\": \"";
1647
1648 std::string BuiltinName = Def->getMangledName(CK: Def->getClassKind());
1649
1650 OS << Def->getReturnType().str() << " " << BuiltinName << "(";
1651 for (unsigned I = 0; I < Def->getTypes().size() - 1; ++I) {
1652 if (I != 0)
1653 OS << ", ";
1654
1655 SVEType ParamType = Def->getParamType(I);
1656
1657 // These are ImmCheck'd but their type names are sufficiently clear.
1658 if (ParamType.isPredicatePattern() || ParamType.isPrefetchOp()) {
1659 OS << ParamType.str();
1660 continue;
1661 }
1662
1663 // Pass ImmCheck information by pretending it's a type.
1664 auto Iter = llvm::find_if(Range: Def->getImmChecks(), P: [I](const auto &Chk) {
1665 return (unsigned)Chk.getImmArgIdx() == I;
1666 });
1667 if (Iter != Def->getImmChecks().end())
1668 OS << getImmCheckForEnumValue(Id: Iter->getKind());
1669 else
1670 OS << ParamType.str();
1671 }
1672 OS << ");\" }";
1673 FirstDef = false;
1674 }
1675
1676 OS << "\n]\n";
1677}
1678
1679void SVEEmitter::createCodeGenMap(raw_ostream &OS) {
1680 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1681 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1682 for (auto *R : RV)
1683 createIntrinsic(R, Out&: Defs);
1684
1685 // The mappings must be sorted based on BuiltinID.
1686 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1687 const std::unique_ptr<Intrinsic> &B) {
1688 return A->getMangledName() < B->getMangledName();
1689 });
1690
1691 OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n";
1692 for (auto &Def : Defs) {
1693 // Builtins only exist for non-overloaded intrinsics, overloaded
1694 // declarations only live in the header file.
1695 if (Def->getClassKind() == ClassG)
1696 continue;
1697
1698 uint64_t Flags = Def->getFlags();
1699 auto FlagString = std::to_string(val: Flags);
1700
1701 std::string LLVMName = Def->getMangledLLVMName();
1702 std::string Builtin = Def->getMangledName();
1703 if (!LLVMName.empty())
1704 OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1705 << "),\n";
1706 else
1707 OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n";
1708 }
1709 OS << "#endif\n\n";
1710}
1711
1712void SVEEmitter::createRangeChecks(raw_ostream &OS) {
1713 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1714 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1715 for (auto *R : RV)
1716 createIntrinsic(R, Out&: Defs);
1717
1718 // The mappings must be sorted based on BuiltinID.
1719 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1720 const std::unique_ptr<Intrinsic> &B) {
1721 return A->getMangledName() < B->getMangledName();
1722 });
1723
1724 OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n";
1725
1726 // Ensure these are only emitted once.
1727 std::set<std::string> Emitted;
1728
1729 for (auto &Def : Defs) {
1730 if (Emitted.find(x: Def->getMangledName()) != Emitted.end() ||
1731 Def->getImmChecks().empty())
1732 continue;
1733
1734 OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n";
1735 for (auto &Check : Def->getImmChecks())
1736 OS << "ImmChecks.emplace_back(" << Check.getImmArgIdx() << ", "
1737 << Check.getKind() << ", " << Check.getElementSizeInBits() << ");\n";
1738 OS << " break;\n";
1739
1740 Emitted.insert(x: Def->getMangledName());
1741 }
1742
1743 OS << "#endif\n\n";
1744}
1745
1746/// Create the SVETypeFlags used in CGBuiltins
1747void SVEEmitter::createTypeFlags(raw_ostream &OS) {
1748 OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n";
1749 for (auto &KV : FlagTypes)
1750 OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n";
1751 OS << "#endif\n\n";
1752
1753 OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n";
1754 for (auto &KV : EltTypes)
1755 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1756 OS << "#endif\n\n";
1757
1758 OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n";
1759 for (auto &KV : MemEltTypes)
1760 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1761 OS << "#endif\n\n";
1762
1763 OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n";
1764 for (auto &KV : MergeTypes)
1765 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1766 OS << "#endif\n\n";
1767}
1768
1769void SVEEmitter::createImmCheckTypes(raw_ostream &OS) {
1770 OS << "#ifdef LLVM_GET_ARM_INTRIN_IMMCHECKTYPES\n";
1771 for (auto &KV : ImmCheckTypes)
1772 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
1773 OS << "#endif\n\n";
1774}
1775
1776void SVEEmitter::createSMEHeader(raw_ostream &OS) {
1777 OS << "/*===---- arm_sme.h - ARM SME intrinsics "
1778 "------===\n"
1779 " *\n"
1780 " *\n"
1781 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
1782 "Exceptions.\n"
1783 " * See https://llvm.org/LICENSE.txt for license information.\n"
1784 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
1785 " *\n"
1786 " *===-----------------------------------------------------------------"
1787 "------===\n"
1788 " */\n\n";
1789
1790 OS << "#ifndef __ARM_SME_H\n";
1791 OS << "#define __ARM_SME_H\n\n";
1792
1793 OS << "#if !defined(__LITTLE_ENDIAN__)\n";
1794 OS << "#error \"Big endian is currently not supported for arm_sme.h\"\n";
1795 OS << "#endif\n";
1796
1797 OS << "#include <arm_sve.h>\n\n";
1798 OS << "#include <stddef.h>\n\n";
1799
1800 OS << "/* Function attributes */\n";
1801 OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
1802 "__nodebug__))\n\n";
1803 OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
1804 "__nodebug__, __overloadable__))\n\n";
1805
1806 OS << "#ifdef __cplusplus\n";
1807 OS << "extern \"C\" {\n";
1808 OS << "#endif\n\n";
1809
1810 OS << "void __arm_za_disable(void) __arm_streaming_compatible;\n\n";
1811
1812 OS << "__ai bool __arm_has_sme(void) __arm_streaming_compatible {\n";
1813 OS << " uint64_t x0, x1;\n";
1814 OS << " __builtin_arm_get_sme_state(&x0, &x1);\n";
1815 OS << " return x0 & (1ULL << 63);\n";
1816 OS << "}\n\n";
1817
1818 OS << "void *__arm_sc_memcpy(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
1819 OS << "void *__arm_sc_memmove(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
1820 OS << "void *__arm_sc_memset(void *s, int c, size_t n) __arm_streaming_compatible;\n";
1821 OS << "void *__arm_sc_memchr(void *s, int c, size_t n) __arm_streaming_compatible;\n\n";
1822
1823 OS << "__ai __attribute__((target(\"sme\"))) void svundef_za(void) "
1824 "__arm_streaming_compatible __arm_out(\"za\") "
1825 "{ }\n\n";
1826
1827 createCoreHeaderIntrinsics(OS, Emitter&: *this, Kind: ACLEKind::SME);
1828
1829 OS << "#ifdef __cplusplus\n";
1830 OS << "} // extern \"C\"\n";
1831 OS << "#endif\n\n";
1832 OS << "#undef __ai\n\n";
1833 OS << "#endif /* __ARM_SME_H */\n";
1834}
1835
1836void SVEEmitter::createSMEBuiltins(raw_ostream &OS) {
1837 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1838 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1839 for (auto *R : RV) {
1840 createIntrinsic(R, Out&: Defs);
1841 }
1842
1843 // The mappings must be sorted based on BuiltinID.
1844 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1845 const std::unique_ptr<Intrinsic> &B) {
1846 return A->getMangledName() < B->getMangledName();
1847 });
1848
1849 llvm::StringToOffsetTable Table;
1850 Table.GetOrAddStringOffset(Str: "");
1851 Table.GetOrAddStringOffset(Str: "n");
1852
1853 for (const auto &Def : Defs)
1854 if (Def->getClassKind() != ClassG) {
1855 Table.GetOrAddStringOffset(Str: Def->getMangledName());
1856 Table.GetOrAddStringOffset(Str: Def->getBuiltinTypeStr());
1857 Table.GetOrAddStringOffset(Str: Def->getGuard());
1858 }
1859
1860 OS << "#ifdef GET_SME_BUILTIN_ENUMERATORS\n";
1861 for (const auto &Def : Defs)
1862 if (Def->getClassKind() != ClassG)
1863 OS << " BI__builtin_sme_" << Def->getMangledName() << ",\n";
1864 OS << "#endif // GET_SME_BUILTIN_ENUMERATORS\n\n";
1865
1866 OS << "#ifdef GET_SME_BUILTIN_STR_TABLE\n";
1867 Table.EmitStringTableDef(OS, Name: "BuiltinStrings");
1868 OS << "#endif // GET_SME_BUILTIN_STR_TABLE\n\n";
1869
1870 OS << "#ifdef GET_SME_BUILTIN_INFOS\n";
1871 for (const auto &Def : Defs) {
1872 // Only create BUILTINs for non-overloaded intrinsics, as overloaded
1873 // declarations only live in the header file.
1874 if (Def->getClassKind() != ClassG) {
1875 OS << " Builtin::Info{Builtin::Info::StrOffsets{"
1876 << Table.GetStringOffset(Str: Def->getMangledName()) << " /* "
1877 << Def->getMangledName() << " */, ";
1878 OS << Table.GetStringOffset(Str: Def->getBuiltinTypeStr()) << " /* "
1879 << Def->getBuiltinTypeStr() << " */, ";
1880 OS << Table.GetStringOffset(Str: "n") << " /* n */, ";
1881 OS << Table.GetStringOffset(Str: Def->getGuard()) << " /* " << Def->getGuard()
1882 << " */}, ";
1883 OS << "HeaderDesc::NO_HEADER, ALL_LANGUAGES},\n";
1884 }
1885 }
1886 OS << "#endif // GET_SME_BUILTIN_INFOS\n\n";
1887}
1888
1889void SVEEmitter::createSMECodeGenMap(raw_ostream &OS) {
1890 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1891 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1892 for (auto *R : RV) {
1893 createIntrinsic(R, Out&: Defs);
1894 }
1895
1896 // The mappings must be sorted based on BuiltinID.
1897 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1898 const std::unique_ptr<Intrinsic> &B) {
1899 return A->getMangledName() < B->getMangledName();
1900 });
1901
1902 OS << "#ifdef GET_SME_LLVM_INTRINSIC_MAP\n";
1903 for (auto &Def : Defs) {
1904 // Builtins only exist for non-overloaded intrinsics, overloaded
1905 // declarations only live in the header file.
1906 if (Def->getClassKind() == ClassG)
1907 continue;
1908
1909 uint64_t Flags = Def->getFlags();
1910 auto FlagString = std::to_string(val: Flags);
1911
1912 std::string LLVMName = Def->getLLVMName();
1913 std::string Builtin = Def->getMangledName();
1914 if (!LLVMName.empty())
1915 OS << "SMEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
1916 << "),\n";
1917 else
1918 OS << "SMEMAP2(" << Builtin << ", " << FlagString << "),\n";
1919 }
1920 OS << "#endif\n\n";
1921}
1922
1923void SVEEmitter::createSMERangeChecks(raw_ostream &OS) {
1924 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1925 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1926 for (auto *R : RV) {
1927 createIntrinsic(R, Out&: Defs);
1928 }
1929
1930 // The mappings must be sorted based on BuiltinID.
1931 sort(C&: Defs, Comp: [](const std::unique_ptr<Intrinsic> &A,
1932 const std::unique_ptr<Intrinsic> &B) {
1933 return A->getMangledName() < B->getMangledName();
1934 });
1935
1936 OS << "#ifdef GET_SME_IMMEDIATE_CHECK\n";
1937
1938 // Ensure these are only emitted once.
1939 std::set<std::string> Emitted;
1940
1941 for (auto &Def : Defs) {
1942 if (Emitted.find(x: Def->getMangledName()) != Emitted.end() ||
1943 Def->getImmChecks().empty())
1944 continue;
1945
1946 OS << "case SME::BI__builtin_sme_" << Def->getMangledName() << ":\n";
1947 for (auto &Check : Def->getImmChecks())
1948 OS << "ImmChecks.push_back(std::make_tuple(" << Check.getImmArgIdx()
1949 << ", " << Check.getKind() << ", " << Check.getElementSizeInBits()
1950 << "));\n";
1951 OS << " break;\n";
1952
1953 Emitted.insert(x: Def->getMangledName());
1954 }
1955
1956 OS << "#endif\n\n";
1957}
1958
1959void SVEEmitter::createBuiltinZAState(raw_ostream &OS) {
1960 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
1961 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
1962 for (auto *R : RV)
1963 createIntrinsic(R, Out&: Defs);
1964
1965 std::map<std::string, std::set<std::string>> IntrinsicsPerState;
1966 for (auto &Def : Defs) {
1967 std::string Key;
1968 auto AddToKey = [&Key](const std::string &S) -> void {
1969 Key = Key.empty() ? S : (Key + " | " + S);
1970 };
1971
1972 if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInZA")))
1973 AddToKey("ArmInZA");
1974 else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsOutZA")))
1975 AddToKey("ArmOutZA");
1976 else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInOutZA")))
1977 AddToKey("ArmInOutZA");
1978
1979 if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInZT0")))
1980 AddToKey("ArmInZT0");
1981 else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsOutZT0")))
1982 AddToKey("ArmOutZT0");
1983 else if (Def->isFlagSet(Flag: getEnumValueForFlag(C: "IsInOutZT0")))
1984 AddToKey("ArmInOutZT0");
1985
1986 if (!Key.empty())
1987 IntrinsicsPerState[Key].insert(x: Def->getMangledName());
1988 }
1989
1990 OS << "#ifdef GET_SME_BUILTIN_GET_STATE\n";
1991 for (auto &KV : IntrinsicsPerState) {
1992 for (StringRef Name : KV.second)
1993 OS << "case SME::BI__builtin_sme_" << Name << ":\n";
1994 OS << " return " << KV.first << ";\n";
1995 }
1996 OS << "#endif\n\n";
1997}
1998
1999void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) {
2000 std::vector<const Record *> RV = Records.getAllDerivedDefinitions(ClassName: "Inst");
2001 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
2002 for (auto *R : RV)
2003 createIntrinsic(R, Out&: Defs);
2004
2005 StringRef ExtensionKind;
2006 switch (Kind) {
2007 case ACLEKind::SME:
2008 ExtensionKind = "SME";
2009 break;
2010 case ACLEKind::SVE:
2011 ExtensionKind = "SVE";
2012 break;
2013 }
2014
2015 OS << "#ifdef GET_" << ExtensionKind << "_STREAMING_ATTRS\n";
2016
2017 StringMap<std::set<std::string>> StreamingMap;
2018
2019 uint64_t IsStreamingFlag = getEnumValueForFlag(C: "IsStreaming");
2020 uint64_t VerifyRuntimeMode = getEnumValueForFlag(C: "VerifyRuntimeMode");
2021 uint64_t IsStreamingCompatibleFlag =
2022 getEnumValueForFlag(C: "IsStreamingCompatible");
2023
2024 for (auto &Def : Defs) {
2025 if (!Def->isFlagSet(Flag: VerifyRuntimeMode) && !Def->getSVEGuard().empty() &&
2026 !Def->getSMEGuard().empty())
2027 report_fatal_error(reason: "Missing VerifyRuntimeMode flag");
2028 if (Def->isFlagSet(Flag: VerifyRuntimeMode) &&
2029 (Def->getSVEGuard().empty() || Def->getSMEGuard().empty()))
2030 report_fatal_error(reason: "VerifyRuntimeMode requires SVE and SME guards");
2031
2032 if (Def->isFlagSet(Flag: IsStreamingFlag))
2033 StreamingMap["ArmStreaming"].insert(x: Def->getMangledName());
2034 else if (Def->isFlagSet(Flag: VerifyRuntimeMode))
2035 StreamingMap["VerifyRuntimeMode"].insert(x: Def->getMangledName());
2036 else if (Def->isFlagSet(Flag: IsStreamingCompatibleFlag))
2037 StreamingMap["ArmStreamingCompatible"].insert(x: Def->getMangledName());
2038 else
2039 StreamingMap["ArmNonStreaming"].insert(x: Def->getMangledName());
2040 }
2041
2042 for (auto BuiltinType : StreamingMap.keys()) {
2043 for (auto Name : StreamingMap[BuiltinType]) {
2044 OS << "case " << ExtensionKind << "::BI__builtin_"
2045 << ExtensionKind.lower() << "_";
2046 OS << Name << ":\n";
2047 }
2048 OS << " BuiltinType = " << BuiltinType << ";\n";
2049 OS << " break;\n";
2050 }
2051
2052 OS << "#endif\n\n";
2053}
2054
2055namespace clang {
2056void EmitSveHeader(const RecordKeeper &Records, raw_ostream &OS) {
2057 SVEEmitter(Records).createHeader(OS);
2058}
2059
2060void EmitSveBuiltins(const RecordKeeper &Records, raw_ostream &OS) {
2061 SVEEmitter(Records).createBuiltins(OS);
2062}
2063
2064void EmitSveBuiltinsJSON(const RecordKeeper &Records, raw_ostream &OS) {
2065 SVEEmitter(Records).createBuiltinsJSON(OS);
2066}
2067
2068void EmitSveBuiltinCG(const RecordKeeper &Records, raw_ostream &OS) {
2069 SVEEmitter(Records).createCodeGenMap(OS);
2070}
2071
2072void EmitSveRangeChecks(const RecordKeeper &Records, raw_ostream &OS) {
2073 SVEEmitter(Records).createRangeChecks(OS);
2074}
2075
2076void EmitSveTypeFlags(const RecordKeeper &Records, raw_ostream &OS) {
2077 SVEEmitter(Records).createTypeFlags(OS);
2078}
2079
2080void EmitImmCheckTypes(const RecordKeeper &Records, raw_ostream &OS) {
2081 SVEEmitter(Records).createImmCheckTypes(OS);
2082}
2083
2084void EmitSveStreamingAttrs(const RecordKeeper &Records, raw_ostream &OS) {
2085 SVEEmitter(Records).createStreamingAttrs(OS, Kind: ACLEKind::SVE);
2086}
2087
2088void EmitSmeHeader(const RecordKeeper &Records, raw_ostream &OS) {
2089 SVEEmitter(Records).createSMEHeader(OS);
2090}
2091
2092void EmitSmeBuiltins(const RecordKeeper &Records, raw_ostream &OS) {
2093 SVEEmitter(Records).createSMEBuiltins(OS);
2094}
2095
2096void EmitSmeBuiltinsJSON(const RecordKeeper &Records, raw_ostream &OS) {
2097 SVEEmitter(Records).createBuiltinsJSON(OS);
2098}
2099
2100void EmitSmeBuiltinCG(const RecordKeeper &Records, raw_ostream &OS) {
2101 SVEEmitter(Records).createSMECodeGenMap(OS);
2102}
2103
2104void EmitSmeRangeChecks(const RecordKeeper &Records, raw_ostream &OS) {
2105 SVEEmitter(Records).createSMERangeChecks(OS);
2106}
2107
2108void EmitSmeStreamingAttrs(const RecordKeeper &Records, raw_ostream &OS) {
2109 SVEEmitter(Records).createStreamingAttrs(OS, Kind: ACLEKind::SME);
2110}
2111
2112void EmitSmeBuiltinZAState(const RecordKeeper &Records, raw_ostream &OS) {
2113 SVEEmitter(Records).createBuiltinZAState(OS);
2114}
2115} // End namespace clang
2116