1//===--- RISCVVIntrinsicUtils.h - RISC-V Vector Intrinsic Utils -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
10#define CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
11
12#include "llvm/ADT/ArrayRef.h"
13#include "llvm/ADT/BitmaskEnum.h"
14#include "llvm/ADT/SmallVector.h"
15#include "llvm/ADT/StringRef.h"
16#include <cstdint>
17#include <optional>
18#include <set>
19#include <string>
20#include <unordered_map>
21#include <vector>
22
23namespace llvm {
24class raw_ostream;
25} // end namespace llvm
26
27namespace clang {
28namespace RISCV {
29
30using VScaleVal = std::optional<unsigned>;
31
32// Modifier for vector type.
33enum class VectorTypeModifier : uint8_t {
34 NoModifier,
35 Widening2XVector,
36 Widening4XVector,
37 Widening8XVector,
38 DoubleLMULVector,
39 MaskVector,
40 DoubleLMULMaskVector,
41 Log2EEW3,
42 Log2EEW4,
43 Log2EEW5,
44 Log2EEW6,
45 FixedSEW8,
46 FixedSEW16,
47 FixedSEW32,
48 FixedSEW64,
49 LFixedLog2LMULN3,
50 LFixedLog2LMULN2,
51 LFixedLog2LMULN1,
52 LFixedLog2LMUL0,
53 LFixedLog2LMUL1,
54 LFixedLog2LMUL2,
55 LFixedLog2LMUL3,
56 SFixedLog2LMULN3,
57 SFixedLog2LMULN2,
58 SFixedLog2LMULN1,
59 SFixedLog2LMUL0,
60 SFixedLog2LMUL1,
61 SFixedLog2LMUL2,
62 SFixedLog2LMUL3,
63 SEFixedLog2LMULN3,
64 SEFixedLog2LMULN2,
65 SEFixedLog2LMULN1,
66 SEFixedLog2LMUL0,
67 SEFixedLog2LMUL1,
68 SEFixedLog2LMUL2,
69 SEFixedLog2LMUL3,
70 Tuple2,
71 Tuple3,
72 Tuple4,
73 Tuple5,
74 Tuple6,
75 Tuple7,
76 Tuple8,
77};
78
79// Similar to basic type but used to describe what's kind of type related to
80// basic vector type, used to compute type info of arguments.
81enum class BaseTypeModifier : uint8_t {
82 Invalid,
83 Scalar,
84 Vector,
85 Void,
86 SizeT,
87 Ptrdiff,
88 UnsignedLong,
89 SignedLong,
90 Float32
91};
92
93// Modifier for type, used for both scalar and vector types.
94enum class TypeModifier : uint16_t {
95 NoModifier = 0,
96 Pointer = 1 << 0,
97 Const = 1 << 1,
98 Immediate = 1 << 2,
99 UnsignedInteger = 1 << 3,
100 SignedInteger = 1 << 4,
101 Float = 1 << 5,
102 BFloat = 1 << 6,
103 // LMUL1 should be kind of VectorTypeModifier, but that might come with
104 // Widening2XVector for widening reduction.
105 // However that might require VectorTypeModifier become bitmask rather than
106 // simple enum, so we decide keek LMUL1 in TypeModifier for code size
107 // optimization of clang binary size.
108 LMUL1 = 1 << 7,
109 // Toggle between the two OFP8 element types (FloatE4M3 <-> FloatE5M2).
110 AltFP8 = 1 << 8,
111 MaxOffset = 8,
112 LLVM_MARK_AS_BITMASK_ENUM(AltFP8),
113};
114
115class Policy {
116public:
117 enum PolicyType {
118 Undisturbed,
119 Agnostic,
120 };
121
122private:
123 // The default assumption for an RVV instruction is TAMA, as an undisturbed
124 // policy generally will affect the performance of an out-of-order core.
125 const PolicyType TailPolicy = Agnostic;
126 const PolicyType MaskPolicy = Agnostic;
127
128public:
129 Policy() = default;
130 Policy(PolicyType TailPolicy) : TailPolicy(TailPolicy) {}
131 Policy(PolicyType TailPolicy, PolicyType MaskPolicy)
132 : TailPolicy(TailPolicy), MaskPolicy(MaskPolicy) {}
133
134 bool isTAMAPolicy() const {
135 return TailPolicy == Agnostic && MaskPolicy == Agnostic;
136 }
137
138 bool isTAMUPolicy() const {
139 return TailPolicy == Agnostic && MaskPolicy == Undisturbed;
140 }
141
142 bool isTUMAPolicy() const {
143 return TailPolicy == Undisturbed && MaskPolicy == Agnostic;
144 }
145
146 bool isTUMUPolicy() const {
147 return TailPolicy == Undisturbed && MaskPolicy == Undisturbed;
148 }
149
150 bool isTAPolicy() const { return TailPolicy == Agnostic; }
151
152 bool isTUPolicy() const { return TailPolicy == Undisturbed; }
153
154 bool isMAPolicy() const { return MaskPolicy == Agnostic; }
155
156 bool isMUPolicy() const { return MaskPolicy == Undisturbed; }
157
158 bool operator==(const Policy &Other) const {
159 return TailPolicy == Other.TailPolicy && MaskPolicy == Other.MaskPolicy;
160 }
161
162 bool operator!=(const Policy &Other) const { return !(*this == Other); }
163
164 bool operator<(const Policy &Other) const {
165 // Just for maintain the old order for quick test.
166 if (MaskPolicy != Other.MaskPolicy)
167 return Other.MaskPolicy < MaskPolicy;
168 return TailPolicy < Other.TailPolicy;
169 }
170};
171
172// PrototypeDescriptor is used to compute type info of arguments or return
173// value.
174struct PrototypeDescriptor {
175 constexpr PrototypeDescriptor() = default;
176 constexpr PrototypeDescriptor(
177 BaseTypeModifier PT,
178 VectorTypeModifier VTM = VectorTypeModifier::NoModifier,
179 TypeModifier TM = TypeModifier::NoModifier)
180 : PT(PT), VTM(VTM), TM(TM) {}
181 constexpr PrototypeDescriptor(uint8_t PT, uint8_t VTM, uint16_t TM)
182 : PT(static_cast<BaseTypeModifier>(PT)),
183 VTM(static_cast<VectorTypeModifier>(VTM)),
184 TM(static_cast<TypeModifier>(TM)) {}
185
186 BaseTypeModifier PT = BaseTypeModifier::Invalid;
187 VectorTypeModifier VTM = VectorTypeModifier::NoModifier;
188 TypeModifier TM = TypeModifier::NoModifier;
189
190 bool operator!=(const PrototypeDescriptor &PD) const {
191 return !(*this == PD);
192 }
193 bool operator==(const PrototypeDescriptor &PD) const {
194 return PD.PT == PT && PD.VTM == VTM && PD.TM == TM;
195 }
196 bool operator<(const PrototypeDescriptor &PD) const {
197 return std::tie(args: PT, args: VTM, args: TM) < std::tie(args: PD.PT, args: PD.VTM, args: PD.TM);
198 }
199 static const PrototypeDescriptor Mask;
200 static const PrototypeDescriptor Vector;
201 static const PrototypeDescriptor VL;
202 static std::optional<PrototypeDescriptor>
203 parsePrototypeDescriptor(llvm::StringRef PrototypeStr);
204};
205
206llvm::SmallVector<PrototypeDescriptor>
207parsePrototypes(llvm::StringRef Prototypes);
208
209// Basic type of vector type.
210enum class BasicType : uint16_t {
211 Unknown = 0,
212 Int8 = 1 << 0,
213 Int16 = 1 << 1,
214 Int32 = 1 << 2,
215 Int64 = 1 << 3,
216 BFloat16 = 1 << 4,
217 Float16 = 1 << 5,
218 Float32 = 1 << 6,
219 Float64 = 1 << 7,
220 F8E4M3 = 1 << 8,
221 F8E5M2 = 1 << 9,
222 MaxOffset = 9,
223 LLVM_MARK_AS_BITMASK_ENUM(F8E5M2),
224};
225
226// Type of vector type.
227enum ScalarTypeKind : uint8_t {
228 Void,
229 Size_t,
230 Ptrdiff_t,
231 UnsignedLong,
232 SignedLong,
233 Boolean,
234 SignedInteger,
235 UnsignedInteger,
236 Float,
237 BFloat,
238 FloatE4M3,
239 FloatE5M2,
240 Invalid,
241 Undefined,
242};
243
244// Exponential LMUL
245struct LMULType {
246 int Log2LMUL;
247 LMULType(int Log2LMUL);
248 // Return the C/C++ string representation of LMUL
249 std::string str() const;
250 std::optional<unsigned> getScale(unsigned ElementBitwidth) const;
251 void MulLog2LMUL(int Log2LMUL);
252};
253
254class RVVType;
255using RVVTypePtr = RVVType *;
256using RVVTypes = std::vector<RVVTypePtr>;
257class RVVTypeCache;
258
259// This class is compact representation of a valid and invalid RVVType.
260class RVVType {
261 friend class RVVTypeCache;
262
263 BasicType BT;
264 ScalarTypeKind ScalarType = Undefined;
265 LMULType LMUL;
266 bool IsPointer = false;
267 // IsConstant indices are "int", but have the constant expression.
268 bool IsImmediate = false;
269 // Const qualifier for pointer to const object or object of const type.
270 bool IsConstant = false;
271 unsigned ElementBitwidth = 0;
272 VScaleVal Scale = 0;
273 bool Valid;
274 bool IsTuple = false;
275 unsigned NF = 0;
276
277 std::string BuiltinStr;
278 std::string ClangBuiltinStr;
279 std::string Str;
280 std::string ShortStr;
281
282 enum class FixedLMULType { LargerThan, SmallerThan, SmallerOrEqual };
283
284 RVVType(BasicType BT, int Log2LMUL, const PrototypeDescriptor &Profile);
285
286public:
287 // Return the string representation of a type, which is an encoded string for
288 // passing to the BUILTIN() macro in Builtins.def.
289 const std::string &getBuiltinStr() const { return BuiltinStr; }
290
291 // Return the clang builtin type for RVV vector type which are used in the
292 // riscv_vector.h header file.
293 const std::string &getClangBuiltinStr() const { return ClangBuiltinStr; }
294
295 // Return the C/C++ string representation of a type for use in the
296 // riscv_vector.h header file.
297 const std::string &getTypeStr() const { return Str; }
298
299 // Return the short name of a type for C/C++ name suffix.
300 const std::string &getShortStr() {
301 // Not all types are used in short name, so compute the short name by
302 // demanded.
303 if (ShortStr.empty())
304 initShortStr();
305 return ShortStr;
306 }
307
308 bool isValid() const { return Valid; }
309 bool isScalar() const { return Scale && *Scale == 0; }
310 bool isVector() const { return Scale && *Scale != 0; }
311 bool isVector(unsigned Width) const {
312 return isVector() && ElementBitwidth == Width;
313 }
314 bool isFloat() const { return ScalarType == ScalarTypeKind::Float; }
315 bool isBFloat() const { return ScalarType == ScalarTypeKind::BFloat; }
316 bool isSignedInteger() const {
317 return ScalarType == ScalarTypeKind::SignedInteger;
318 }
319 bool isFloatVector(unsigned Width) const {
320 return isVector() && isFloat() && ElementBitwidth == Width;
321 }
322 bool isFloat(unsigned Width) const {
323 return isFloat() && ElementBitwidth == Width;
324 }
325 bool isConstant() const { return IsConstant; }
326 bool isPointer() const { return IsPointer; }
327 bool isTuple() const { return IsTuple; }
328 unsigned getElementBitwidth() const { return ElementBitwidth; }
329
330 ScalarTypeKind getScalarType() const { return ScalarType; }
331 VScaleVal getScale() const { return Scale; }
332 unsigned getNF() const {
333 assert(NF > 1 && NF <= 8 && "Only legal NF should be fetched");
334 return NF;
335 }
336
337private:
338 // Verify RVV vector type and set Valid.
339 bool verifyType() const;
340
341 // Creates a type based on basic types of TypeRange
342 void applyBasicType();
343
344 // Applies a prototype modifier to the current type. The result maybe an
345 // invalid type.
346 void applyModifier(const PrototypeDescriptor &prototype);
347
348 void applyLog2EEW(unsigned Log2EEW);
349 void applyFixedSEW(unsigned NewSEW);
350 void applyFixedLog2LMUL(int Log2LMUL, enum FixedLMULType Type);
351
352 // Compute and record a string for legal type.
353 void initBuiltinStr();
354 // Compute and record a builtin RVV vector type string.
355 void initClangBuiltinStr();
356 // Compute and record a type string for used in the header.
357 void initTypeStr();
358 // Compute and record a short name of a type for C/C++ name suffix.
359 void initShortStr();
360};
361
362// This class is used to manage RVVType, RVVType should only created by this
363// class, also provided thread-safe cache capability.
364class RVVTypeCache {
365private:
366 std::unordered_map<uint64_t, RVVType> LegalTypes;
367 std::set<uint64_t> IllegalTypes;
368
369public:
370 /// Compute output and input types by applying different config (basic type
371 /// and LMUL with type transformers). It also record result of type in legal
372 /// or illegal set to avoid compute the same config again. The result maybe
373 /// have illegal RVVType.
374 std::optional<RVVTypes>
375 computeTypes(BasicType BT, int Log2LMUL, unsigned NF,
376 llvm::ArrayRef<PrototypeDescriptor> Prototype);
377 std::optional<RVVTypePtr> computeType(BasicType BT, int Log2LMUL,
378 PrototypeDescriptor Proto);
379};
380
381enum PolicyScheme : uint8_t {
382 SchemeNone,
383 // Passthru operand is at first parameter in C builtin.
384 HasPassthruOperand,
385 HasPolicyOperand,
386};
387
388llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, enum PolicyScheme PS);
389
390// TODO refactor RVVIntrinsic class design after support all intrinsic
391// combination. This represents an instantiation of an intrinsic with a
392// particular type and prototype
393class RVVIntrinsic {
394
395private:
396 std::string BuiltinName; // Builtin name
397 std::string Name; // C intrinsic name.
398 std::string OverloadedName;
399 std::string IRName;
400 bool IsMasked;
401 bool HasMaskedOffOperand;
402 bool HasVL;
403 PolicyScheme Scheme;
404 bool SupportOverloading;
405 bool HasBuiltinAlias;
406 std::string ManualCodegen;
407 RVVTypePtr OutputType; // Builtin output type
408 RVVTypes InputTypes; // Builtin input types
409 // The types we use to obtain the specific LLVM intrinsic. They are index of
410 // InputTypes. -1 means the return type.
411 std::vector<int64_t> IntrinsicTypes;
412 unsigned NF = 1;
413 bool HasSegInstSEW = false;
414 Policy PolicyAttrs;
415 unsigned TWiden = 0;
416
417public:
418 RVVIntrinsic(llvm::StringRef Name, llvm::StringRef Suffix,
419 llvm::StringRef OverloadedName, llvm::StringRef OverloadedSuffix,
420 llvm::StringRef IRName, bool IsMasked, bool HasMaskedOffOperand,
421 bool HasVL, PolicyScheme Scheme, bool SupportOverloading,
422 bool HasBuiltinAlias, llvm::StringRef ManualCodegen,
423 const RVVTypes &Types,
424 const std::vector<int64_t> &IntrinsicTypes, unsigned NF,
425 bool HasSegInstSEW, Policy PolicyAttrs, bool HasFRMRoundModeOp,
426 unsigned TWiden, bool AltFmt);
427 ~RVVIntrinsic() = default;
428
429 RVVTypePtr getOutputType() const { return OutputType; }
430 const RVVTypes &getInputTypes() const { return InputTypes; }
431 llvm::StringRef getBuiltinName() const { return BuiltinName; }
432 bool hasMaskedOffOperand() const { return HasMaskedOffOperand; }
433 bool hasVL() const { return HasVL; }
434 bool hasPolicy() const { return Scheme != PolicyScheme::SchemeNone; }
435 bool hasPassthruOperand() const {
436 return Scheme == PolicyScheme::HasPassthruOperand;
437 }
438 bool hasPolicyOperand() const {
439 return Scheme == PolicyScheme::HasPolicyOperand;
440 }
441 bool supportOverloading() const { return SupportOverloading; }
442 bool hasBuiltinAlias() const { return HasBuiltinAlias; }
443 bool hasManualCodegen() const { return !ManualCodegen.empty(); }
444 bool isMasked() const { return IsMasked; }
445 llvm::StringRef getOverloadedName() const { return OverloadedName; }
446 llvm::StringRef getIRName() const { return IRName; }
447 llvm::StringRef getManualCodegen() const { return ManualCodegen; }
448 PolicyScheme getPolicyScheme() const { return Scheme; }
449 unsigned getNF() const { return NF; }
450 bool hasSegInstSEW() const { return HasSegInstSEW; }
451 unsigned getTWiden() const { return TWiden; }
452 const std::vector<int64_t> &getIntrinsicTypes() const {
453 return IntrinsicTypes;
454 }
455 Policy getPolicyAttrs() const {
456 return PolicyAttrs;
457 }
458 unsigned getPolicyAttrsBits() const {
459 // CGBuiltin.cpp
460 // The 0th bit simulates the `vta` of RVV
461 // The 1st bit simulates the `vma` of RVV
462 // int PolicyAttrs = 0;
463
464 if (PolicyAttrs.isTUMAPolicy())
465 return 2;
466 if (PolicyAttrs.isTAMAPolicy())
467 return 3;
468 if (PolicyAttrs.isTUMUPolicy())
469 return 0;
470 if (PolicyAttrs.isTAMUPolicy())
471 return 1;
472
473 llvm_unreachable("unsupport policy");
474 return 0;
475 }
476
477 // Return the type string for a BUILTIN() macro in Builtins.def.
478 std::string getBuiltinTypeStr() const;
479
480 static std::string
481 getSuffixStr(RVVTypeCache &TypeCache, BasicType Type, int Log2LMUL,
482 llvm::ArrayRef<PrototypeDescriptor> PrototypeDescriptors);
483
484 static llvm::SmallVector<PrototypeDescriptor>
485 computeBuiltinTypes(llvm::ArrayRef<PrototypeDescriptor> Prototype,
486 bool IsMasked, bool HasMaskedOffOperand,
487 bool MaskedPrototypeHasResultMask, bool HasVL,
488 unsigned NF, PolicyScheme DefaultScheme,
489 Policy PolicyAttrs, bool IsTuple);
490
491 static llvm::SmallVector<Policy> getSupportedUnMaskedPolicies();
492 static llvm::SmallVector<Policy>
493 getSupportedMaskedPolicies(bool HasTailPolicy, bool HasMaskPolicy);
494
495 static void updateNamesAndPolicy(bool IsMasked, bool HasPolicy,
496 std::string &Name, std::string &BuiltinName,
497 std::string &OverloadedName,
498 Policy &PolicyAttrs, bool HasFRMRoundModeOp,
499 bool AltFmt);
500};
501
502// Raw RVV intrinsic info, used to expand later.
503// This struct is highly compact for minimized code size.
504struct RVVIntrinsicRecord {
505 // Intrinsic name, e.g. vadd_vv
506 const char *Name;
507
508 // Overloaded intrinsic name, could be empty if it can be computed from Name.
509 // e.g. vadd
510 const char *OverloadedName;
511
512 // Required target features for this intrinsic.
513 const char *RequiredExtensions;
514
515 // Prototype for this intrinsic, index of RVVSignatureTable.
516 uint16_t PrototypeIndex;
517
518 // Suffix of intrinsic name, index of RVVSignatureTable.
519 uint16_t SuffixIndex;
520
521 // Suffix of overloaded intrinsic name, index of RVVSignatureTable.
522 uint16_t OverloadedSuffixIndex;
523
524 // Length of the prototype.
525 uint8_t PrototypeLength;
526
527 // Length of intrinsic name suffix.
528 uint8_t SuffixLength;
529
530 // Length of overloaded intrinsic suffix.
531 uint8_t OverloadedSuffixSize;
532
533 // Supported type, mask of BasicType.
534 uint16_t TypeRangeMask;
535
536 // Supported LMUL.
537 uint8_t Log2LMULMask;
538
539 // Number of fields, greater than 1 if it's segment load/store.
540 uint8_t NF;
541
542 bool HasMasked : 1;
543 bool HasVL : 1;
544 bool HasMaskedOffOperand : 1;
545 bool HasTailPolicy : 1;
546 bool HasMaskPolicy : 1;
547 bool HasFRMRoundModeOp : 1;
548 bool MaskedPrototypeHasResultMask : 1;
549 bool AltFmt : 1;
550 bool IsTuple : 1;
551 LLVM_PREFERRED_TYPE(PolicyScheme)
552 uint8_t UnMaskedPolicyScheme : 2;
553 LLVM_PREFERRED_TYPE(PolicyScheme)
554 uint8_t MaskedPolicyScheme : 2;
555};
556
557llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
558 const RVVIntrinsicRecord &RVVInstrRecord);
559
560LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
561} // end namespace RISCV
562
563} // end namespace clang
564
565#endif // CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
566