1//===--- RISCVVIntrinsicUtils.h - RISC-V Vector Intrinsic Utils -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
10#define CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
11
12#include "llvm/ADT/ArrayRef.h"
13#include "llvm/ADT/BitmaskEnum.h"
14#include "llvm/ADT/SmallVector.h"
15#include "llvm/ADT/StringRef.h"
16#include <cstdint>
17#include <optional>
18#include <set>
19#include <string>
20#include <unordered_map>
21#include <vector>
22
23namespace llvm {
24class raw_ostream;
25} // end namespace llvm
26
27namespace clang {
28namespace RISCV {
29
30using VScaleVal = std::optional<unsigned>;
31
32// Modifier for vector type.
33enum class VectorTypeModifier : uint8_t {
34 NoModifier,
35 Widening2XVector,
36 Widening4XVector,
37 Widening8XVector,
38 MaskVector,
39 Log2EEW3,
40 Log2EEW4,
41 Log2EEW5,
42 Log2EEW6,
43 FixedSEW8,
44 FixedSEW16,
45 FixedSEW32,
46 FixedSEW64,
47 LFixedLog2LMULN3,
48 LFixedLog2LMULN2,
49 LFixedLog2LMULN1,
50 LFixedLog2LMUL0,
51 LFixedLog2LMUL1,
52 LFixedLog2LMUL2,
53 LFixedLog2LMUL3,
54 SFixedLog2LMULN3,
55 SFixedLog2LMULN2,
56 SFixedLog2LMULN1,
57 SFixedLog2LMUL0,
58 SFixedLog2LMUL1,
59 SFixedLog2LMUL2,
60 SFixedLog2LMUL3,
61 SEFixedLog2LMULN3,
62 SEFixedLog2LMULN2,
63 SEFixedLog2LMULN1,
64 SEFixedLog2LMUL0,
65 SEFixedLog2LMUL1,
66 SEFixedLog2LMUL2,
67 SEFixedLog2LMUL3,
68 Tuple2,
69 Tuple3,
70 Tuple4,
71 Tuple5,
72 Tuple6,
73 Tuple7,
74 Tuple8,
75};
76
77// Similar to basic type but used to describe what's kind of type related to
78// basic vector type, used to compute type info of arguments.
79enum class BaseTypeModifier : uint8_t {
80 Invalid,
81 Scalar,
82 Vector,
83 Void,
84 SizeT,
85 Ptrdiff,
86 UnsignedLong,
87 SignedLong,
88 Float32
89};
90
91// Modifier for type, used for both scalar and vector types.
92enum class TypeModifier : uint8_t {
93 NoModifier = 0,
94 Pointer = 1 << 0,
95 Const = 1 << 1,
96 Immediate = 1 << 2,
97 UnsignedInteger = 1 << 3,
98 SignedInteger = 1 << 4,
99 Float = 1 << 5,
100 BFloat = 1 << 6,
101 // LMUL1 should be kind of VectorTypeModifier, but that might come with
102 // Widening2XVector for widening reduction.
103 // However that might require VectorTypeModifier become bitmask rather than
104 // simple enum, so we decide keek LMUL1 in TypeModifier for code size
105 // optimization of clang binary size.
106 LMUL1 = 1 << 7,
107 MaxOffset = 7,
108 LLVM_MARK_AS_BITMASK_ENUM(LMUL1),
109};
110
111class Policy {
112public:
113 enum PolicyType {
114 Undisturbed,
115 Agnostic,
116 };
117
118private:
119 // The default assumption for an RVV instruction is TAMA, as an undisturbed
120 // policy generally will affect the performance of an out-of-order core.
121 const PolicyType TailPolicy = Agnostic;
122 const PolicyType MaskPolicy = Agnostic;
123
124public:
125 Policy() = default;
126 Policy(PolicyType TailPolicy) : TailPolicy(TailPolicy) {}
127 Policy(PolicyType TailPolicy, PolicyType MaskPolicy)
128 : TailPolicy(TailPolicy), MaskPolicy(MaskPolicy) {}
129
130 bool isTAMAPolicy() const {
131 return TailPolicy == Agnostic && MaskPolicy == Agnostic;
132 }
133
134 bool isTAMUPolicy() const {
135 return TailPolicy == Agnostic && MaskPolicy == Undisturbed;
136 }
137
138 bool isTUMAPolicy() const {
139 return TailPolicy == Undisturbed && MaskPolicy == Agnostic;
140 }
141
142 bool isTUMUPolicy() const {
143 return TailPolicy == Undisturbed && MaskPolicy == Undisturbed;
144 }
145
146 bool isTAPolicy() const { return TailPolicy == Agnostic; }
147
148 bool isTUPolicy() const { return TailPolicy == Undisturbed; }
149
150 bool isMAPolicy() const { return MaskPolicy == Agnostic; }
151
152 bool isMUPolicy() const { return MaskPolicy == Undisturbed; }
153
154 bool operator==(const Policy &Other) const {
155 return TailPolicy == Other.TailPolicy && MaskPolicy == Other.MaskPolicy;
156 }
157
158 bool operator!=(const Policy &Other) const { return !(*this == Other); }
159
160 bool operator<(const Policy &Other) const {
161 // Just for maintain the old order for quick test.
162 if (MaskPolicy != Other.MaskPolicy)
163 return Other.MaskPolicy < MaskPolicy;
164 return TailPolicy < Other.TailPolicy;
165 }
166};
167
168// PrototypeDescriptor is used to compute type info of arguments or return
169// value.
170struct PrototypeDescriptor {
171 constexpr PrototypeDescriptor() = default;
172 constexpr PrototypeDescriptor(
173 BaseTypeModifier PT,
174 VectorTypeModifier VTM = VectorTypeModifier::NoModifier,
175 TypeModifier TM = TypeModifier::NoModifier)
176 : PT(PT), VTM(VTM), TM(TM) {}
177 constexpr PrototypeDescriptor(uint8_t PT, uint8_t VTM, uint8_t TM)
178 : PT(static_cast<BaseTypeModifier>(PT)),
179 VTM(static_cast<VectorTypeModifier>(VTM)),
180 TM(static_cast<TypeModifier>(TM)) {}
181
182 BaseTypeModifier PT = BaseTypeModifier::Invalid;
183 VectorTypeModifier VTM = VectorTypeModifier::NoModifier;
184 TypeModifier TM = TypeModifier::NoModifier;
185
186 bool operator!=(const PrototypeDescriptor &PD) const {
187 return !(*this == PD);
188 }
189 bool operator==(const PrototypeDescriptor &PD) const {
190 return PD.PT == PT && PD.VTM == VTM && PD.TM == TM;
191 }
192 bool operator<(const PrototypeDescriptor &PD) const {
193 return std::tie(args: PT, args: VTM, args: TM) < std::tie(args: PD.PT, args: PD.VTM, args: PD.TM);
194 }
195 static const PrototypeDescriptor Mask;
196 static const PrototypeDescriptor Vector;
197 static const PrototypeDescriptor VL;
198 static std::optional<PrototypeDescriptor>
199 parsePrototypeDescriptor(llvm::StringRef PrototypeStr);
200};
201
202llvm::SmallVector<PrototypeDescriptor>
203parsePrototypes(llvm::StringRef Prototypes);
204
205// Basic type of vector type.
206enum class BasicType : uint16_t {
207 Unknown = 0,
208 Int8 = 1 << 0,
209 Int16 = 1 << 1,
210 Int32 = 1 << 2,
211 Int64 = 1 << 3,
212 BFloat16 = 1 << 4,
213 Float16 = 1 << 5,
214 Float32 = 1 << 6,
215 Float64 = 1 << 7,
216 F8E4M3 = 1 << 8,
217 F8E5M2 = 1 << 9,
218 MaxOffset = 9,
219 LLVM_MARK_AS_BITMASK_ENUM(F8E5M2),
220};
221
222// Type of vector type.
223enum ScalarTypeKind : uint8_t {
224 Void,
225 Size_t,
226 Ptrdiff_t,
227 UnsignedLong,
228 SignedLong,
229 Boolean,
230 SignedInteger,
231 UnsignedInteger,
232 Float,
233 BFloat,
234 FloatE4M3,
235 FloatE5M2,
236 Invalid,
237 Undefined,
238};
239
240// Exponential LMUL
241struct LMULType {
242 int Log2LMUL;
243 LMULType(int Log2LMUL);
244 // Return the C/C++ string representation of LMUL
245 std::string str() const;
246 std::optional<unsigned> getScale(unsigned ElementBitwidth) const;
247 void MulLog2LMUL(int Log2LMUL);
248};
249
250class RVVType;
251using RVVTypePtr = RVVType *;
252using RVVTypes = std::vector<RVVTypePtr>;
253class RVVTypeCache;
254
255// This class is compact representation of a valid and invalid RVVType.
256class RVVType {
257 friend class RVVTypeCache;
258
259 BasicType BT;
260 ScalarTypeKind ScalarType = Undefined;
261 LMULType LMUL;
262 bool IsPointer = false;
263 // IsConstant indices are "int", but have the constant expression.
264 bool IsImmediate = false;
265 // Const qualifier for pointer to const object or object of const type.
266 bool IsConstant = false;
267 unsigned ElementBitwidth = 0;
268 VScaleVal Scale = 0;
269 bool Valid;
270 bool IsTuple = false;
271 unsigned NF = 0;
272
273 std::string BuiltinStr;
274 std::string ClangBuiltinStr;
275 std::string Str;
276 std::string ShortStr;
277
278 enum class FixedLMULType { LargerThan, SmallerThan, SmallerOrEqual };
279
280 RVVType(BasicType BT, int Log2LMUL, const PrototypeDescriptor &Profile);
281
282public:
283 // Return the string representation of a type, which is an encoded string for
284 // passing to the BUILTIN() macro in Builtins.def.
285 const std::string &getBuiltinStr() const { return BuiltinStr; }
286
287 // Return the clang builtin type for RVV vector type which are used in the
288 // riscv_vector.h header file.
289 const std::string &getClangBuiltinStr() const { return ClangBuiltinStr; }
290
291 // Return the C/C++ string representation of a type for use in the
292 // riscv_vector.h header file.
293 const std::string &getTypeStr() const { return Str; }
294
295 // Return the short name of a type for C/C++ name suffix.
296 const std::string &getShortStr() {
297 // Not all types are used in short name, so compute the short name by
298 // demanded.
299 if (ShortStr.empty())
300 initShortStr();
301 return ShortStr;
302 }
303
304 bool isValid() const { return Valid; }
305 bool isScalar() const { return Scale && *Scale == 0; }
306 bool isVector() const { return Scale && *Scale != 0; }
307 bool isVector(unsigned Width) const {
308 return isVector() && ElementBitwidth == Width;
309 }
310 bool isFloat() const { return ScalarType == ScalarTypeKind::Float; }
311 bool isBFloat() const { return ScalarType == ScalarTypeKind::BFloat; }
312 bool isSignedInteger() const {
313 return ScalarType == ScalarTypeKind::SignedInteger;
314 }
315 bool isFloatVector(unsigned Width) const {
316 return isVector() && isFloat() && ElementBitwidth == Width;
317 }
318 bool isFloat(unsigned Width) const {
319 return isFloat() && ElementBitwidth == Width;
320 }
321 bool isConstant() const { return IsConstant; }
322 bool isPointer() const { return IsPointer; }
323 bool isTuple() const { return IsTuple; }
324 unsigned getElementBitwidth() const { return ElementBitwidth; }
325
326 ScalarTypeKind getScalarType() const { return ScalarType; }
327 VScaleVal getScale() const { return Scale; }
328 unsigned getNF() const {
329 assert(NF > 1 && NF <= 8 && "Only legal NF should be fetched");
330 return NF;
331 }
332
333private:
334 // Verify RVV vector type and set Valid.
335 bool verifyType() const;
336
337 // Creates a type based on basic types of TypeRange
338 void applyBasicType();
339
340 // Applies a prototype modifier to the current type. The result maybe an
341 // invalid type.
342 void applyModifier(const PrototypeDescriptor &prototype);
343
344 void applyLog2EEW(unsigned Log2EEW);
345 void applyFixedSEW(unsigned NewSEW);
346 void applyFixedLog2LMUL(int Log2LMUL, enum FixedLMULType Type);
347
348 // Compute and record a string for legal type.
349 void initBuiltinStr();
350 // Compute and record a builtin RVV vector type string.
351 void initClangBuiltinStr();
352 // Compute and record a type string for used in the header.
353 void initTypeStr();
354 // Compute and record a short name of a type for C/C++ name suffix.
355 void initShortStr();
356};
357
358// This class is used to manage RVVType, RVVType should only created by this
359// class, also provided thread-safe cache capability.
360class RVVTypeCache {
361private:
362 std::unordered_map<uint64_t, RVVType> LegalTypes;
363 std::set<uint64_t> IllegalTypes;
364
365public:
366 /// Compute output and input types by applying different config (basic type
367 /// and LMUL with type transformers). It also record result of type in legal
368 /// or illegal set to avoid compute the same config again. The result maybe
369 /// have illegal RVVType.
370 std::optional<RVVTypes>
371 computeTypes(BasicType BT, int Log2LMUL, unsigned NF,
372 llvm::ArrayRef<PrototypeDescriptor> Prototype);
373 std::optional<RVVTypePtr> computeType(BasicType BT, int Log2LMUL,
374 PrototypeDescriptor Proto);
375};
376
377enum PolicyScheme : uint8_t {
378 SchemeNone,
379 // Passthru operand is at first parameter in C builtin.
380 HasPassthruOperand,
381 HasPolicyOperand,
382};
383
384llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, enum PolicyScheme PS);
385
386// TODO refactor RVVIntrinsic class design after support all intrinsic
387// combination. This represents an instantiation of an intrinsic with a
388// particular type and prototype
389class RVVIntrinsic {
390
391private:
392 std::string BuiltinName; // Builtin name
393 std::string Name; // C intrinsic name.
394 std::string OverloadedName;
395 std::string IRName;
396 bool IsMasked;
397 bool HasMaskedOffOperand;
398 bool HasVL;
399 PolicyScheme Scheme;
400 bool SupportOverloading;
401 bool HasBuiltinAlias;
402 std::string ManualCodegen;
403 RVVTypePtr OutputType; // Builtin output type
404 RVVTypes InputTypes; // Builtin input types
405 // The types we use to obtain the specific LLVM intrinsic. They are index of
406 // InputTypes. -1 means the return type.
407 std::vector<int64_t> IntrinsicTypes;
408 unsigned NF = 1;
409 Policy PolicyAttrs;
410 unsigned TWiden = 0;
411
412public:
413 RVVIntrinsic(llvm::StringRef Name, llvm::StringRef Suffix,
414 llvm::StringRef OverloadedName, llvm::StringRef OverloadedSuffix,
415 llvm::StringRef IRName, bool IsMasked, bool HasMaskedOffOperand,
416 bool HasVL, PolicyScheme Scheme, bool SupportOverloading,
417 bool HasBuiltinAlias, llvm::StringRef ManualCodegen,
418 const RVVTypes &Types,
419 const std::vector<int64_t> &IntrinsicTypes, unsigned NF,
420 Policy PolicyAttrs, bool HasFRMRoundModeOp, unsigned TWiden,
421 bool AltFmt);
422 ~RVVIntrinsic() = default;
423
424 RVVTypePtr getOutputType() const { return OutputType; }
425 const RVVTypes &getInputTypes() const { return InputTypes; }
426 llvm::StringRef getBuiltinName() const { return BuiltinName; }
427 bool hasMaskedOffOperand() const { return HasMaskedOffOperand; }
428 bool hasVL() const { return HasVL; }
429 bool hasPolicy() const { return Scheme != PolicyScheme::SchemeNone; }
430 bool hasPassthruOperand() const {
431 return Scheme == PolicyScheme::HasPassthruOperand;
432 }
433 bool hasPolicyOperand() const {
434 return Scheme == PolicyScheme::HasPolicyOperand;
435 }
436 bool supportOverloading() const { return SupportOverloading; }
437 bool hasBuiltinAlias() const { return HasBuiltinAlias; }
438 bool hasManualCodegen() const { return !ManualCodegen.empty(); }
439 bool isMasked() const { return IsMasked; }
440 llvm::StringRef getOverloadedName() const { return OverloadedName; }
441 llvm::StringRef getIRName() const { return IRName; }
442 llvm::StringRef getManualCodegen() const { return ManualCodegen; }
443 PolicyScheme getPolicyScheme() const { return Scheme; }
444 unsigned getNF() const { return NF; }
445 unsigned getTWiden() const { return TWiden; }
446 const std::vector<int64_t> &getIntrinsicTypes() const {
447 return IntrinsicTypes;
448 }
449 Policy getPolicyAttrs() const {
450 return PolicyAttrs;
451 }
452 unsigned getPolicyAttrsBits() const {
453 // CGBuiltin.cpp
454 // The 0th bit simulates the `vta` of RVV
455 // The 1st bit simulates the `vma` of RVV
456 // int PolicyAttrs = 0;
457
458 if (PolicyAttrs.isTUMAPolicy())
459 return 2;
460 if (PolicyAttrs.isTAMAPolicy())
461 return 3;
462 if (PolicyAttrs.isTUMUPolicy())
463 return 0;
464 if (PolicyAttrs.isTAMUPolicy())
465 return 1;
466
467 llvm_unreachable("unsupport policy");
468 return 0;
469 }
470
471 // Return the type string for a BUILTIN() macro in Builtins.def.
472 std::string getBuiltinTypeStr() const;
473
474 static std::string
475 getSuffixStr(RVVTypeCache &TypeCache, BasicType Type, int Log2LMUL,
476 llvm::ArrayRef<PrototypeDescriptor> PrototypeDescriptors);
477
478 static llvm::SmallVector<PrototypeDescriptor>
479 computeBuiltinTypes(llvm::ArrayRef<PrototypeDescriptor> Prototype,
480 bool IsMasked, bool HasMaskedOffOperand, bool HasVL,
481 unsigned NF, PolicyScheme DefaultScheme,
482 Policy PolicyAttrs, bool IsTuple);
483
484 static llvm::SmallVector<Policy> getSupportedUnMaskedPolicies();
485 static llvm::SmallVector<Policy>
486 getSupportedMaskedPolicies(bool HasTailPolicy, bool HasMaskPolicy);
487
488 static void updateNamesAndPolicy(bool IsMasked, bool HasPolicy,
489 std::string &Name, std::string &BuiltinName,
490 std::string &OverloadedName,
491 Policy &PolicyAttrs, bool HasFRMRoundModeOp,
492 bool AltFmt);
493};
494
495// Raw RVV intrinsic info, used to expand later.
496// This struct is highly compact for minimized code size.
497struct RVVIntrinsicRecord {
498 // Intrinsic name, e.g. vadd_vv
499 const char *Name;
500
501 // Overloaded intrinsic name, could be empty if it can be computed from Name.
502 // e.g. vadd
503 const char *OverloadedName;
504
505 // Required target features for this intrinsic.
506 const char *RequiredExtensions;
507
508 // Prototype for this intrinsic, index of RVVSignatureTable.
509 uint16_t PrototypeIndex;
510
511 // Suffix of intrinsic name, index of RVVSignatureTable.
512 uint16_t SuffixIndex;
513
514 // Suffix of overloaded intrinsic name, index of RVVSignatureTable.
515 uint16_t OverloadedSuffixIndex;
516
517 // Length of the prototype.
518 uint8_t PrototypeLength;
519
520 // Length of intrinsic name suffix.
521 uint8_t SuffixLength;
522
523 // Length of overloaded intrinsic suffix.
524 uint8_t OverloadedSuffixSize;
525
526 // Supported type, mask of BasicType.
527 uint16_t TypeRangeMask;
528
529 // Supported LMUL.
530 uint8_t Log2LMULMask;
531
532 // Number of fields, greater than 1 if it's segment load/store.
533 uint8_t NF;
534
535 bool HasMasked : 1;
536 bool HasVL : 1;
537 bool HasMaskedOffOperand : 1;
538 bool HasTailPolicy : 1;
539 bool HasMaskPolicy : 1;
540 bool HasFRMRoundModeOp : 1;
541 bool AltFmt : 1;
542 bool IsTuple : 1;
543 LLVM_PREFERRED_TYPE(PolicyScheme)
544 uint8_t UnMaskedPolicyScheme : 2;
545 LLVM_PREFERRED_TYPE(PolicyScheme)
546 uint8_t MaskedPolicyScheme : 2;
547};
548
549llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
550 const RVVIntrinsicRecord &RVVInstrRecord);
551
552LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
553} // end namespace RISCV
554
555} // end namespace clang
556
557#endif // CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
558