1//===--- RISCVVIntrinsicUtils.h - RISC-V Vector Intrinsic Utils -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
10#define CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
11
12#include "llvm/ADT/ArrayRef.h"
13#include "llvm/ADT/BitmaskEnum.h"
14#include "llvm/ADT/SmallVector.h"
15#include "llvm/ADT/StringRef.h"
16#include <cstdint>
17#include <optional>
18#include <set>
19#include <string>
20#include <unordered_map>
21#include <vector>
22
23namespace llvm {
24class raw_ostream;
25} // end namespace llvm
26
27namespace clang {
28namespace RISCV {
29
30using VScaleVal = std::optional<unsigned>;
31
32// Modifier for vector type.
33enum class VectorTypeModifier : uint8_t {
34 NoModifier,
35 Widening2XVector,
36 Widening4XVector,
37 Widening8XVector,
38 MaskVector,
39 Log2EEW3,
40 Log2EEW4,
41 Log2EEW5,
42 Log2EEW6,
43 FixedSEW8,
44 FixedSEW16,
45 FixedSEW32,
46 FixedSEW64,
47 LFixedLog2LMULN3,
48 LFixedLog2LMULN2,
49 LFixedLog2LMULN1,
50 LFixedLog2LMUL0,
51 LFixedLog2LMUL1,
52 LFixedLog2LMUL2,
53 LFixedLog2LMUL3,
54 SFixedLog2LMULN3,
55 SFixedLog2LMULN2,
56 SFixedLog2LMULN1,
57 SFixedLog2LMUL0,
58 SFixedLog2LMUL1,
59 SFixedLog2LMUL2,
60 SFixedLog2LMUL3,
61 SEFixedLog2LMULN3,
62 SEFixedLog2LMULN2,
63 SEFixedLog2LMULN1,
64 SEFixedLog2LMUL0,
65 SEFixedLog2LMUL1,
66 SEFixedLog2LMUL2,
67 SEFixedLog2LMUL3,
68 Tuple2,
69 Tuple3,
70 Tuple4,
71 Tuple5,
72 Tuple6,
73 Tuple7,
74 Tuple8,
75};
76
77// Similar to basic type but used to describe what's kind of type related to
78// basic vector type, used to compute type info of arguments.
79enum class BaseTypeModifier : uint8_t {
80 Invalid,
81 Scalar,
82 Vector,
83 Void,
84 SizeT,
85 Ptrdiff,
86 UnsignedLong,
87 SignedLong,
88 Float32
89};
90
91// Modifier for type, used for both scalar and vector types.
92enum class TypeModifier : uint8_t {
93 NoModifier = 0,
94 Pointer = 1 << 0,
95 Const = 1 << 1,
96 Immediate = 1 << 2,
97 UnsignedInteger = 1 << 3,
98 SignedInteger = 1 << 4,
99 Float = 1 << 5,
100 BFloat = 1 << 6,
101 // LMUL1 should be kind of VectorTypeModifier, but that might come with
102 // Widening2XVector for widening reduction.
103 // However that might require VectorTypeModifier become bitmask rather than
104 // simple enum, so we decide keek LMUL1 in TypeModifier for code size
105 // optimization of clang binary size.
106 LMUL1 = 1 << 7,
107 MaxOffset = 7,
108 LLVM_MARK_AS_BITMASK_ENUM(LMUL1),
109};
110
111class Policy {
112public:
113 enum PolicyType {
114 Undisturbed,
115 Agnostic,
116 };
117
118private:
119 // The default assumption for an RVV instruction is TAMA, as an undisturbed
120 // policy generally will affect the performance of an out-of-order core.
121 const PolicyType TailPolicy = Agnostic;
122 const PolicyType MaskPolicy = Agnostic;
123
124public:
125 Policy() = default;
126 Policy(PolicyType TailPolicy) : TailPolicy(TailPolicy) {}
127 Policy(PolicyType TailPolicy, PolicyType MaskPolicy)
128 : TailPolicy(TailPolicy), MaskPolicy(MaskPolicy) {}
129
130 bool isTAMAPolicy() const {
131 return TailPolicy == Agnostic && MaskPolicy == Agnostic;
132 }
133
134 bool isTAMUPolicy() const {
135 return TailPolicy == Agnostic && MaskPolicy == Undisturbed;
136 }
137
138 bool isTUMAPolicy() const {
139 return TailPolicy == Undisturbed && MaskPolicy == Agnostic;
140 }
141
142 bool isTUMUPolicy() const {
143 return TailPolicy == Undisturbed && MaskPolicy == Undisturbed;
144 }
145
146 bool isTAPolicy() const { return TailPolicy == Agnostic; }
147
148 bool isTUPolicy() const { return TailPolicy == Undisturbed; }
149
150 bool isMAPolicy() const { return MaskPolicy == Agnostic; }
151
152 bool isMUPolicy() const { return MaskPolicy == Undisturbed; }
153
154 bool operator==(const Policy &Other) const {
155 return TailPolicy == Other.TailPolicy && MaskPolicy == Other.MaskPolicy;
156 }
157
158 bool operator!=(const Policy &Other) const { return !(*this == Other); }
159
160 bool operator<(const Policy &Other) const {
161 // Just for maintain the old order for quick test.
162 if (MaskPolicy != Other.MaskPolicy)
163 return Other.MaskPolicy < MaskPolicy;
164 return TailPolicy < Other.TailPolicy;
165 }
166};
167
168// PrototypeDescriptor is used to compute type info of arguments or return
169// value.
170struct PrototypeDescriptor {
171 constexpr PrototypeDescriptor() = default;
172 constexpr PrototypeDescriptor(
173 BaseTypeModifier PT,
174 VectorTypeModifier VTM = VectorTypeModifier::NoModifier,
175 TypeModifier TM = TypeModifier::NoModifier)
176 : PT(static_cast<uint8_t>(PT)), VTM(static_cast<uint8_t>(VTM)),
177 TM(static_cast<uint8_t>(TM)) {}
178 constexpr PrototypeDescriptor(uint8_t PT, uint8_t VTM, uint8_t TM)
179 : PT(PT), VTM(VTM), TM(TM) {}
180
181 uint8_t PT = static_cast<uint8_t>(BaseTypeModifier::Invalid);
182 uint8_t VTM = static_cast<uint8_t>(VectorTypeModifier::NoModifier);
183 uint8_t TM = static_cast<uint8_t>(TypeModifier::NoModifier);
184
185 bool operator!=(const PrototypeDescriptor &PD) const {
186 return !(*this == PD);
187 }
188 bool operator==(const PrototypeDescriptor &PD) const {
189 return PD.PT == PT && PD.VTM == VTM && PD.TM == TM;
190 }
191 bool operator<(const PrototypeDescriptor &PD) const {
192 return std::tie(args: PT, args: VTM, args: TM) < std::tie(args: PD.PT, args: PD.VTM, args: PD.TM);
193 }
194 static const PrototypeDescriptor Mask;
195 static const PrototypeDescriptor Vector;
196 static const PrototypeDescriptor VL;
197 static std::optional<PrototypeDescriptor>
198 parsePrototypeDescriptor(llvm::StringRef PrototypeStr);
199};
200
201llvm::SmallVector<PrototypeDescriptor>
202parsePrototypes(llvm::StringRef Prototypes);
203
204// Basic type of vector type.
205enum class BasicType : uint16_t {
206 Unknown = 0,
207 Int8 = 1 << 0,
208 Int16 = 1 << 1,
209 Int32 = 1 << 2,
210 Int64 = 1 << 3,
211 BFloat16 = 1 << 4,
212 Float16 = 1 << 5,
213 Float32 = 1 << 6,
214 Float64 = 1 << 7,
215 F8E4M3 = 1 << 8,
216 F8E5M2 = 1 << 9,
217 MaxOffset = 9,
218 LLVM_MARK_AS_BITMASK_ENUM(F8E5M2),
219};
220
221// Type of vector type.
222enum ScalarTypeKind : uint8_t {
223 Void,
224 Size_t,
225 Ptrdiff_t,
226 UnsignedLong,
227 SignedLong,
228 Boolean,
229 SignedInteger,
230 UnsignedInteger,
231 Float,
232 BFloat,
233 FloatE4M3,
234 FloatE5M2,
235 Invalid,
236 Undefined,
237};
238
239// Exponential LMUL
240struct LMULType {
241 int Log2LMUL;
242 LMULType(int Log2LMUL);
243 // Return the C/C++ string representation of LMUL
244 std::string str() const;
245 std::optional<unsigned> getScale(unsigned ElementBitwidth) const;
246 void MulLog2LMUL(int Log2LMUL);
247};
248
249class RVVType;
250using RVVTypePtr = RVVType *;
251using RVVTypes = std::vector<RVVTypePtr>;
252class RVVTypeCache;
253
254// This class is compact representation of a valid and invalid RVVType.
255class RVVType {
256 friend class RVVTypeCache;
257
258 BasicType BT;
259 ScalarTypeKind ScalarType = Undefined;
260 LMULType LMUL;
261 bool IsPointer = false;
262 // IsConstant indices are "int", but have the constant expression.
263 bool IsImmediate = false;
264 // Const qualifier for pointer to const object or object of const type.
265 bool IsConstant = false;
266 unsigned ElementBitwidth = 0;
267 VScaleVal Scale = 0;
268 bool Valid;
269 bool IsTuple = false;
270 unsigned NF = 0;
271
272 std::string BuiltinStr;
273 std::string ClangBuiltinStr;
274 std::string Str;
275 std::string ShortStr;
276
277 enum class FixedLMULType { LargerThan, SmallerThan, SmallerOrEqual };
278
279 RVVType(BasicType BT, int Log2LMUL, const PrototypeDescriptor &Profile);
280
281public:
282 // Return the string representation of a type, which is an encoded string for
283 // passing to the BUILTIN() macro in Builtins.def.
284 const std::string &getBuiltinStr() const { return BuiltinStr; }
285
286 // Return the clang builtin type for RVV vector type which are used in the
287 // riscv_vector.h header file.
288 const std::string &getClangBuiltinStr() const { return ClangBuiltinStr; }
289
290 // Return the C/C++ string representation of a type for use in the
291 // riscv_vector.h header file.
292 const std::string &getTypeStr() const { return Str; }
293
294 // Return the short name of a type for C/C++ name suffix.
295 const std::string &getShortStr() {
296 // Not all types are used in short name, so compute the short name by
297 // demanded.
298 if (ShortStr.empty())
299 initShortStr();
300 return ShortStr;
301 }
302
303 bool isValid() const { return Valid; }
304 bool isScalar() const { return Scale && *Scale == 0; }
305 bool isVector() const { return Scale && *Scale != 0; }
306 bool isVector(unsigned Width) const {
307 return isVector() && ElementBitwidth == Width;
308 }
309 bool isFloat() const { return ScalarType == ScalarTypeKind::Float; }
310 bool isBFloat() const { return ScalarType == ScalarTypeKind::BFloat; }
311 bool isSignedInteger() const {
312 return ScalarType == ScalarTypeKind::SignedInteger;
313 }
314 bool isFloatVector(unsigned Width) const {
315 return isVector() && isFloat() && ElementBitwidth == Width;
316 }
317 bool isFloat(unsigned Width) const {
318 return isFloat() && ElementBitwidth == Width;
319 }
320 bool isConstant() const { return IsConstant; }
321 bool isPointer() const { return IsPointer; }
322 bool isTuple() const { return IsTuple; }
323 unsigned getElementBitwidth() const { return ElementBitwidth; }
324
325 ScalarTypeKind getScalarType() const { return ScalarType; }
326 VScaleVal getScale() const { return Scale; }
327 unsigned getNF() const {
328 assert(NF > 1 && NF <= 8 && "Only legal NF should be fetched");
329 return NF;
330 }
331
332private:
333 // Verify RVV vector type and set Valid.
334 bool verifyType() const;
335
336 // Creates a type based on basic types of TypeRange
337 void applyBasicType();
338
339 // Applies a prototype modifier to the current type. The result maybe an
340 // invalid type.
341 void applyModifier(const PrototypeDescriptor &prototype);
342
343 void applyLog2EEW(unsigned Log2EEW);
344 void applyFixedSEW(unsigned NewSEW);
345 void applyFixedLog2LMUL(int Log2LMUL, enum FixedLMULType Type);
346
347 // Compute and record a string for legal type.
348 void initBuiltinStr();
349 // Compute and record a builtin RVV vector type string.
350 void initClangBuiltinStr();
351 // Compute and record a type string for used in the header.
352 void initTypeStr();
353 // Compute and record a short name of a type for C/C++ name suffix.
354 void initShortStr();
355};
356
357// This class is used to manage RVVType, RVVType should only created by this
358// class, also provided thread-safe cache capability.
359class RVVTypeCache {
360private:
361 std::unordered_map<uint64_t, RVVType> LegalTypes;
362 std::set<uint64_t> IllegalTypes;
363
364public:
365 /// Compute output and input types by applying different config (basic type
366 /// and LMUL with type transformers). It also record result of type in legal
367 /// or illegal set to avoid compute the same config again. The result maybe
368 /// have illegal RVVType.
369 std::optional<RVVTypes>
370 computeTypes(BasicType BT, int Log2LMUL, unsigned NF,
371 llvm::ArrayRef<PrototypeDescriptor> Prototype);
372 std::optional<RVVTypePtr> computeType(BasicType BT, int Log2LMUL,
373 PrototypeDescriptor Proto);
374};
375
376enum PolicyScheme : uint8_t {
377 SchemeNone,
378 // Passthru operand is at first parameter in C builtin.
379 HasPassthruOperand,
380 HasPolicyOperand,
381};
382
383llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, enum PolicyScheme PS);
384
385// TODO refactor RVVIntrinsic class design after support all intrinsic
386// combination. This represents an instantiation of an intrinsic with a
387// particular type and prototype
388class RVVIntrinsic {
389
390private:
391 std::string BuiltinName; // Builtin name
392 std::string Name; // C intrinsic name.
393 std::string OverloadedName;
394 std::string IRName;
395 bool IsMasked;
396 bool HasMaskedOffOperand;
397 bool HasVL;
398 PolicyScheme Scheme;
399 bool SupportOverloading;
400 bool HasBuiltinAlias;
401 std::string ManualCodegen;
402 RVVTypePtr OutputType; // Builtin output type
403 RVVTypes InputTypes; // Builtin input types
404 // The types we use to obtain the specific LLVM intrinsic. They are index of
405 // InputTypes. -1 means the return type.
406 std::vector<int64_t> IntrinsicTypes;
407 unsigned NF = 1;
408 Policy PolicyAttrs;
409 unsigned TWiden = 0;
410
411public:
412 RVVIntrinsic(llvm::StringRef Name, llvm::StringRef Suffix,
413 llvm::StringRef OverloadedName, llvm::StringRef OverloadedSuffix,
414 llvm::StringRef IRName, bool IsMasked, bool HasMaskedOffOperand,
415 bool HasVL, PolicyScheme Scheme, bool SupportOverloading,
416 bool HasBuiltinAlias, llvm::StringRef ManualCodegen,
417 const RVVTypes &Types,
418 const std::vector<int64_t> &IntrinsicTypes, unsigned NF,
419 Policy PolicyAttrs, bool HasFRMRoundModeOp, unsigned TWiden,
420 bool AltFmt);
421 ~RVVIntrinsic() = default;
422
423 RVVTypePtr getOutputType() const { return OutputType; }
424 const RVVTypes &getInputTypes() const { return InputTypes; }
425 llvm::StringRef getBuiltinName() const { return BuiltinName; }
426 bool hasMaskedOffOperand() const { return HasMaskedOffOperand; }
427 bool hasVL() const { return HasVL; }
428 bool hasPolicy() const { return Scheme != PolicyScheme::SchemeNone; }
429 bool hasPassthruOperand() const {
430 return Scheme == PolicyScheme::HasPassthruOperand;
431 }
432 bool hasPolicyOperand() const {
433 return Scheme == PolicyScheme::HasPolicyOperand;
434 }
435 bool supportOverloading() const { return SupportOverloading; }
436 bool hasBuiltinAlias() const { return HasBuiltinAlias; }
437 bool hasManualCodegen() const { return !ManualCodegen.empty(); }
438 bool isMasked() const { return IsMasked; }
439 llvm::StringRef getOverloadedName() const { return OverloadedName; }
440 llvm::StringRef getIRName() const { return IRName; }
441 llvm::StringRef getManualCodegen() const { return ManualCodegen; }
442 PolicyScheme getPolicyScheme() const { return Scheme; }
443 unsigned getNF() const { return NF; }
444 unsigned getTWiden() const { return TWiden; }
445 const std::vector<int64_t> &getIntrinsicTypes() const {
446 return IntrinsicTypes;
447 }
448 Policy getPolicyAttrs() const {
449 return PolicyAttrs;
450 }
451 unsigned getPolicyAttrsBits() const {
452 // CGBuiltin.cpp
453 // The 0th bit simulates the `vta` of RVV
454 // The 1st bit simulates the `vma` of RVV
455 // int PolicyAttrs = 0;
456
457 if (PolicyAttrs.isTUMAPolicy())
458 return 2;
459 if (PolicyAttrs.isTAMAPolicy())
460 return 3;
461 if (PolicyAttrs.isTUMUPolicy())
462 return 0;
463 if (PolicyAttrs.isTAMUPolicy())
464 return 1;
465
466 llvm_unreachable("unsupport policy");
467 return 0;
468 }
469
470 // Return the type string for a BUILTIN() macro in Builtins.def.
471 std::string getBuiltinTypeStr() const;
472
473 static std::string
474 getSuffixStr(RVVTypeCache &TypeCache, BasicType Type, int Log2LMUL,
475 llvm::ArrayRef<PrototypeDescriptor> PrototypeDescriptors);
476
477 static llvm::SmallVector<PrototypeDescriptor>
478 computeBuiltinTypes(llvm::ArrayRef<PrototypeDescriptor> Prototype,
479 bool IsMasked, bool HasMaskedOffOperand, bool HasVL,
480 unsigned NF, PolicyScheme DefaultScheme,
481 Policy PolicyAttrs, bool IsTuple);
482
483 static llvm::SmallVector<Policy> getSupportedUnMaskedPolicies();
484 static llvm::SmallVector<Policy>
485 getSupportedMaskedPolicies(bool HasTailPolicy, bool HasMaskPolicy);
486
487 static void updateNamesAndPolicy(bool IsMasked, bool HasPolicy,
488 std::string &Name, std::string &BuiltinName,
489 std::string &OverloadedName,
490 Policy &PolicyAttrs, bool HasFRMRoundModeOp,
491 bool AltFmt);
492};
493
494// Raw RVV intrinsic info, used to expand later.
495// This struct is highly compact for minimized code size.
496struct RVVIntrinsicRecord {
497 // Intrinsic name, e.g. vadd_vv
498 const char *Name;
499
500 // Overloaded intrinsic name, could be empty if it can be computed from Name.
501 // e.g. vadd
502 const char *OverloadedName;
503
504 // Required target features for this intrinsic.
505 const char *RequiredExtensions;
506
507 // Prototype for this intrinsic, index of RVVSignatureTable.
508 uint16_t PrototypeIndex;
509
510 // Suffix of intrinsic name, index of RVVSignatureTable.
511 uint16_t SuffixIndex;
512
513 // Suffix of overloaded intrinsic name, index of RVVSignatureTable.
514 uint16_t OverloadedSuffixIndex;
515
516 // Length of the prototype.
517 uint8_t PrototypeLength;
518
519 // Length of intrinsic name suffix.
520 uint8_t SuffixLength;
521
522 // Length of overloaded intrinsic suffix.
523 uint8_t OverloadedSuffixSize;
524
525 // Supported type, mask of BasicType.
526 uint16_t TypeRangeMask;
527
528 // Supported LMUL.
529 uint8_t Log2LMULMask;
530
531 // Number of fields, greater than 1 if it's segment load/store.
532 uint8_t NF;
533
534 bool HasMasked : 1;
535 bool HasVL : 1;
536 bool HasMaskedOffOperand : 1;
537 bool HasTailPolicy : 1;
538 bool HasMaskPolicy : 1;
539 bool HasFRMRoundModeOp : 1;
540 bool AltFmt : 1;
541 bool IsTuple : 1;
542 LLVM_PREFERRED_TYPE(PolicyScheme)
543 uint8_t UnMaskedPolicyScheme : 2;
544 LLVM_PREFERRED_TYPE(PolicyScheme)
545 uint8_t MaskedPolicyScheme : 2;
546};
547
548llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
549 const RVVIntrinsicRecord &RVVInstrRecord);
550
551LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
552} // end namespace RISCV
553
554} // end namespace clang
555
556#endif // CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
557