1//===--- RISCVVIntrinsicUtils.h - RISC-V Vector Intrinsic Utils -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
10#define CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
11
12#include "llvm/ADT/ArrayRef.h"
13#include "llvm/ADT/BitmaskEnum.h"
14#include "llvm/ADT/SmallVector.h"
15#include "llvm/ADT/StringRef.h"
16#include <cstdint>
17#include <optional>
18#include <set>
19#include <string>
20#include <unordered_map>
21#include <vector>
22
23namespace llvm {
24class raw_ostream;
25} // end namespace llvm
26
27namespace clang {
28namespace RISCV {
29
30using VScaleVal = std::optional<unsigned>;
31
32// Modifier for vector type.
33enum class VectorTypeModifier : uint8_t {
34 NoModifier,
35 Widening2XVector,
36 Widening4XVector,
37 Widening8XVector,
38 DoubleLMULVector,
39 MaskVector,
40 Log2EEW3,
41 Log2EEW4,
42 Log2EEW5,
43 Log2EEW6,
44 FixedSEW8,
45 FixedSEW16,
46 FixedSEW32,
47 FixedSEW64,
48 LFixedLog2LMULN3,
49 LFixedLog2LMULN2,
50 LFixedLog2LMULN1,
51 LFixedLog2LMUL0,
52 LFixedLog2LMUL1,
53 LFixedLog2LMUL2,
54 LFixedLog2LMUL3,
55 SFixedLog2LMULN3,
56 SFixedLog2LMULN2,
57 SFixedLog2LMULN1,
58 SFixedLog2LMUL0,
59 SFixedLog2LMUL1,
60 SFixedLog2LMUL2,
61 SFixedLog2LMUL3,
62 SEFixedLog2LMULN3,
63 SEFixedLog2LMULN2,
64 SEFixedLog2LMULN1,
65 SEFixedLog2LMUL0,
66 SEFixedLog2LMUL1,
67 SEFixedLog2LMUL2,
68 SEFixedLog2LMUL3,
69 Tuple2,
70 Tuple3,
71 Tuple4,
72 Tuple5,
73 Tuple6,
74 Tuple7,
75 Tuple8,
76};
77
78// Similar to basic type but used to describe what's kind of type related to
79// basic vector type, used to compute type info of arguments.
80enum class BaseTypeModifier : uint8_t {
81 Invalid,
82 Scalar,
83 Vector,
84 Void,
85 SizeT,
86 Ptrdiff,
87 UnsignedLong,
88 SignedLong,
89 Float32
90};
91
92// Modifier for type, used for both scalar and vector types.
93enum class TypeModifier : uint16_t {
94 NoModifier = 0,
95 Pointer = 1 << 0,
96 Const = 1 << 1,
97 Immediate = 1 << 2,
98 UnsignedInteger = 1 << 3,
99 SignedInteger = 1 << 4,
100 Float = 1 << 5,
101 BFloat = 1 << 6,
102 // LMUL1 should be kind of VectorTypeModifier, but that might come with
103 // Widening2XVector for widening reduction.
104 // However that might require VectorTypeModifier become bitmask rather than
105 // simple enum, so we decide keek LMUL1 in TypeModifier for code size
106 // optimization of clang binary size.
107 LMUL1 = 1 << 7,
108 // Toggle between the two OFP8 element types (FloatE4M3 <-> FloatE5M2).
109 AltFP8 = 1 << 8,
110 MaxOffset = 8,
111 LLVM_MARK_AS_BITMASK_ENUM(AltFP8),
112};
113
114class Policy {
115public:
116 enum PolicyType {
117 Undisturbed,
118 Agnostic,
119 };
120
121private:
122 // The default assumption for an RVV instruction is TAMA, as an undisturbed
123 // policy generally will affect the performance of an out-of-order core.
124 const PolicyType TailPolicy = Agnostic;
125 const PolicyType MaskPolicy = Agnostic;
126
127public:
128 Policy() = default;
129 Policy(PolicyType TailPolicy) : TailPolicy(TailPolicy) {}
130 Policy(PolicyType TailPolicy, PolicyType MaskPolicy)
131 : TailPolicy(TailPolicy), MaskPolicy(MaskPolicy) {}
132
133 bool isTAMAPolicy() const {
134 return TailPolicy == Agnostic && MaskPolicy == Agnostic;
135 }
136
137 bool isTAMUPolicy() const {
138 return TailPolicy == Agnostic && MaskPolicy == Undisturbed;
139 }
140
141 bool isTUMAPolicy() const {
142 return TailPolicy == Undisturbed && MaskPolicy == Agnostic;
143 }
144
145 bool isTUMUPolicy() const {
146 return TailPolicy == Undisturbed && MaskPolicy == Undisturbed;
147 }
148
149 bool isTAPolicy() const { return TailPolicy == Agnostic; }
150
151 bool isTUPolicy() const { return TailPolicy == Undisturbed; }
152
153 bool isMAPolicy() const { return MaskPolicy == Agnostic; }
154
155 bool isMUPolicy() const { return MaskPolicy == Undisturbed; }
156
157 bool operator==(const Policy &Other) const {
158 return TailPolicy == Other.TailPolicy && MaskPolicy == Other.MaskPolicy;
159 }
160
161 bool operator!=(const Policy &Other) const { return !(*this == Other); }
162
163 bool operator<(const Policy &Other) const {
164 // Just for maintain the old order for quick test.
165 if (MaskPolicy != Other.MaskPolicy)
166 return Other.MaskPolicy < MaskPolicy;
167 return TailPolicy < Other.TailPolicy;
168 }
169};
170
171// PrototypeDescriptor is used to compute type info of arguments or return
172// value.
173struct PrototypeDescriptor {
174 constexpr PrototypeDescriptor() = default;
175 constexpr PrototypeDescriptor(
176 BaseTypeModifier PT,
177 VectorTypeModifier VTM = VectorTypeModifier::NoModifier,
178 TypeModifier TM = TypeModifier::NoModifier)
179 : PT(PT), VTM(VTM), TM(TM) {}
180 constexpr PrototypeDescriptor(uint8_t PT, uint8_t VTM, uint16_t TM)
181 : PT(static_cast<BaseTypeModifier>(PT)),
182 VTM(static_cast<VectorTypeModifier>(VTM)),
183 TM(static_cast<TypeModifier>(TM)) {}
184
185 BaseTypeModifier PT = BaseTypeModifier::Invalid;
186 VectorTypeModifier VTM = VectorTypeModifier::NoModifier;
187 TypeModifier TM = TypeModifier::NoModifier;
188
189 bool operator!=(const PrototypeDescriptor &PD) const {
190 return !(*this == PD);
191 }
192 bool operator==(const PrototypeDescriptor &PD) const {
193 return PD.PT == PT && PD.VTM == VTM && PD.TM == TM;
194 }
195 bool operator<(const PrototypeDescriptor &PD) const {
196 return std::tie(args: PT, args: VTM, args: TM) < std::tie(args: PD.PT, args: PD.VTM, args: PD.TM);
197 }
198 static const PrototypeDescriptor Mask;
199 static const PrototypeDescriptor Vector;
200 static const PrototypeDescriptor VL;
201 static std::optional<PrototypeDescriptor>
202 parsePrototypeDescriptor(llvm::StringRef PrototypeStr);
203};
204
205llvm::SmallVector<PrototypeDescriptor>
206parsePrototypes(llvm::StringRef Prototypes);
207
208// Basic type of vector type.
209enum class BasicType : uint16_t {
210 Unknown = 0,
211 Int8 = 1 << 0,
212 Int16 = 1 << 1,
213 Int32 = 1 << 2,
214 Int64 = 1 << 3,
215 BFloat16 = 1 << 4,
216 Float16 = 1 << 5,
217 Float32 = 1 << 6,
218 Float64 = 1 << 7,
219 F8E4M3 = 1 << 8,
220 F8E5M2 = 1 << 9,
221 MaxOffset = 9,
222 LLVM_MARK_AS_BITMASK_ENUM(F8E5M2),
223};
224
225// Type of vector type.
226enum ScalarTypeKind : uint8_t {
227 Void,
228 Size_t,
229 Ptrdiff_t,
230 UnsignedLong,
231 SignedLong,
232 Boolean,
233 SignedInteger,
234 UnsignedInteger,
235 Float,
236 BFloat,
237 FloatE4M3,
238 FloatE5M2,
239 Invalid,
240 Undefined,
241};
242
243// Exponential LMUL
244struct LMULType {
245 int Log2LMUL;
246 LMULType(int Log2LMUL);
247 // Return the C/C++ string representation of LMUL
248 std::string str() const;
249 std::optional<unsigned> getScale(unsigned ElementBitwidth) const;
250 void MulLog2LMUL(int Log2LMUL);
251};
252
253class RVVType;
254using RVVTypePtr = RVVType *;
255using RVVTypes = std::vector<RVVTypePtr>;
256class RVVTypeCache;
257
258// This class is compact representation of a valid and invalid RVVType.
259class RVVType {
260 friend class RVVTypeCache;
261
262 BasicType BT;
263 ScalarTypeKind ScalarType = Undefined;
264 LMULType LMUL;
265 bool IsPointer = false;
266 // IsConstant indices are "int", but have the constant expression.
267 bool IsImmediate = false;
268 // Const qualifier for pointer to const object or object of const type.
269 bool IsConstant = false;
270 unsigned ElementBitwidth = 0;
271 VScaleVal Scale = 0;
272 bool Valid;
273 bool IsTuple = false;
274 unsigned NF = 0;
275
276 std::string BuiltinStr;
277 std::string ClangBuiltinStr;
278 std::string Str;
279 std::string ShortStr;
280
281 enum class FixedLMULType { LargerThan, SmallerThan, SmallerOrEqual };
282
283 RVVType(BasicType BT, int Log2LMUL, const PrototypeDescriptor &Profile);
284
285public:
286 // Return the string representation of a type, which is an encoded string for
287 // passing to the BUILTIN() macro in Builtins.def.
288 const std::string &getBuiltinStr() const { return BuiltinStr; }
289
290 // Return the clang builtin type for RVV vector type which are used in the
291 // riscv_vector.h header file.
292 const std::string &getClangBuiltinStr() const { return ClangBuiltinStr; }
293
294 // Return the C/C++ string representation of a type for use in the
295 // riscv_vector.h header file.
296 const std::string &getTypeStr() const { return Str; }
297
298 // Return the short name of a type for C/C++ name suffix.
299 const std::string &getShortStr() {
300 // Not all types are used in short name, so compute the short name by
301 // demanded.
302 if (ShortStr.empty())
303 initShortStr();
304 return ShortStr;
305 }
306
307 bool isValid() const { return Valid; }
308 bool isScalar() const { return Scale && *Scale == 0; }
309 bool isVector() const { return Scale && *Scale != 0; }
310 bool isVector(unsigned Width) const {
311 return isVector() && ElementBitwidth == Width;
312 }
313 bool isFloat() const { return ScalarType == ScalarTypeKind::Float; }
314 bool isBFloat() const { return ScalarType == ScalarTypeKind::BFloat; }
315 bool isSignedInteger() const {
316 return ScalarType == ScalarTypeKind::SignedInteger;
317 }
318 bool isFloatVector(unsigned Width) const {
319 return isVector() && isFloat() && ElementBitwidth == Width;
320 }
321 bool isFloat(unsigned Width) const {
322 return isFloat() && ElementBitwidth == Width;
323 }
324 bool isConstant() const { return IsConstant; }
325 bool isPointer() const { return IsPointer; }
326 bool isTuple() const { return IsTuple; }
327 unsigned getElementBitwidth() const { return ElementBitwidth; }
328
329 ScalarTypeKind getScalarType() const { return ScalarType; }
330 VScaleVal getScale() const { return Scale; }
331 unsigned getNF() const {
332 assert(NF > 1 && NF <= 8 && "Only legal NF should be fetched");
333 return NF;
334 }
335
336private:
337 // Verify RVV vector type and set Valid.
338 bool verifyType() const;
339
340 // Creates a type based on basic types of TypeRange
341 void applyBasicType();
342
343 // Applies a prototype modifier to the current type. The result maybe an
344 // invalid type.
345 void applyModifier(const PrototypeDescriptor &prototype);
346
347 void applyLog2EEW(unsigned Log2EEW);
348 void applyFixedSEW(unsigned NewSEW);
349 void applyFixedLog2LMUL(int Log2LMUL, enum FixedLMULType Type);
350
351 // Compute and record a string for legal type.
352 void initBuiltinStr();
353 // Compute and record a builtin RVV vector type string.
354 void initClangBuiltinStr();
355 // Compute and record a type string for used in the header.
356 void initTypeStr();
357 // Compute and record a short name of a type for C/C++ name suffix.
358 void initShortStr();
359};
360
361// This class is used to manage RVVType, RVVType should only created by this
362// class, also provided thread-safe cache capability.
363class RVVTypeCache {
364private:
365 std::unordered_map<uint64_t, RVVType> LegalTypes;
366 std::set<uint64_t> IllegalTypes;
367
368public:
369 /// Compute output and input types by applying different config (basic type
370 /// and LMUL with type transformers). It also record result of type in legal
371 /// or illegal set to avoid compute the same config again. The result maybe
372 /// have illegal RVVType.
373 std::optional<RVVTypes>
374 computeTypes(BasicType BT, int Log2LMUL, unsigned NF,
375 llvm::ArrayRef<PrototypeDescriptor> Prototype);
376 std::optional<RVVTypePtr> computeType(BasicType BT, int Log2LMUL,
377 PrototypeDescriptor Proto);
378};
379
380enum PolicyScheme : uint8_t {
381 SchemeNone,
382 // Passthru operand is at first parameter in C builtin.
383 HasPassthruOperand,
384 HasPolicyOperand,
385};
386
387llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, enum PolicyScheme PS);
388
389// TODO refactor RVVIntrinsic class design after support all intrinsic
390// combination. This represents an instantiation of an intrinsic with a
391// particular type and prototype
392class RVVIntrinsic {
393
394private:
395 std::string BuiltinName; // Builtin name
396 std::string Name; // C intrinsic name.
397 std::string OverloadedName;
398 std::string IRName;
399 bool IsMasked;
400 bool HasMaskedOffOperand;
401 bool HasVL;
402 PolicyScheme Scheme;
403 bool SupportOverloading;
404 bool HasBuiltinAlias;
405 std::string ManualCodegen;
406 RVVTypePtr OutputType; // Builtin output type
407 RVVTypes InputTypes; // Builtin input types
408 // The types we use to obtain the specific LLVM intrinsic. They are index of
409 // InputTypes. -1 means the return type.
410 std::vector<int64_t> IntrinsicTypes;
411 unsigned NF = 1;
412 bool HasSegInstSEW = false;
413 Policy PolicyAttrs;
414 unsigned TWiden = 0;
415
416public:
417 RVVIntrinsic(llvm::StringRef Name, llvm::StringRef Suffix,
418 llvm::StringRef OverloadedName, llvm::StringRef OverloadedSuffix,
419 llvm::StringRef IRName, bool IsMasked, bool HasMaskedOffOperand,
420 bool HasVL, PolicyScheme Scheme, bool SupportOverloading,
421 bool HasBuiltinAlias, llvm::StringRef ManualCodegen,
422 const RVVTypes &Types,
423 const std::vector<int64_t> &IntrinsicTypes, unsigned NF,
424 bool HasSegInstSEW, Policy PolicyAttrs, bool HasFRMRoundModeOp,
425 unsigned TWiden, bool AltFmt);
426 ~RVVIntrinsic() = default;
427
428 RVVTypePtr getOutputType() const { return OutputType; }
429 const RVVTypes &getInputTypes() const { return InputTypes; }
430 llvm::StringRef getBuiltinName() const { return BuiltinName; }
431 bool hasMaskedOffOperand() const { return HasMaskedOffOperand; }
432 bool hasVL() const { return HasVL; }
433 bool hasPolicy() const { return Scheme != PolicyScheme::SchemeNone; }
434 bool hasPassthruOperand() const {
435 return Scheme == PolicyScheme::HasPassthruOperand;
436 }
437 bool hasPolicyOperand() const {
438 return Scheme == PolicyScheme::HasPolicyOperand;
439 }
440 bool supportOverloading() const { return SupportOverloading; }
441 bool hasBuiltinAlias() const { return HasBuiltinAlias; }
442 bool hasManualCodegen() const { return !ManualCodegen.empty(); }
443 bool isMasked() const { return IsMasked; }
444 llvm::StringRef getOverloadedName() const { return OverloadedName; }
445 llvm::StringRef getIRName() const { return IRName; }
446 llvm::StringRef getManualCodegen() const { return ManualCodegen; }
447 PolicyScheme getPolicyScheme() const { return Scheme; }
448 unsigned getNF() const { return NF; }
449 bool hasSegInstSEW() const { return HasSegInstSEW; }
450 unsigned getTWiden() const { return TWiden; }
451 const std::vector<int64_t> &getIntrinsicTypes() const {
452 return IntrinsicTypes;
453 }
454 Policy getPolicyAttrs() const {
455 return PolicyAttrs;
456 }
457 unsigned getPolicyAttrsBits() const {
458 // CGBuiltin.cpp
459 // The 0th bit simulates the `vta` of RVV
460 // The 1st bit simulates the `vma` of RVV
461 // int PolicyAttrs = 0;
462
463 if (PolicyAttrs.isTUMAPolicy())
464 return 2;
465 if (PolicyAttrs.isTAMAPolicy())
466 return 3;
467 if (PolicyAttrs.isTUMUPolicy())
468 return 0;
469 if (PolicyAttrs.isTAMUPolicy())
470 return 1;
471
472 llvm_unreachable("unsupport policy");
473 return 0;
474 }
475
476 // Return the type string for a BUILTIN() macro in Builtins.def.
477 std::string getBuiltinTypeStr() const;
478
479 static std::string
480 getSuffixStr(RVVTypeCache &TypeCache, BasicType Type, int Log2LMUL,
481 llvm::ArrayRef<PrototypeDescriptor> PrototypeDescriptors);
482
483 static llvm::SmallVector<PrototypeDescriptor>
484 computeBuiltinTypes(llvm::ArrayRef<PrototypeDescriptor> Prototype,
485 bool IsMasked, bool HasMaskedOffOperand, bool HasVL,
486 unsigned NF, PolicyScheme DefaultScheme,
487 Policy PolicyAttrs, bool IsTuple);
488
489 static llvm::SmallVector<Policy> getSupportedUnMaskedPolicies();
490 static llvm::SmallVector<Policy>
491 getSupportedMaskedPolicies(bool HasTailPolicy, bool HasMaskPolicy);
492
493 static void updateNamesAndPolicy(bool IsMasked, bool HasPolicy,
494 std::string &Name, std::string &BuiltinName,
495 std::string &OverloadedName,
496 Policy &PolicyAttrs, bool HasFRMRoundModeOp,
497 bool AltFmt);
498};
499
500// Raw RVV intrinsic info, used to expand later.
501// This struct is highly compact for minimized code size.
502struct RVVIntrinsicRecord {
503 // Intrinsic name, e.g. vadd_vv
504 const char *Name;
505
506 // Overloaded intrinsic name, could be empty if it can be computed from Name.
507 // e.g. vadd
508 const char *OverloadedName;
509
510 // Required target features for this intrinsic.
511 const char *RequiredExtensions;
512
513 // Prototype for this intrinsic, index of RVVSignatureTable.
514 uint16_t PrototypeIndex;
515
516 // Suffix of intrinsic name, index of RVVSignatureTable.
517 uint16_t SuffixIndex;
518
519 // Suffix of overloaded intrinsic name, index of RVVSignatureTable.
520 uint16_t OverloadedSuffixIndex;
521
522 // Length of the prototype.
523 uint8_t PrototypeLength;
524
525 // Length of intrinsic name suffix.
526 uint8_t SuffixLength;
527
528 // Length of overloaded intrinsic suffix.
529 uint8_t OverloadedSuffixSize;
530
531 // Supported type, mask of BasicType.
532 uint16_t TypeRangeMask;
533
534 // Supported LMUL.
535 uint8_t Log2LMULMask;
536
537 // Number of fields, greater than 1 if it's segment load/store.
538 uint8_t NF;
539
540 bool HasMasked : 1;
541 bool HasVL : 1;
542 bool HasMaskedOffOperand : 1;
543 bool HasTailPolicy : 1;
544 bool HasMaskPolicy : 1;
545 bool HasFRMRoundModeOp : 1;
546 bool AltFmt : 1;
547 bool IsTuple : 1;
548 LLVM_PREFERRED_TYPE(PolicyScheme)
549 uint8_t UnMaskedPolicyScheme : 2;
550 LLVM_PREFERRED_TYPE(PolicyScheme)
551 uint8_t MaskedPolicyScheme : 2;
552};
553
554llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
555 const RVVIntrinsicRecord &RVVInstrRecord);
556
557LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
558} // end namespace RISCV
559
560} // end namespace clang
561
562#endif // CLANG_SUPPORT_RISCVVINTRINSICUTILS_H
563