1//===- RISCV.cpp ----------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ABIInfoImpl.h"
10#include "TargetInfo.h"
11#include "llvm/IR/IntrinsicsRISCV.h"
12#include "llvm/TargetParser/RISCVTargetParser.h"
13
14using namespace clang;
15using namespace clang::CodeGen;
16
17//===----------------------------------------------------------------------===//
18// RISC-V ABI Implementation
19//===----------------------------------------------------------------------===//
20
21namespace {
22class RISCVABIInfo : public DefaultABIInfo {
23private:
24 // Size of the integer ('x') registers in bits.
25 unsigned XLen;
26 // Size of the floating point ('f') registers in bits. Note that the target
27 // ISA might have a wider FLen than the selected ABI (e.g. an RV32IF target
28 // with soft float ABI has FLen==0).
29 unsigned FLen;
30 const int NumArgGPRs;
31 const int NumArgFPRs;
32 const bool EABI;
33 bool detectFPCCEligibleStructHelper(QualType Ty, CharUnits CurOff,
34 llvm::Type *&Field1Ty,
35 CharUnits &Field1Off,
36 llvm::Type *&Field2Ty,
37 CharUnits &Field2Off) const;
38
39 llvm::Type *detectVLSCCEligibleStruct(QualType Ty, unsigned ABIVLen) const;
40
41 llvm::Type *detectHomogeneousRVVFixedLengthStruct(QualType Ty) const;
42
43public:
44 RISCVABIInfo(CodeGen::CodeGenTypes &CGT, unsigned XLen, unsigned FLen,
45 bool EABI)
46 : DefaultABIInfo(CGT), XLen(XLen), FLen(FLen), NumArgGPRs(EABI ? 6 : 8),
47 NumArgFPRs(FLen != 0 ? 8 : 0), EABI(EABI) {}
48
49 // DefaultABIInfo's classifyReturnType and classifyArgumentType are
50 // non-virtual, but computeInfo is virtual, so we overload it.
51 void computeInfo(CGFunctionInfo &FI) const override;
52
53 ABIArgInfo classifyArgumentType(QualType Ty, bool IsFixed, int &ArgGPRsLeft,
54 int &ArgFPRsLeft, unsigned ABIVLen) const;
55 ABIArgInfo classifyReturnType(QualType RetTy, unsigned ABIVLen) const;
56
57 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
58 AggValueSlot Slot) const override;
59
60 ABIArgInfo extendType(QualType Ty, llvm::Type *CoerceTy = nullptr) const;
61
62 bool detectFPCCEligibleStruct(QualType Ty, llvm::Type *&Field1Ty,
63 CharUnits &Field1Off, llvm::Type *&Field2Ty,
64 CharUnits &Field2Off, int &NeededArgGPRs,
65 int &NeededArgFPRs) const;
66 ABIArgInfo coerceAndExpandFPCCEligibleStruct(llvm::Type *Field1Ty,
67 CharUnits Field1Off,
68 llvm::Type *Field2Ty,
69 CharUnits Field2Off) const;
70
71 ABIArgInfo coerceVLSVector(QualType Ty, unsigned ABIVLen = 0) const;
72 // Some unsupported type e.g. bf16 without zvfbfmin or zvfbfa, should be
73 // passed as same size i8 type. This function check and return the appropriate
74 // fixed vector type.
75 llvm::FixedVectorType *
76 getVLSCCCompatibleType(llvm::FixedVectorType *FixedVecTy) const;
77
78 using ABIInfo::appendAttributeMangling;
79 void appendAttributeMangling(TargetClonesAttr *Attr, unsigned Index,
80 raw_ostream &Out) const override;
81 void appendAttributeMangling(StringRef AttrStr,
82 raw_ostream &Out) const override;
83 llvm::Value *createCoercedLoad(Address SrcAddr, const ABIArgInfo &AI,
84 CodeGenFunction &CGF) const override;
85 void createCoercedStore(llvm::Value *Val, Address DstAddr,
86 const ABIArgInfo &AI, bool DestIsVolatile,
87 CodeGenFunction &CGF) const override;
88};
89} // end anonymous namespace
90
91void RISCVABIInfo::appendAttributeMangling(TargetClonesAttr *Attr,
92 unsigned Index,
93 raw_ostream &Out) const {
94 appendAttributeMangling(AttrStr: Attr->getFeatureStr(Index), Out);
95}
96
97void RISCVABIInfo::appendAttributeMangling(StringRef AttrStr,
98 raw_ostream &Out) const {
99 if (AttrStr == "default") {
100 Out << ".default";
101 return;
102 }
103
104 Out << '.';
105
106 SmallVector<StringRef, 8> Attrs;
107 AttrStr.split(A&: Attrs, Separator: ';');
108
109 // Only consider the arch string.
110 StringRef ArchStr;
111 for (auto &Attr : Attrs) {
112 if (Attr.starts_with(Prefix: "arch="))
113 ArchStr = Attr;
114 }
115
116 // Extract features string.
117 SmallVector<StringRef, 8> Features;
118 ArchStr.consume_front(Prefix: "arch=");
119 ArchStr.split(A&: Features, Separator: ',');
120
121 llvm::stable_sort(Range&: Features);
122
123 for (auto Feat : Features) {
124 Feat.consume_front(Prefix: "+");
125 Out << "_" << Feat;
126 }
127}
128
129void RISCVABIInfo::computeInfo(CGFunctionInfo &FI) const {
130 unsigned ABIVLen;
131 switch (FI.getExtInfo().getCC()) {
132 default:
133 ABIVLen = 0;
134 break;
135#define CC_VLS_CASE(ABI_VLEN) \
136 case CallingConv::CC_RISCVVLSCall_##ABI_VLEN: \
137 ABIVLen = ABI_VLEN; \
138 break;
139 CC_VLS_CASE(32)
140 CC_VLS_CASE(64)
141 CC_VLS_CASE(128)
142 CC_VLS_CASE(256)
143 CC_VLS_CASE(512)
144 CC_VLS_CASE(1024)
145 CC_VLS_CASE(2048)
146 CC_VLS_CASE(4096)
147 CC_VLS_CASE(8192)
148 CC_VLS_CASE(16384)
149 CC_VLS_CASE(32768)
150 CC_VLS_CASE(65536)
151#undef CC_VLS_CASE
152 }
153 QualType RetTy = FI.getReturnType();
154 if (!getCXXABI().classifyReturnType(FI))
155 FI.getReturnInfo() = classifyReturnType(RetTy, ABIVLen);
156
157 // IsRetIndirect is true if classifyArgumentType indicated the value should
158 // be passed indirect, or if the type size is a scalar greater than 2*XLen
159 // and not a complex type with elements <= FLen. e.g. fp128 is passed direct
160 // in LLVM IR, relying on the backend lowering code to rewrite the argument
161 // list and pass indirectly on RV32.
162 bool IsRetIndirect = FI.getReturnInfo().getKind() == ABIArgInfo::Indirect;
163 if (!IsRetIndirect && RetTy->isScalarType() &&
164 getContext().getTypeSize(T: RetTy) > (2 * XLen)) {
165 if (RetTy->isComplexType() && FLen) {
166 QualType EltTy = RetTy->castAs<ComplexType>()->getElementType();
167 IsRetIndirect = getContext().getTypeSize(T: EltTy) > FLen;
168 } else {
169 // This is a normal scalar > 2*XLen, such as fp128 on RV32.
170 IsRetIndirect = true;
171 }
172 }
173
174 int ArgGPRsLeft = IsRetIndirect ? NumArgGPRs - 1 : NumArgGPRs;
175 int ArgFPRsLeft = NumArgFPRs;
176 int NumFixedArgs = FI.getNumRequiredArgs();
177
178 int ArgNum = 0;
179 for (auto &ArgInfo : FI.arguments()) {
180 bool IsFixed = ArgNum < NumFixedArgs;
181 ArgInfo.info = classifyArgumentType(Ty: ArgInfo.type, IsFixed, ArgGPRsLeft,
182 ArgFPRsLeft, ABIVLen);
183 ArgNum++;
184 }
185}
186
187// Returns true if the struct is a potential candidate for the floating point
188// calling convention. If this function returns true, the caller is
189// responsible for checking that if there is only a single field then that
190// field is a float.
191bool RISCVABIInfo::detectFPCCEligibleStructHelper(QualType Ty, CharUnits CurOff,
192 llvm::Type *&Field1Ty,
193 CharUnits &Field1Off,
194 llvm::Type *&Field2Ty,
195 CharUnits &Field2Off) const {
196 bool IsInt = Ty->isIntegralOrEnumerationType();
197 bool IsFloat = Ty->isRealFloatingType();
198
199 if (IsInt || IsFloat) {
200 uint64_t Size = getContext().getTypeSize(T: Ty);
201 if (IsInt && Size > XLen)
202 return false;
203 // Can't be eligible if larger than the FP registers. Handling of half
204 // precision values has been specified in the ABI, so don't block those.
205 if (IsFloat && Size > FLen)
206 return false;
207 // Can't be eligible if an integer type was already found (int+int pairs
208 // are not eligible).
209 if (IsInt && Field1Ty && Field1Ty->isIntegerTy())
210 return false;
211 if (!Field1Ty) {
212 Field1Ty = CGT.ConvertType(T: Ty);
213 Field1Off = CurOff;
214 return true;
215 }
216 if (!Field2Ty) {
217 Field2Ty = CGT.ConvertType(T: Ty);
218 Field2Off = CurOff;
219 return true;
220 }
221 return false;
222 }
223
224 if (auto CTy = Ty->getAs<ComplexType>()) {
225 if (Field1Ty)
226 return false;
227 QualType EltTy = CTy->getElementType();
228 // Only floating-point complex types (e.g. _Complex float/double) are
229 // eligible to be passed in floating-point argument registers. Complex
230 // integer types (a GNU extension) should be treated like a normal
231 // aggregate and packed into GPRs instead.
232 if (!EltTy->isRealFloatingType())
233 return false;
234 if (getContext().getTypeSize(T: EltTy) > FLen)
235 return false;
236 Field1Ty = CGT.ConvertType(T: EltTy);
237 Field1Off = CurOff;
238 Field2Ty = Field1Ty;
239 Field2Off = Field1Off + getContext().getTypeSizeInChars(T: EltTy);
240 return true;
241 }
242
243 if (const ConstantArrayType *ATy = getContext().getAsConstantArrayType(T: Ty)) {
244 uint64_t ArraySize = ATy->getZExtSize();
245 QualType EltTy = ATy->getElementType();
246 // Non-zero-length arrays of empty records make the struct ineligible for
247 // the FP calling convention in C++.
248 if (const auto *RTy = EltTy->getAsCanonical<RecordType>()) {
249 if (ArraySize != 0 && isa<CXXRecordDecl>(Val: RTy->getDecl()) &&
250 isEmptyRecord(Context&: getContext(), T: EltTy, AllowArrays: true, AsIfNoUniqueAddr: true))
251 return false;
252 }
253 CharUnits EltSize = getContext().getTypeSizeInChars(T: EltTy);
254 for (uint64_t i = 0; i < ArraySize; ++i) {
255 bool Ret = detectFPCCEligibleStructHelper(Ty: EltTy, CurOff, Field1Ty,
256 Field1Off, Field2Ty, Field2Off);
257 if (!Ret)
258 return false;
259 CurOff += EltSize;
260 }
261 return true;
262 }
263
264 if (const auto *RTy = Ty->getAsCanonical<RecordType>()) {
265 // Structures with either a non-trivial destructor or a non-trivial
266 // copy constructor are not eligible for the FP calling convention.
267 if (getRecordArgABI(T: Ty, CXXABI&: CGT.getCXXABI()))
268 return false;
269 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true, AsIfNoUniqueAddr: true))
270 return true;
271 const RecordDecl *RD = RTy->getDecl()->getDefinitionOrSelf();
272 // Unions aren't eligible unless they're empty (which is caught above).
273 if (RD->isUnion())
274 return false;
275 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: RD);
276 // If this is a C++ record, check the bases first.
277 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
278 for (const CXXBaseSpecifier &B : CXXRD->bases()) {
279 const auto *BDecl = B.getType()->castAsCXXRecordDecl();
280 CharUnits BaseOff = Layout.getBaseClassOffset(Base: BDecl);
281 bool Ret = detectFPCCEligibleStructHelper(Ty: B.getType(), CurOff: CurOff + BaseOff,
282 Field1Ty, Field1Off, Field2Ty,
283 Field2Off);
284 if (!Ret)
285 return false;
286 }
287 }
288 int ZeroWidthBitFieldCount = 0;
289 for (const FieldDecl *FD : RD->fields()) {
290 uint64_t FieldOffInBits = Layout.getFieldOffset(FieldNo: FD->getFieldIndex());
291 QualType QTy = FD->getType();
292 if (FD->isBitField()) {
293 unsigned BitWidth = FD->getBitWidthValue();
294 // Allow a bitfield with a type greater than XLen as long as the
295 // bitwidth is XLen or less.
296 if (getContext().getTypeSize(T: QTy) > XLen && BitWidth <= XLen)
297 QTy = getContext().getIntTypeForBitwidth(DestWidth: XLen, Signed: false);
298 // Trim type to bitwidth if possible
299 else if (getContext().getTypeSize(T: QTy) > BitWidth) {
300 bool IsSigned =
301 FD->getType().getTypePtr()->hasSignedIntegerRepresentation();
302 unsigned Bits = std::max(a: 8U, b: (unsigned)llvm::PowerOf2Ceil(A: BitWidth));
303 QTy = getContext().getIntTypeForBitwidth(DestWidth: Bits, Signed: IsSigned);
304 }
305 if (BitWidth == 0) {
306 ZeroWidthBitFieldCount++;
307 continue;
308 }
309 }
310
311 bool Ret = detectFPCCEligibleStructHelper(
312 Ty: QTy, CurOff: CurOff + getContext().toCharUnitsFromBits(BitSize: FieldOffInBits),
313 Field1Ty, Field1Off, Field2Ty, Field2Off);
314 if (!Ret)
315 return false;
316
317 // As a quirk of the ABI, zero-width bitfields aren't ignored for fp+fp
318 // or int+fp structs, but are ignored for a struct with an fp field and
319 // any number of zero-width bitfields.
320 if (Field2Ty && ZeroWidthBitFieldCount > 0)
321 return false;
322 }
323 return Field1Ty != nullptr;
324 }
325
326 return false;
327}
328
329// Determine if a struct is eligible for passing according to the floating
330// point calling convention (i.e., when flattened it contains a single fp
331// value, fp+fp, or int+fp of appropriate size). If so, NeededArgFPRs and
332// NeededArgGPRs are incremented appropriately.
333bool RISCVABIInfo::detectFPCCEligibleStruct(QualType Ty, llvm::Type *&Field1Ty,
334 CharUnits &Field1Off,
335 llvm::Type *&Field2Ty,
336 CharUnits &Field2Off,
337 int &NeededArgGPRs,
338 int &NeededArgFPRs) const {
339 Field1Ty = nullptr;
340 Field2Ty = nullptr;
341 NeededArgGPRs = 0;
342 NeededArgFPRs = 0;
343 bool IsCandidate = detectFPCCEligibleStructHelper(
344 Ty, CurOff: CharUnits::Zero(), Field1Ty, Field1Off, Field2Ty, Field2Off);
345 if (!Field1Ty)
346 return false;
347 // Not really a candidate if we have a single int but no float.
348 if (Field1Ty && !Field2Ty && !Field1Ty->isFloatingPointTy())
349 return false;
350 if (!IsCandidate)
351 return false;
352 if (Field1Ty && Field1Ty->isFloatingPointTy())
353 NeededArgFPRs++;
354 else if (Field1Ty)
355 NeededArgGPRs++;
356 if (Field2Ty && Field2Ty->isFloatingPointTy())
357 NeededArgFPRs++;
358 else if (Field2Ty)
359 NeededArgGPRs++;
360 return true;
361}
362
363// Call getCoerceAndExpand for the two-element flattened struct described by
364// Field1Ty, Field1Off, Field2Ty, Field2Off. This method will create an
365// appropriate coerceToType and unpaddedCoerceToType.
366ABIArgInfo RISCVABIInfo::coerceAndExpandFPCCEligibleStruct(
367 llvm::Type *Field1Ty, CharUnits Field1Off, llvm::Type *Field2Ty,
368 CharUnits Field2Off) const {
369 SmallVector<llvm::Type *, 3> CoerceElts;
370 SmallVector<llvm::Type *, 2> UnpaddedCoerceElts;
371 if (!Field1Off.isZero())
372 CoerceElts.push_back(Elt: llvm::ArrayType::get(
373 ElementType: llvm::Type::getInt8Ty(C&: getVMContext()), NumElements: Field1Off.getQuantity()));
374
375 CoerceElts.push_back(Elt: Field1Ty);
376 UnpaddedCoerceElts.push_back(Elt: Field1Ty);
377
378 if (!Field2Ty) {
379 return ABIArgInfo::getCoerceAndExpand(
380 coerceToType: llvm::StructType::get(Context&: getVMContext(), Elements: CoerceElts, isPacked: !Field1Off.isZero()),
381 unpaddedCoerceToType: UnpaddedCoerceElts[0]);
382 }
383
384 CharUnits Field2Align =
385 CharUnits::fromQuantity(Quantity: getDataLayout().getABITypeAlign(Ty: Field2Ty));
386 CharUnits Field1End = Field1Off +
387 CharUnits::fromQuantity(Quantity: getDataLayout().getTypeStoreSize(Ty: Field1Ty));
388 CharUnits Field2OffNoPadNoPack = Field1End.alignTo(Align: Field2Align);
389
390 CharUnits Padding = CharUnits::Zero();
391 if (Field2Off > Field2OffNoPadNoPack)
392 Padding = Field2Off - Field2OffNoPadNoPack;
393 else if (Field2Off != Field2Align && Field2Off > Field1End)
394 Padding = Field2Off - Field1End;
395
396 bool IsPacked = !Field2Off.isMultipleOf(N: Field2Align);
397
398 if (!Padding.isZero())
399 CoerceElts.push_back(Elt: llvm::ArrayType::get(
400 ElementType: llvm::Type::getInt8Ty(C&: getVMContext()), NumElements: Padding.getQuantity()));
401
402 CoerceElts.push_back(Elt: Field2Ty);
403 UnpaddedCoerceElts.push_back(Elt: Field2Ty);
404
405 auto CoerceToType =
406 llvm::StructType::get(Context&: getVMContext(), Elements: CoerceElts, isPacked: IsPacked);
407 auto UnpaddedCoerceToType =
408 llvm::StructType::get(Context&: getVMContext(), Elements: UnpaddedCoerceElts, isPacked: IsPacked);
409
410 return ABIArgInfo::getCoerceAndExpand(coerceToType: CoerceToType, unpaddedCoerceToType: UnpaddedCoerceToType);
411}
412
413llvm::Type *RISCVABIInfo::detectVLSCCEligibleStruct(QualType Ty,
414 unsigned ABIVLen) const {
415 // No riscv_vls_cc attribute.
416 if (ABIVLen == 0)
417 return nullptr;
418
419 // Legal struct for VLS calling convention should fulfill following rules:
420 // 1. Struct element should be either "homogeneous fixed-length vectors" or "a
421 // fixed-length vector array".
422 // 2. Number of struct elements or array elements should be greater or equal
423 // to 1 and less or equal to 8
424 // 3. Total number of vector registers needed should not exceed 8.
425 //
426 // Examples: Assume ABI_VLEN = 128.
427 // These are legal structs:
428 // a. Structs with 1~8 "same" fixed-length vectors, e.g.
429 // struct {
430 // __attribute__((vector_size(16))) int a;
431 // __attribute__((vector_size(16))) int b;
432 // }
433 //
434 // b. Structs with "single" fixed-length vector array with lengh 1~8, e.g.
435 // struct {
436 // __attribute__((vector_size(16))) int a[3];
437 // }
438 // These are illegal structs:
439 // a. Structs with 9 fixed-length vectors, e.g.
440 // struct {
441 // __attribute__((vector_size(16))) int a;
442 // __attribute__((vector_size(16))) int b;
443 // __attribute__((vector_size(16))) int c;
444 // __attribute__((vector_size(16))) int d;
445 // __attribute__((vector_size(16))) int e;
446 // __attribute__((vector_size(16))) int f;
447 // __attribute__((vector_size(16))) int g;
448 // __attribute__((vector_size(16))) int h;
449 // __attribute__((vector_size(16))) int i;
450 // }
451 //
452 // b. Structs with "multiple" fixed-length vector array, e.g.
453 // struct {
454 // __attribute__((vector_size(16))) int a[2];
455 // __attribute__((vector_size(16))) int b[2];
456 // }
457 //
458 // c. Vector registers needed exceeds 8, e.g.
459 // struct {
460 // // Registers needed for single fixed-length element:
461 // // 64 * 8 / ABI_VLEN = 4
462 // __attribute__((vector_size(64))) int a;
463 // __attribute__((vector_size(64))) int b;
464 // __attribute__((vector_size(64))) int c;
465 // __attribute__((vector_size(64))) int d;
466 // }
467 //
468 // 1. Struct of 1 fixed-length vector is passed as a scalable vector.
469 // 2. Struct of >1 fixed-length vectors are passed as vector tuple.
470 // 3. Struct of an array with 1 element of fixed-length vectors is passed as a
471 // scalable vector.
472 // 4. Struct of an array with >1 elements of fixed-length vectors is passed as
473 // vector tuple.
474 // 5. Otherwise, pass the struct indirectly.
475
476 llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: CGT.ConvertType(T: Ty));
477 if (!STy)
478 return nullptr;
479
480 unsigned NumElts = STy->getStructNumElements();
481 if (NumElts > 8)
482 return nullptr;
483
484 auto *FirstEltTy = STy->getElementType(N: 0);
485 if (!STy->containsHomogeneousTypes())
486 return nullptr;
487
488 if (auto *ArrayTy = dyn_cast<llvm::ArrayType>(Val: FirstEltTy)) {
489 // Only struct of single array is accepted
490 if (NumElts != 1)
491 return nullptr;
492 FirstEltTy = ArrayTy->getArrayElementType();
493 NumElts = ArrayTy->getNumElements();
494 }
495
496 auto *FixedVecTy = dyn_cast<llvm::FixedVectorType>(Val: FirstEltTy);
497 if (!FixedVecTy)
498 return nullptr;
499
500 // Check registers needed <= 8.
501 if (NumElts * llvm::divideCeil(
502 Numerator: FixedVecTy->getNumElements() *
503 FixedVecTy->getElementType()->getScalarSizeInBits(),
504 Denominator: ABIVLen) >
505 8)
506 return nullptr;
507
508 // Turn them into scalable vector type or vector tuple type if legal.
509 if (NumElts == 1) {
510 // Handle single fixed-length vector.
511 llvm::FixedVectorType *VLSTy = getVLSCCCompatibleType(FixedVecTy);
512 return llvm::ScalableVectorType::get(
513 ElementType: VLSTy->getElementType(),
514 MinNumElts: llvm::divideCeil(Numerator: VLSTy->getNumElements() * llvm::RISCV::RVVBitsPerBlock,
515 Denominator: ABIVLen));
516 }
517
518 // LMUL
519 // = fixed-length vector size / ABIVLen
520 // = 8 * I8EltCount / RVVBitsPerBlock
521 // =>
522 // I8EltCount
523 // = (fixed-length vector size * RVVBitsPerBlock) / (ABIVLen * 8)
524 unsigned I8EltCount =
525 llvm::divideCeil(Numerator: FixedVecTy->getNumElements() *
526 FixedVecTy->getElementType()->getScalarSizeInBits() *
527 llvm::RISCV::RVVBitsPerBlock,
528 Denominator: ABIVLen * 8);
529 return llvm::TargetExtType::get(
530 Context&: getVMContext(), Name: "riscv.vector.tuple",
531 Types: llvm::ScalableVectorType::get(ElementType: llvm::Type::getInt8Ty(C&: getVMContext()),
532 MinNumElts: I8EltCount),
533 Ints: NumElts);
534}
535
536llvm::Type *
537RISCVABIInfo::detectHomogeneousRVVFixedLengthStruct(QualType Ty) const {
538 const auto *RT = Ty->getAsCanonical<RecordType>();
539 if (!RT)
540 return nullptr;
541
542 const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf();
543 if (RD->isUnion())
544 return nullptr;
545 if (getRecordArgABI(T: Ty, CXXABI&: getCXXABI()))
546 return nullptr;
547
548 // Reject C++ types with base classes.
549 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD))
550 if (CXXRD->getNumBases() != 0)
551 return nullptr;
552
553 SmallVector<const FieldDecl *, 8> Fields(RD->fields());
554
555 if (Fields.empty())
556 return nullptr;
557
558 auto IsFixedLengthRVVVector = [](const VectorType *VT) {
559 switch (VT->getVectorKind()) {
560 case VectorKind::RVVFixedLengthData:
561 case VectorKind::RVVFixedLengthMask:
562 case VectorKind::RVVFixedLengthMask_1:
563 case VectorKind::RVVFixedLengthMask_2:
564 case VectorKind::RVVFixedLengthMask_4:
565 return true;
566 default:
567 return false;
568 }
569 };
570
571 QualType CommonTy;
572 unsigned Count = 0;
573
574 // Single array field: struct { fixed-length RVV T a[N]; }
575 if (Fields.size() == 1) {
576 QualType FieldTy = Fields[0]->getType().getCanonicalType();
577 if (const ConstantArrayType *AT =
578 getContext().getAsConstantArrayType(T: FieldTy)) {
579 QualType EltTy = AT->getElementType().getCanonicalType();
580 if (const auto *VT = EltTy->getAs<VectorType>();
581 VT && IsFixedLengthRVVVector(VT)) {
582 CommonTy = EltTy;
583 Count = AT->getZExtSize();
584 }
585 }
586 }
587
588 // All fields are the same fixed-length RVV vector type (data or mask).
589 if (CommonTy.isNull()) {
590 if (Fields.size() > 8)
591 return nullptr;
592 for (const FieldDecl *FD : Fields) {
593 QualType FieldTy = FD->getType().getCanonicalType();
594 const auto *VT = FieldTy->getAs<VectorType>();
595 if (!VT || !IsFixedLengthRVVVector(VT))
596 return nullptr;
597 if (CommonTy.isNull())
598 CommonTy = FieldTy;
599 else if (!getContext().hasSameType(T1: CommonTy, T2: FieldTy))
600 return nullptr;
601 }
602 Count = Fields.size();
603 }
604
605 if (Count == 0 || Count > 8)
606 return nullptr;
607
608 const auto *VT = CommonTy->castAs<VectorType>();
609 llvm::Type *EltType = CGT.ConvertType(T: VT->getElementType());
610 auto VScale = getContext().getTargetInfo().getVScaleRange(
611 LangOpts: getContext().getLangOpts(), Mode: TargetInfo::ArmStreamingKind::NotStreaming);
612
613 // Ensure total register usage does not exceed 8.
614 if (Count > 1 &&
615 Count * llvm::divideCeil(Numerator: (uint64_t)VT->getNumElements() *
616 EltType->getScalarSizeInBits(),
617 Denominator: VScale->first * llvm::RISCV::RVVBitsPerBlock) >
618 8)
619 return nullptr;
620
621 unsigned MinElts = llvm::divideCeil(Numerator: VT->getNumElements(), Denominator: VScale->first);
622 if (Count == 1)
623 return llvm::ScalableVectorType::get(ElementType: EltType, MinNumElts: MinElts);
624
625 unsigned I8EltCount = llvm::divideCeil(Numerator: (uint64_t)VT->getNumElements() *
626 EltType->getScalarSizeInBits(),
627 Denominator: VScale->first * 8);
628 auto *I8Vec = llvm::ScalableVectorType::get(
629 ElementType: llvm::Type::getInt8Ty(C&: getVMContext()), MinNumElts: I8EltCount);
630 return llvm::TargetExtType::get(Context&: getVMContext(), Name: "riscv.vector.tuple", Types: I8Vec,
631 Ints: Count);
632}
633
634llvm::FixedVectorType *
635RISCVABIInfo::getVLSCCCompatibleType(llvm::FixedVectorType *FixedVecTy) const {
636 llvm::Type *EltType = FixedVecTy->getElementType();
637 const TargetInfo &TI = getContext().getTargetInfo();
638 if ((EltType->isHalfTy() && !TI.hasFeature(Feature: "zvfhmin")) ||
639 (EltType->isBFloatTy() &&
640 !(TI.hasFeature(Feature: "zvfbfmin") || TI.hasFeature(Feature: "experimental-zvfbfa"))) ||
641 (EltType->isFloatTy() && !TI.hasFeature(Feature: "zve32f")) ||
642 (EltType->isDoubleTy() && !TI.hasFeature(Feature: "zve64d")) ||
643 (EltType->isIntegerTy(BitWidth: 64) && !TI.hasFeature(Feature: "zve64x")) ||
644 EltType->isIntegerTy(BitWidth: 128))
645 return llvm::FixedVectorType::get(ElementType: llvm::Type::getInt8Ty(C&: getVMContext()),
646 NumElts: FixedVecTy->getNumElements() *
647 EltType->getScalarSizeInBits() / 8);
648 return FixedVecTy;
649}
650
651// Fixed-length RVV vectors are represented as scalable vectors in function
652// args/return and must be coerced from fixed vectors.
653ABIArgInfo RISCVABIInfo::coerceVLSVector(QualType Ty, unsigned ABIVLen) const {
654 assert(Ty->isVectorType() && "expected vector type!");
655
656 const auto *VT = Ty->castAs<VectorType>();
657 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
658
659 auto VScale = getContext().getTargetInfo().getVScaleRange(
660 LangOpts: getContext().getLangOpts(), Mode: TargetInfo::ArmStreamingKind::NotStreaming);
661
662 unsigned NumElts = VT->getNumElements();
663 llvm::Type *EltType = llvm::Type::getInt1Ty(C&: getVMContext());
664 switch (VT->getVectorKind()) {
665 case VectorKind::RVVFixedLengthMask_1:
666 break;
667 case VectorKind::RVVFixedLengthMask_2:
668 NumElts *= 2;
669 break;
670 case VectorKind::RVVFixedLengthMask_4:
671 NumElts *= 4;
672 break;
673 case VectorKind::RVVFixedLengthMask:
674 NumElts *= 8;
675 break;
676 default:
677 assert((VT->getVectorKind() == VectorKind::Generic ||
678 VT->getVectorKind() == VectorKind::RVVFixedLengthData) &&
679 "Unexpected vector kind");
680 EltType = CGT.ConvertType(T: VT->getElementType());
681 }
682
683 llvm::ScalableVectorType *ResType;
684
685 if (ABIVLen == 0) {
686 // The MinNumElts is simplified from equation:
687 // NumElts / VScale =
688 // (EltSize * NumElts / (VScale * RVVBitsPerBlock))
689 // * (RVVBitsPerBlock / EltSize)
690 ResType = llvm::ScalableVectorType::get(ElementType: EltType, MinNumElts: NumElts / VScale->first);
691 } else {
692 // Check registers needed <= 8.
693 if ((EltType->getScalarSizeInBits() * NumElts / ABIVLen) > 8)
694 return getNaturalAlignIndirect(
695 Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
696 /*ByVal=*/false);
697
698 // Generic vector
699 // The number of elements needs to be at least 1.
700 llvm::FixedVectorType *VLSTy =
701 getVLSCCCompatibleType(FixedVecTy: llvm::FixedVectorType::get(ElementType: EltType, NumElts));
702 ResType = llvm::ScalableVectorType::get(
703 ElementType: VLSTy->getElementType(),
704 MinNumElts: llvm::divideCeil(Numerator: VLSTy->getNumElements() * llvm::RISCV::RVVBitsPerBlock,
705 Denominator: ABIVLen));
706 }
707
708 return ABIArgInfo::getDirect(T: ResType);
709}
710
711ABIArgInfo RISCVABIInfo::classifyArgumentType(QualType Ty, bool IsFixed,
712 int &ArgGPRsLeft,
713 int &ArgFPRsLeft,
714 unsigned ABIVLen) const {
715 assert(ArgGPRsLeft <= NumArgGPRs && "Arg GPR tracking underflow");
716 Ty = useFirstFieldIfTransparentUnion(Ty);
717
718 // Structures with either a non-trivial destructor or a non-trivial
719 // copy constructor are always passed indirectly.
720 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(T: Ty, CXXABI&: getCXXABI())) {
721 if (ArgGPRsLeft)
722 ArgGPRsLeft -= 1;
723 return getNaturalAlignIndirect(
724 Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
725 /*ByVal=*/RAA == CGCXXABI::RAA_DirectInMemory);
726 }
727
728 uint64_t Size = getContext().getTypeSize(T: Ty);
729
730 // Ignore empty structs/unions whose size is zero. According to the calling
731 // convention empty structs/unions are required to be sized types in C++.
732 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true) && Size == 0)
733 return ABIArgInfo::getIgnore();
734
735 // Pass floating point values via FPRs if possible.
736 if (IsFixed && Ty->isFloatingType() && !Ty->isComplexType() &&
737 FLen >= Size && ArgFPRsLeft) {
738 ArgFPRsLeft--;
739 return ABIArgInfo::getDirect();
740 }
741
742 // Complex types for the hard float ABI must be passed direct rather than
743 // using CoerceAndExpand.
744 if (IsFixed && Ty->isComplexType() && FLen && ArgFPRsLeft >= 2) {
745 QualType EltTy = Ty->castAs<ComplexType>()->getElementType();
746 if (getContext().getTypeSize(T: EltTy) <= FLen) {
747 ArgFPRsLeft -= 2;
748 return ABIArgInfo::getDirect();
749 }
750 }
751
752 if (IsFixed && FLen && Ty->isStructureOrClassType()) {
753 llvm::Type *Field1Ty = nullptr;
754 llvm::Type *Field2Ty = nullptr;
755 CharUnits Field1Off = CharUnits::Zero();
756 CharUnits Field2Off = CharUnits::Zero();
757 int NeededArgGPRs = 0;
758 int NeededArgFPRs = 0;
759 bool IsCandidate =
760 detectFPCCEligibleStruct(Ty, Field1Ty, Field1Off, Field2Ty, Field2Off,
761 NeededArgGPRs, NeededArgFPRs);
762 if (IsCandidate && NeededArgGPRs <= ArgGPRsLeft &&
763 NeededArgFPRs <= ArgFPRsLeft) {
764 ArgGPRsLeft -= NeededArgGPRs;
765 ArgFPRsLeft -= NeededArgFPRs;
766 return coerceAndExpandFPCCEligibleStruct(Field1Ty, Field1Off, Field2Ty,
767 Field2Off);
768 }
769 }
770
771 if (IsFixed && Ty->isStructureOrClassType()) {
772 if (llvm::Type *CoerceTy = detectHomogeneousRVVFixedLengthStruct(Ty))
773 return ABIArgInfo::getTargetSpecific(T: CoerceTy);
774
775 if (llvm::Type *VLSType = detectVLSCCEligibleStruct(Ty, ABIVLen))
776 return ABIArgInfo::getTargetSpecific(T: VLSType);
777 }
778
779 uint64_t NeededAlign = getContext().getTypeAlign(T: Ty);
780 // Determine the number of GPRs needed to pass the current argument
781 // according to the ABI. 2*XLen-aligned varargs are passed in "aligned"
782 // register pairs, so may consume 3 registers.
783 // TODO: To be compatible with GCC's behaviors, we don't align registers
784 // currently if we are using ILP32E calling convention. This behavior may be
785 // changed when RV32E/ILP32E is ratified.
786 int NeededArgGPRs = 1;
787 if (!IsFixed && NeededAlign == 2 * XLen)
788 NeededArgGPRs = 2 + (EABI && XLen == 32 ? 0 : (ArgGPRsLeft % 2));
789 else if (Size > XLen && Size <= 2 * XLen)
790 NeededArgGPRs = 2;
791
792 if (NeededArgGPRs > ArgGPRsLeft) {
793 NeededArgGPRs = ArgGPRsLeft;
794 }
795
796 ArgGPRsLeft -= NeededArgGPRs;
797
798 if (!isAggregateTypeForABI(T: Ty) && !Ty->isVectorType()) {
799 // Treat an enum type as its underlying type.
800 if (const auto *ED = Ty->getAsEnumDecl())
801 Ty = ED->getIntegerType();
802
803 if (const auto *EIT = Ty->getAs<BitIntType>()) {
804
805 if (XLen == 64 && EIT->getNumBits() == 32)
806 return extendType(Ty, CoerceTy: CGT.ConvertType(T: Ty));
807
808 if (EIT->getNumBits() <= 2 * XLen)
809 return ABIArgInfo::getExtend(Ty, T: CGT.ConvertType(T: Ty));
810 return getNaturalAlignIndirect(
811 Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
812 /*ByVal=*/false);
813 }
814
815 // All integral types are promoted to XLen width
816 if (Size < XLen && Ty->isIntegralOrEnumerationType())
817 return extendType(Ty, CoerceTy: CGT.ConvertType(T: Ty));
818
819 return ABIArgInfo::getDirect();
820 }
821
822 // TODO: _BitInt is not handled yet in VLS calling convention since _BitInt
823 // ABI is also not merged yet in RISC-V:
824 // https://github.com/riscv-non-isa/riscv-elf-psabi-doc/pull/419
825 if (const VectorType *VT = Ty->getAs<VectorType>();
826 VT && !VT->getElementType()->isBitIntType()) {
827 if (VT->getVectorKind() == VectorKind::RVVFixedLengthData ||
828 VT->getVectorKind() == VectorKind::RVVFixedLengthMask ||
829 VT->getVectorKind() == VectorKind::RVVFixedLengthMask_1 ||
830 VT->getVectorKind() == VectorKind::RVVFixedLengthMask_2 ||
831 VT->getVectorKind() == VectorKind::RVVFixedLengthMask_4)
832 return coerceVLSVector(Ty);
833 if (VT->getVectorKind() == VectorKind::Generic && ABIVLen != 0)
834 // Generic vector without riscv_vls_cc should fall through and pass by
835 // reference.
836 return coerceVLSVector(Ty, ABIVLen);
837 }
838
839 // Aggregates which are <= 2*XLen will be passed in registers if possible,
840 // so coerce to integers.
841 if (Size <= 2 * XLen) {
842 unsigned Alignment = getContext().getTypeAlign(T: Ty);
843
844 if (Size <= XLen) {
845 // Use the smallest integer type we can.
846 return ABIArgInfo::getDirect(
847 T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Size));
848 }
849 // Use 2*XLen if 2*XLen alignment is required.
850 if (Alignment == 2 * XLen)
851 return ABIArgInfo::getDirect(
852 T: llvm::IntegerType::get(C&: getVMContext(), NumBits: 2 * XLen));
853 // Use 2-element XLen array if only XLen alignment is required.
854 return ABIArgInfo::getDirect(
855 T: llvm::ArrayType::get(ElementType: llvm::IntegerType::get(C&: getVMContext(), NumBits: XLen), NumElements: 2));
856 }
857 return getNaturalAlignIndirect(
858 Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
859 /*ByVal=*/false);
860}
861
862ABIArgInfo RISCVABIInfo::classifyReturnType(QualType RetTy,
863 unsigned ABIVLen) const {
864 if (RetTy->isVoidType())
865 return ABIArgInfo::getIgnore();
866
867 int ArgGPRsLeft = 2;
868 int ArgFPRsLeft = FLen ? 2 : 0;
869
870 // The rules for return and argument types are the same, so defer to
871 // classifyArgumentType.
872 return classifyArgumentType(Ty: RetTy, /*IsFixed=*/true, ArgGPRsLeft, ArgFPRsLeft,
873 ABIVLen);
874}
875
876RValue RISCVABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
877 QualType Ty, AggValueSlot Slot) const {
878 CharUnits SlotSize = CharUnits::fromQuantity(Quantity: XLen / 8);
879
880 // Empty records are ignored for parameter passing purposes.
881 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
882 return Slot.asRValue();
883
884 auto TInfo = getContext().getTypeInfoInChars(T: Ty);
885
886 // TODO: To be compatible with GCC's behaviors, we force arguments with
887 // 2×XLEN-bit alignment and size at most 2×XLEN bits like `long long`,
888 // `unsigned long long` and `double` to have 4-byte alignment. This
889 // behavior may be changed when RV32E/ILP32E is ratified.
890 if (EABI && XLen == 32)
891 TInfo.Align = std::min(a: TInfo.Align, b: CharUnits::fromQuantity(Quantity: 4));
892
893 // Arguments bigger than 2*Xlen bytes are passed indirectly.
894 bool IsIndirect = TInfo.Width > 2 * SlotSize;
895
896 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect, ValueInfo: TInfo, SlotSizeAndAlign: SlotSize,
897 /*AllowHigherAlign=*/true, Slot);
898}
899
900ABIArgInfo RISCVABIInfo::extendType(QualType Ty, llvm::Type *CoerceTy) const {
901 int TySize = getContext().getTypeSize(T: Ty);
902 // RV64 ABI requires unsigned 32 bit integers to be sign extended.
903 if (XLen == 64 && Ty->isUnsignedIntegerOrEnumerationType() && TySize == 32)
904 return ABIArgInfo::getSignExtend(Ty, T: CoerceTy);
905 return ABIArgInfo::getExtend(Ty, T: CoerceTy);
906}
907
908llvm::Value *RISCVABIInfo::createCoercedLoad(Address Src, const ABIArgInfo &AI,
909 CodeGenFunction &CGF) const {
910 llvm::Type *Ty = AI.getCoerceToType();
911 llvm::Type *SrcTy = Src.getElementType();
912 llvm::StructType *SrcSTy = cast<llvm::StructType>(Val: SrcTy);
913 assert((Ty->isScalableTy() || Ty->isTargetExtTy()) &&
914 "Only scalable vector type and vector tuple type are allowed for load "
915 "type.");
916 if (llvm::TargetExtType *TupTy = dyn_cast<llvm::TargetExtType>(Val: Ty)) {
917 // In RISC-V VLS calling convention, struct of fixed vectors or struct of
918 // array of fixed vector of length >1 might be lowered using vector tuple
919 // type, we consider it as a valid load, e.g.
920 // struct i32x4x2 {
921 // __attribute__((vector_size(16))) int i;
922 // __attribute__((vector_size(16))) int i;
923 // };
924 // or
925 // struct i32x4 {
926 // __attribute__((vector_size(16))) int i[2];
927 // };
928 // is lowered to target("riscv.vector.tuple", <vscale x 8 x i8>, 2)
929 // when ABI_VLEN = 128 bits, please checkout
930 // clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
931 // for more information.
932 assert(TupTy->getName() == "riscv.vector.tuple");
933 llvm::Type *EltTy = TupTy->getTypeParameter(i: 0);
934 unsigned NumElts = TupTy->getIntParameter(i: 0);
935
936 if (auto *ArrayTy = dyn_cast<llvm::ArrayType>(Val: SrcSTy->getElementType(N: 0)))
937 Src = Src.withElementType(ElemTy: ArrayTy);
938
939 // Perform extract element and load
940 llvm::Value *TupleVal = llvm::PoisonValue::get(T: Ty);
941 auto *Load = CGF.Builder.CreateLoad(Addr: Src);
942 for (unsigned i = 0; i < NumElts; ++i) {
943 // Extract from struct
944 llvm::Value *ExtractFromLoad = CGF.Builder.CreateExtractValue(Agg: Load, Idxs: i);
945 auto *FixedVecTy =
946 cast<llvm::FixedVectorType>(Val: ExtractFromLoad->getType());
947 llvm::FixedVectorType *VLSTy = getVLSCCCompatibleType(FixedVecTy);
948 if (VLSTy != FixedVecTy)
949 ExtractFromLoad = CGF.Builder.CreateBitCast(V: ExtractFromLoad, DestTy: VLSTy);
950 // Element in vector tuple type is always i8, so we need to cast back to
951 // it's original element type.
952 EltTy =
953 cast<llvm::ScalableVectorType>(Val: llvm::VectorType::getWithSizeAndScalar(
954 SizeTy: cast<llvm::VectorType>(Val: EltTy), EltTy: VLSTy));
955 llvm::Value *VectorVal = llvm::PoisonValue::get(T: EltTy);
956 // Insert to scalable vector
957 VectorVal = CGF.Builder.CreateInsertVector(
958 DstType: EltTy, SrcVec: VectorVal, SubVec: ExtractFromLoad, Idx: uint64_t(0), Name: "cast.scalable");
959 // Insert scalable vector to vector tuple
960 llvm::Value *Idx = CGF.Builder.getInt32(C: i);
961 TupleVal =
962 CGF.Builder.CreateIntrinsic(ID: llvm::Intrinsic::riscv_tuple_insert,
963 OverloadTypes: {Ty, EltTy}, Args: {TupleVal, VectorVal, Idx});
964 }
965 return TupleVal;
966 }
967
968 // In RISC-V VLS calling convention, struct of fixed vector or struct of
969 // fixed vector array of length 1 might be lowered using scalable vector,
970 // we consider it as a valid load, e.g.
971 // struct i32x4 {
972 // __attribute__((vector_size(16))) int i;
973 // };
974 // or
975 // struct i32x4 {
976 // __attribute__((vector_size(16))) int i[1];
977 // };
978 // is lowered to <vscale x 2 x i32>
979 // when ABI_VLEN = 128 bits, please checkout
980 // clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
981 // for more information.
982 auto *ScalableDstTy = cast<llvm::ScalableVectorType>(Val: Ty);
983 SrcTy = SrcSTy->getElementType(N: 0);
984 if (auto *ArrayTy = dyn_cast<llvm::ArrayType>(Val: SrcTy))
985 SrcTy = ArrayTy->getElementType();
986 Src = Src.withElementType(ElemTy: SrcTy);
987 auto *FixedSrcTy = cast<llvm::FixedVectorType>(Val: SrcTy);
988 llvm::Value *Load = CGF.Builder.CreateLoad(Addr: Src);
989 llvm::FixedVectorType *VLSTy = getVLSCCCompatibleType(FixedVecTy: FixedSrcTy);
990 if (VLSTy != FixedSrcTy)
991 Load = CGF.Builder.CreateBitCast(V: Load, DestTy: VLSTy);
992 auto *VectorVal = llvm::PoisonValue::get(T: ScalableDstTy);
993 llvm::Value *Result = CGF.Builder.CreateInsertVector(
994 DstType: ScalableDstTy, SrcVec: VectorVal, SubVec: Load, Idx: uint64_t(0), Name: "cast.scalable");
995 return Result;
996}
997
998void RISCVABIInfo::createCoercedStore(llvm::Value *Val, Address Dst,
999 const ABIArgInfo &AI, bool DestIsVolatile,
1000 CodeGenFunction &CGF) const {
1001 llvm::Type *SrcTy = Val->getType();
1002 llvm::StructType *DstSTy = cast<llvm::StructType>(Val: Dst.getElementType());
1003 assert((SrcTy->isScalableTy() || SrcTy->isTargetExtTy()) &&
1004 "Only scalable vector type and vector tuple type are allowed for "
1005 "store value.");
1006 if (llvm::TargetExtType *TupTy = dyn_cast<llvm::TargetExtType>(Val: SrcTy)) {
1007 // In RISC-V VLS calling convention, struct of fixed vectors or struct
1008 // of array of fixed vector of length >1 might be lowered using vector
1009 // tuple type, we consider it as a valid load, e.g.
1010 // struct i32x4x2 {
1011 // __attribute__((vector_size(16))) int i;
1012 // __attribute__((vector_size(16))) int i;
1013 // };
1014 // or
1015 // struct i32x4 {
1016 // __attribute__((vector_size(16))) int i[2];
1017 // };
1018 // is lowered to target("riscv.vector.tuple", <vscale x 8 x i8>, 2)
1019 // when ABI_VLEN = 128 bits, please checkout
1020 // clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
1021 // for more information.
1022 assert(TupTy->getName() == "riscv.vector.tuple");
1023 llvm::Type *EltTy = TupTy->getTypeParameter(i: 0);
1024 unsigned NumElts = TupTy->getIntParameter(i: 0);
1025
1026 llvm::Type *FixedVecTy = DstSTy->getElementType(N: 0);
1027 if (auto *ArrayTy = dyn_cast<llvm::ArrayType>(Val: DstSTy->getElementType(N: 0))) {
1028 Dst = Dst.withElementType(ElemTy: ArrayTy);
1029 FixedVecTy = ArrayTy->getArrayElementType();
1030 }
1031
1032 llvm::FixedVectorType *VLSTy =
1033 getVLSCCCompatibleType(FixedVecTy: cast<llvm::FixedVectorType>(Val: FixedVecTy));
1034
1035 // Perform extract element and store
1036 for (unsigned i = 0; i < NumElts; ++i) {
1037 // Element in vector tuple type is always i8, so we need to cast back
1038 // to it's original element type.
1039 EltTy =
1040 cast<llvm::ScalableVectorType>(Val: llvm::VectorType::getWithSizeAndScalar(
1041 SizeTy: cast<llvm::VectorType>(Val: EltTy), EltTy: VLSTy));
1042 // Extract scalable vector from tuple
1043 llvm::Value *Idx = CGF.Builder.getInt32(C: i);
1044 auto *TupleElement = CGF.Builder.CreateIntrinsic(
1045 ID: llvm::Intrinsic::riscv_tuple_extract, OverloadTypes: {EltTy, TupTy}, Args: {Val, Idx});
1046
1047 // Extract fixed vector from scalable vector
1048 llvm::Value *ExtractVec =
1049 CGF.Builder.CreateExtractVector(DstType: VLSTy, SrcVec: TupleElement, Idx: uint64_t(0));
1050 if (VLSTy != FixedVecTy)
1051 ExtractVec = CGF.Builder.CreateBitCast(V: ExtractVec, DestTy: FixedVecTy);
1052 // Store fixed vector to corresponding address
1053 Address EltPtr = Address::invalid();
1054 if (Dst.getElementType()->isStructTy())
1055 EltPtr = CGF.Builder.CreateStructGEP(Addr: Dst, Index: i);
1056 else
1057 EltPtr = CGF.Builder.CreateConstArrayGEP(Addr: Dst, Index: i);
1058 auto *I = CGF.Builder.CreateStore(Val: ExtractVec, Addr: EltPtr, IsVolatile: DestIsVolatile);
1059 CGF.addInstToCurrentSourceAtom(KeyInstruction: I, Backup: ExtractVec);
1060 }
1061 return;
1062 }
1063
1064 // In RISC-V VLS calling convention, struct of fixed vector or struct of
1065 // fixed vector array of length 1 might be lowered using scalable
1066 // vector, we consider it as a valid load, e.g.
1067 // struct i32x4 {
1068 // __attribute__((vector_size(16))) int i;
1069 // };
1070 // or
1071 // struct i32x4 {
1072 // __attribute__((vector_size(16))) int i[1];
1073 // };
1074 // is lowered to <vscale x 2 x i32>
1075 // when ABI_VLEN = 128 bits, please checkout
1076 // clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
1077 // for more information.
1078 llvm::Type *EltTy = DstSTy->getElementType(N: 0);
1079 if (auto *ArrayTy = dyn_cast<llvm::ArrayType>(Val: EltTy)) {
1080 assert(ArrayTy->getNumElements() == 1);
1081 EltTy = ArrayTy->getElementType();
1082 }
1083 auto *FixedVecTy = cast<llvm::FixedVectorType>(Val: EltTy);
1084 llvm::FixedVectorType *VLSTy = getVLSCCCompatibleType(FixedVecTy);
1085 llvm::Value *Coerced =
1086 CGF.Builder.CreateExtractVector(DstType: VLSTy, SrcVec: Val, Idx: uint64_t(0));
1087 if (VLSTy != FixedVecTy)
1088 Coerced = CGF.Builder.CreateBitCast(V: Coerced, DestTy: FixedVecTy);
1089 auto *I = CGF.Builder.CreateStore(Val: Coerced, Addr: Dst, IsVolatile: DestIsVolatile);
1090 CGF.addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Val);
1091}
1092
1093namespace {
1094class RISCVTargetCodeGenInfo : public TargetCodeGenInfo {
1095public:
1096 RISCVTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, unsigned XLen,
1097 unsigned FLen, bool EABI)
1098 : TargetCodeGenInfo(
1099 std::make_unique<RISCVABIInfo>(args&: CGT, args&: XLen, args&: FLen, args&: EABI)) {
1100 SwiftInfo =
1101 std::make_unique<SwiftABIInfo>(args&: CGT, /*SwiftErrorInRegister=*/args: false);
1102 }
1103
1104 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1105 CodeGen::CodeGenModule &CGM) const override {
1106 const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
1107 if (!FD) return;
1108
1109 auto *Fn = cast<llvm::Function>(Val: GV);
1110
1111 if (CGM.getCodeGenOpts().CFProtectionReturn)
1112 Fn->addFnAttr(Kind: "hw-shadow-stack");
1113
1114 const auto *Attr = FD->getAttr<RISCVInterruptAttr>();
1115 if (!Attr)
1116 return;
1117
1118 StringRef Kind = "machine";
1119 bool HasSiFiveCLICPreemptible = false;
1120 bool HasSiFiveCLICStackSwap = false;
1121 for (RISCVInterruptAttr::InterruptType type : Attr->interrupt()) {
1122 switch (type) {
1123 case RISCVInterruptAttr::machine:
1124 // Do not update `Kind` because `Kind` is already "machine", or the
1125 // kinds also contains SiFive types which need to be applied.
1126 break;
1127 case RISCVInterruptAttr::supervisor:
1128 Kind = "supervisor";
1129 break;
1130 case RISCVInterruptAttr::rnmi:
1131 Kind = "rnmi";
1132 break;
1133 case RISCVInterruptAttr::qcinest:
1134 Kind = "qci-nest";
1135 break;
1136 case RISCVInterruptAttr::qcinonest:
1137 Kind = "qci-nonest";
1138 break;
1139 // There are three different LLVM IR attribute values for SiFive CLIC
1140 // interrupt kinds, one for each kind and one extra for their combination.
1141 case RISCVInterruptAttr::SiFiveCLICPreemptible: {
1142 HasSiFiveCLICPreemptible = true;
1143 Kind = HasSiFiveCLICStackSwap ? "SiFive-CLIC-preemptible-stack-swap"
1144 : "SiFive-CLIC-preemptible";
1145 break;
1146 }
1147 case RISCVInterruptAttr::SiFiveCLICStackSwap: {
1148 HasSiFiveCLICStackSwap = true;
1149 Kind = HasSiFiveCLICPreemptible ? "SiFive-CLIC-preemptible-stack-swap"
1150 : "SiFive-CLIC-stack-swap";
1151 break;
1152 }
1153 }
1154 }
1155
1156 Fn->addFnAttr(Kind: "interrupt", Val: Kind);
1157 }
1158};
1159} // namespace
1160
1161std::unique_ptr<TargetCodeGenInfo>
1162CodeGen::createRISCVTargetCodeGenInfo(CodeGenModule &CGM, unsigned XLen,
1163 unsigned FLen, bool EABI) {
1164 return std::make_unique<RISCVTargetCodeGenInfo>(args&: CGM.getTypes(), args&: XLen, args&: FLen,
1165 args&: EABI);
1166}
1167