| 1 | //===- RISCV.cpp ----------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "ABIInfoImpl.h" |
| 10 | #include "TargetInfo.h" |
| 11 | #include "llvm/IR/IntrinsicsRISCV.h" |
| 12 | #include "llvm/TargetParser/RISCVTargetParser.h" |
| 13 | |
| 14 | using namespace clang; |
| 15 | using namespace clang::CodeGen; |
| 16 | |
| 17 | //===----------------------------------------------------------------------===// |
| 18 | // RISC-V ABI Implementation |
| 19 | //===----------------------------------------------------------------------===// |
| 20 | |
| 21 | namespace { |
| 22 | class RISCVABIInfo : public DefaultABIInfo { |
| 23 | private: |
| 24 | // Size of the integer ('x') registers in bits. |
| 25 | unsigned XLen; |
| 26 | // Size of the floating point ('f') registers in bits. Note that the target |
| 27 | // ISA might have a wider FLen than the selected ABI (e.g. an RV32IF target |
| 28 | // with soft float ABI has FLen==0). |
| 29 | unsigned FLen; |
| 30 | const int NumArgGPRs; |
| 31 | const int NumArgFPRs; |
| 32 | const bool EABI; |
| 33 | bool detectFPCCEligibleStructHelper(QualType Ty, CharUnits CurOff, |
| 34 | llvm::Type *&Field1Ty, |
| 35 | CharUnits &Field1Off, |
| 36 | llvm::Type *&Field2Ty, |
| 37 | CharUnits &Field2Off) const; |
| 38 | |
| 39 | bool detectVLSCCEligibleStruct(QualType Ty, unsigned ABIVLen, |
| 40 | llvm::Type *&VLSType) const; |
| 41 | |
| 42 | public: |
| 43 | RISCVABIInfo(CodeGen::CodeGenTypes &CGT, unsigned XLen, unsigned FLen, |
| 44 | bool EABI) |
| 45 | : DefaultABIInfo(CGT), XLen(XLen), FLen(FLen), NumArgGPRs(EABI ? 6 : 8), |
| 46 | NumArgFPRs(FLen != 0 ? 8 : 0), EABI(EABI) {} |
| 47 | |
| 48 | // DefaultABIInfo's classifyReturnType and classifyArgumentType are |
| 49 | // non-virtual, but computeInfo is virtual, so we overload it. |
| 50 | void computeInfo(CGFunctionInfo &FI) const override; |
| 51 | |
| 52 | ABIArgInfo classifyArgumentType(QualType Ty, bool IsFixed, int &ArgGPRsLeft, |
| 53 | int &ArgFPRsLeft, unsigned ABIVLen) const; |
| 54 | ABIArgInfo classifyReturnType(QualType RetTy, unsigned ABIVLen) const; |
| 55 | |
| 56 | RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, |
| 57 | AggValueSlot Slot) const override; |
| 58 | |
| 59 | ABIArgInfo extendType(QualType Ty, llvm::Type *CoerceTy = nullptr) const; |
| 60 | |
| 61 | bool detectFPCCEligibleStruct(QualType Ty, llvm::Type *&Field1Ty, |
| 62 | CharUnits &Field1Off, llvm::Type *&Field2Ty, |
| 63 | CharUnits &Field2Off, int &NeededArgGPRs, |
| 64 | int &NeededArgFPRs) const; |
| 65 | ABIArgInfo coerceAndExpandFPCCEligibleStruct(llvm::Type *Field1Ty, |
| 66 | CharUnits Field1Off, |
| 67 | llvm::Type *Field2Ty, |
| 68 | CharUnits Field2Off) const; |
| 69 | |
| 70 | ABIArgInfo coerceVLSVector(QualType Ty, unsigned ABIVLen = 0) const; |
| 71 | |
| 72 | using ABIInfo::appendAttributeMangling; |
| 73 | void appendAttributeMangling(TargetClonesAttr *Attr, unsigned Index, |
| 74 | raw_ostream &Out) const override; |
| 75 | void appendAttributeMangling(StringRef AttrStr, |
| 76 | raw_ostream &Out) const override; |
| 77 | llvm::Value *createCoercedLoad(Address SrcAddr, const ABIArgInfo &AI, |
| 78 | CodeGenFunction &CGF) const override; |
| 79 | void createCoercedStore(llvm::Value *Val, Address DstAddr, |
| 80 | const ABIArgInfo &AI, bool DestIsVolatile, |
| 81 | CodeGenFunction &CGF) const override; |
| 82 | }; |
| 83 | } // end anonymous namespace |
| 84 | |
| 85 | void RISCVABIInfo::appendAttributeMangling(TargetClonesAttr *Attr, |
| 86 | unsigned Index, |
| 87 | raw_ostream &Out) const { |
| 88 | appendAttributeMangling(AttrStr: Attr->getFeatureStr(Index), Out); |
| 89 | } |
| 90 | |
| 91 | void RISCVABIInfo::appendAttributeMangling(StringRef AttrStr, |
| 92 | raw_ostream &Out) const { |
| 93 | if (AttrStr == "default" ) { |
| 94 | Out << ".default" ; |
| 95 | return; |
| 96 | } |
| 97 | |
| 98 | Out << '.'; |
| 99 | |
| 100 | SmallVector<StringRef, 8> Attrs; |
| 101 | AttrStr.split(A&: Attrs, Separator: ';'); |
| 102 | |
| 103 | // Only consider the arch string. |
| 104 | StringRef ArchStr; |
| 105 | for (auto &Attr : Attrs) { |
| 106 | if (Attr.starts_with(Prefix: "arch=" )) |
| 107 | ArchStr = Attr; |
| 108 | } |
| 109 | |
| 110 | // Extract features string. |
| 111 | SmallVector<StringRef, 8> Features; |
| 112 | ArchStr.consume_front(Prefix: "arch=" ); |
| 113 | ArchStr.split(A&: Features, Separator: ','); |
| 114 | |
| 115 | llvm::stable_sort(Range&: Features); |
| 116 | |
| 117 | for (auto Feat : Features) { |
| 118 | Feat.consume_front(Prefix: "+" ); |
| 119 | Out << "_" << Feat; |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | void RISCVABIInfo::computeInfo(CGFunctionInfo &FI) const { |
| 124 | unsigned ABIVLen; |
| 125 | switch (FI.getExtInfo().getCC()) { |
| 126 | default: |
| 127 | ABIVLen = 0; |
| 128 | break; |
| 129 | #define CC_VLS_CASE(ABI_VLEN) \ |
| 130 | case CallingConv::CC_RISCVVLSCall_##ABI_VLEN: \ |
| 131 | ABIVLen = ABI_VLEN; \ |
| 132 | break; |
| 133 | CC_VLS_CASE(32) |
| 134 | CC_VLS_CASE(64) |
| 135 | CC_VLS_CASE(128) |
| 136 | CC_VLS_CASE(256) |
| 137 | CC_VLS_CASE(512) |
| 138 | CC_VLS_CASE(1024) |
| 139 | CC_VLS_CASE(2048) |
| 140 | CC_VLS_CASE(4096) |
| 141 | CC_VLS_CASE(8192) |
| 142 | CC_VLS_CASE(16384) |
| 143 | CC_VLS_CASE(32768) |
| 144 | CC_VLS_CASE(65536) |
| 145 | #undef CC_VLS_CASE |
| 146 | } |
| 147 | QualType RetTy = FI.getReturnType(); |
| 148 | if (!getCXXABI().classifyReturnType(FI)) |
| 149 | FI.getReturnInfo() = classifyReturnType(RetTy, ABIVLen); |
| 150 | |
| 151 | // IsRetIndirect is true if classifyArgumentType indicated the value should |
| 152 | // be passed indirect, or if the type size is a scalar greater than 2*XLen |
| 153 | // and not a complex type with elements <= FLen. e.g. fp128 is passed direct |
| 154 | // in LLVM IR, relying on the backend lowering code to rewrite the argument |
| 155 | // list and pass indirectly on RV32. |
| 156 | bool IsRetIndirect = FI.getReturnInfo().getKind() == ABIArgInfo::Indirect; |
| 157 | if (!IsRetIndirect && RetTy->isScalarType() && |
| 158 | getContext().getTypeSize(T: RetTy) > (2 * XLen)) { |
| 159 | if (RetTy->isComplexType() && FLen) { |
| 160 | QualType EltTy = RetTy->castAs<ComplexType>()->getElementType(); |
| 161 | IsRetIndirect = getContext().getTypeSize(T: EltTy) > FLen; |
| 162 | } else { |
| 163 | // This is a normal scalar > 2*XLen, such as fp128 on RV32. |
| 164 | IsRetIndirect = true; |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | int ArgGPRsLeft = IsRetIndirect ? NumArgGPRs - 1 : NumArgGPRs; |
| 169 | int ArgFPRsLeft = NumArgFPRs; |
| 170 | int NumFixedArgs = FI.getNumRequiredArgs(); |
| 171 | |
| 172 | int ArgNum = 0; |
| 173 | for (auto &ArgInfo : FI.arguments()) { |
| 174 | bool IsFixed = ArgNum < NumFixedArgs; |
| 175 | ArgInfo.info = classifyArgumentType(Ty: ArgInfo.type, IsFixed, ArgGPRsLeft, |
| 176 | ArgFPRsLeft, ABIVLen); |
| 177 | ArgNum++; |
| 178 | } |
| 179 | } |
| 180 | |
| 181 | // Returns true if the struct is a potential candidate for the floating point |
| 182 | // calling convention. If this function returns true, the caller is |
| 183 | // responsible for checking that if there is only a single field then that |
| 184 | // field is a float. |
| 185 | bool RISCVABIInfo::detectFPCCEligibleStructHelper(QualType Ty, CharUnits CurOff, |
| 186 | llvm::Type *&Field1Ty, |
| 187 | CharUnits &Field1Off, |
| 188 | llvm::Type *&Field2Ty, |
| 189 | CharUnits &Field2Off) const { |
| 190 | bool IsInt = Ty->isIntegralOrEnumerationType(); |
| 191 | bool IsFloat = Ty->isRealFloatingType(); |
| 192 | |
| 193 | if (IsInt || IsFloat) { |
| 194 | uint64_t Size = getContext().getTypeSize(T: Ty); |
| 195 | if (IsInt && Size > XLen) |
| 196 | return false; |
| 197 | // Can't be eligible if larger than the FP registers. Handling of half |
| 198 | // precision values has been specified in the ABI, so don't block those. |
| 199 | if (IsFloat && Size > FLen) |
| 200 | return false; |
| 201 | // Can't be eligible if an integer type was already found (int+int pairs |
| 202 | // are not eligible). |
| 203 | if (IsInt && Field1Ty && Field1Ty->isIntegerTy()) |
| 204 | return false; |
| 205 | if (!Field1Ty) { |
| 206 | Field1Ty = CGT.ConvertType(T: Ty); |
| 207 | Field1Off = CurOff; |
| 208 | return true; |
| 209 | } |
| 210 | if (!Field2Ty) { |
| 211 | Field2Ty = CGT.ConvertType(T: Ty); |
| 212 | Field2Off = CurOff; |
| 213 | return true; |
| 214 | } |
| 215 | return false; |
| 216 | } |
| 217 | |
| 218 | if (auto CTy = Ty->getAs<ComplexType>()) { |
| 219 | if (Field1Ty) |
| 220 | return false; |
| 221 | QualType EltTy = CTy->getElementType(); |
| 222 | if (getContext().getTypeSize(T: EltTy) > FLen) |
| 223 | return false; |
| 224 | Field1Ty = CGT.ConvertType(T: EltTy); |
| 225 | Field1Off = CurOff; |
| 226 | Field2Ty = Field1Ty; |
| 227 | Field2Off = Field1Off + getContext().getTypeSizeInChars(T: EltTy); |
| 228 | return true; |
| 229 | } |
| 230 | |
| 231 | if (const ConstantArrayType *ATy = getContext().getAsConstantArrayType(T: Ty)) { |
| 232 | uint64_t ArraySize = ATy->getZExtSize(); |
| 233 | QualType EltTy = ATy->getElementType(); |
| 234 | // Non-zero-length arrays of empty records make the struct ineligible for |
| 235 | // the FP calling convention in C++. |
| 236 | if (const auto *RTy = EltTy->getAsCanonical<RecordType>()) { |
| 237 | if (ArraySize != 0 && isa<CXXRecordDecl>(Val: RTy->getDecl()) && |
| 238 | isEmptyRecord(Context&: getContext(), T: EltTy, AllowArrays: true, AsIfNoUniqueAddr: true)) |
| 239 | return false; |
| 240 | } |
| 241 | CharUnits EltSize = getContext().getTypeSizeInChars(T: EltTy); |
| 242 | for (uint64_t i = 0; i < ArraySize; ++i) { |
| 243 | bool Ret = detectFPCCEligibleStructHelper(Ty: EltTy, CurOff, Field1Ty, |
| 244 | Field1Off, Field2Ty, Field2Off); |
| 245 | if (!Ret) |
| 246 | return false; |
| 247 | CurOff += EltSize; |
| 248 | } |
| 249 | return true; |
| 250 | } |
| 251 | |
| 252 | if (const auto *RTy = Ty->getAsCanonical<RecordType>()) { |
| 253 | // Structures with either a non-trivial destructor or a non-trivial |
| 254 | // copy constructor are not eligible for the FP calling convention. |
| 255 | if (getRecordArgABI(T: Ty, CXXABI&: CGT.getCXXABI())) |
| 256 | return false; |
| 257 | if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true, AsIfNoUniqueAddr: true)) |
| 258 | return true; |
| 259 | const RecordDecl *RD = RTy->getDecl()->getDefinitionOrSelf(); |
| 260 | // Unions aren't eligible unless they're empty (which is caught above). |
| 261 | if (RD->isUnion()) |
| 262 | return false; |
| 263 | const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: RD); |
| 264 | // If this is a C++ record, check the bases first. |
| 265 | if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) { |
| 266 | for (const CXXBaseSpecifier &B : CXXRD->bases()) { |
| 267 | const auto *BDecl = B.getType()->castAsCXXRecordDecl(); |
| 268 | CharUnits BaseOff = Layout.getBaseClassOffset(Base: BDecl); |
| 269 | bool Ret = detectFPCCEligibleStructHelper(Ty: B.getType(), CurOff: CurOff + BaseOff, |
| 270 | Field1Ty, Field1Off, Field2Ty, |
| 271 | Field2Off); |
| 272 | if (!Ret) |
| 273 | return false; |
| 274 | } |
| 275 | } |
| 276 | int ZeroWidthBitFieldCount = 0; |
| 277 | for (const FieldDecl *FD : RD->fields()) { |
| 278 | uint64_t FieldOffInBits = Layout.getFieldOffset(FieldNo: FD->getFieldIndex()); |
| 279 | QualType QTy = FD->getType(); |
| 280 | if (FD->isBitField()) { |
| 281 | unsigned BitWidth = FD->getBitWidthValue(); |
| 282 | // Allow a bitfield with a type greater than XLen as long as the |
| 283 | // bitwidth is XLen or less. |
| 284 | if (getContext().getTypeSize(T: QTy) > XLen && BitWidth <= XLen) |
| 285 | QTy = getContext().getIntTypeForBitwidth(DestWidth: XLen, Signed: false); |
| 286 | if (BitWidth == 0) { |
| 287 | ZeroWidthBitFieldCount++; |
| 288 | continue; |
| 289 | } |
| 290 | } |
| 291 | |
| 292 | bool Ret = detectFPCCEligibleStructHelper( |
| 293 | Ty: QTy, CurOff: CurOff + getContext().toCharUnitsFromBits(BitSize: FieldOffInBits), |
| 294 | Field1Ty, Field1Off, Field2Ty, Field2Off); |
| 295 | if (!Ret) |
| 296 | return false; |
| 297 | |
| 298 | // As a quirk of the ABI, zero-width bitfields aren't ignored for fp+fp |
| 299 | // or int+fp structs, but are ignored for a struct with an fp field and |
| 300 | // any number of zero-width bitfields. |
| 301 | if (Field2Ty && ZeroWidthBitFieldCount > 0) |
| 302 | return false; |
| 303 | } |
| 304 | return Field1Ty != nullptr; |
| 305 | } |
| 306 | |
| 307 | return false; |
| 308 | } |
| 309 | |
| 310 | // Determine if a struct is eligible for passing according to the floating |
| 311 | // point calling convention (i.e., when flattened it contains a single fp |
| 312 | // value, fp+fp, or int+fp of appropriate size). If so, NeededArgFPRs and |
| 313 | // NeededArgGPRs are incremented appropriately. |
| 314 | bool RISCVABIInfo::detectFPCCEligibleStruct(QualType Ty, llvm::Type *&Field1Ty, |
| 315 | CharUnits &Field1Off, |
| 316 | llvm::Type *&Field2Ty, |
| 317 | CharUnits &Field2Off, |
| 318 | int &NeededArgGPRs, |
| 319 | int &NeededArgFPRs) const { |
| 320 | Field1Ty = nullptr; |
| 321 | Field2Ty = nullptr; |
| 322 | NeededArgGPRs = 0; |
| 323 | NeededArgFPRs = 0; |
| 324 | bool IsCandidate = detectFPCCEligibleStructHelper( |
| 325 | Ty, CurOff: CharUnits::Zero(), Field1Ty, Field1Off, Field2Ty, Field2Off); |
| 326 | if (!Field1Ty) |
| 327 | return false; |
| 328 | // Not really a candidate if we have a single int but no float. |
| 329 | if (Field1Ty && !Field2Ty && !Field1Ty->isFloatingPointTy()) |
| 330 | return false; |
| 331 | if (!IsCandidate) |
| 332 | return false; |
| 333 | if (Field1Ty && Field1Ty->isFloatingPointTy()) |
| 334 | NeededArgFPRs++; |
| 335 | else if (Field1Ty) |
| 336 | NeededArgGPRs++; |
| 337 | if (Field2Ty && Field2Ty->isFloatingPointTy()) |
| 338 | NeededArgFPRs++; |
| 339 | else if (Field2Ty) |
| 340 | NeededArgGPRs++; |
| 341 | return true; |
| 342 | } |
| 343 | |
| 344 | // Call getCoerceAndExpand for the two-element flattened struct described by |
| 345 | // Field1Ty, Field1Off, Field2Ty, Field2Off. This method will create an |
| 346 | // appropriate coerceToType and unpaddedCoerceToType. |
| 347 | ABIArgInfo RISCVABIInfo::coerceAndExpandFPCCEligibleStruct( |
| 348 | llvm::Type *Field1Ty, CharUnits Field1Off, llvm::Type *Field2Ty, |
| 349 | CharUnits Field2Off) const { |
| 350 | SmallVector<llvm::Type *, 3> CoerceElts; |
| 351 | SmallVector<llvm::Type *, 2> UnpaddedCoerceElts; |
| 352 | if (!Field1Off.isZero()) |
| 353 | CoerceElts.push_back(Elt: llvm::ArrayType::get( |
| 354 | ElementType: llvm::Type::getInt8Ty(C&: getVMContext()), NumElements: Field1Off.getQuantity())); |
| 355 | |
| 356 | CoerceElts.push_back(Elt: Field1Ty); |
| 357 | UnpaddedCoerceElts.push_back(Elt: Field1Ty); |
| 358 | |
| 359 | if (!Field2Ty) { |
| 360 | return ABIArgInfo::getCoerceAndExpand( |
| 361 | coerceToType: llvm::StructType::get(Context&: getVMContext(), Elements: CoerceElts, isPacked: !Field1Off.isZero()), |
| 362 | unpaddedCoerceToType: UnpaddedCoerceElts[0]); |
| 363 | } |
| 364 | |
| 365 | CharUnits Field2Align = |
| 366 | CharUnits::fromQuantity(Quantity: getDataLayout().getABITypeAlign(Ty: Field2Ty)); |
| 367 | CharUnits Field1End = Field1Off + |
| 368 | CharUnits::fromQuantity(Quantity: getDataLayout().getTypeStoreSize(Ty: Field1Ty)); |
| 369 | CharUnits Field2OffNoPadNoPack = Field1End.alignTo(Align: Field2Align); |
| 370 | |
| 371 | CharUnits Padding = CharUnits::Zero(); |
| 372 | if (Field2Off > Field2OffNoPadNoPack) |
| 373 | Padding = Field2Off - Field2OffNoPadNoPack; |
| 374 | else if (Field2Off != Field2Align && Field2Off > Field1End) |
| 375 | Padding = Field2Off - Field1End; |
| 376 | |
| 377 | bool IsPacked = !Field2Off.isMultipleOf(N: Field2Align); |
| 378 | |
| 379 | if (!Padding.isZero()) |
| 380 | CoerceElts.push_back(Elt: llvm::ArrayType::get( |
| 381 | ElementType: llvm::Type::getInt8Ty(C&: getVMContext()), NumElements: Padding.getQuantity())); |
| 382 | |
| 383 | CoerceElts.push_back(Elt: Field2Ty); |
| 384 | UnpaddedCoerceElts.push_back(Elt: Field2Ty); |
| 385 | |
| 386 | auto CoerceToType = |
| 387 | llvm::StructType::get(Context&: getVMContext(), Elements: CoerceElts, isPacked: IsPacked); |
| 388 | auto UnpaddedCoerceToType = |
| 389 | llvm::StructType::get(Context&: getVMContext(), Elements: UnpaddedCoerceElts, isPacked: IsPacked); |
| 390 | |
| 391 | return ABIArgInfo::getCoerceAndExpand(coerceToType: CoerceToType, unpaddedCoerceToType: UnpaddedCoerceToType); |
| 392 | } |
| 393 | |
| 394 | bool RISCVABIInfo::detectVLSCCEligibleStruct(QualType Ty, unsigned ABIVLen, |
| 395 | llvm::Type *&VLSType) const { |
| 396 | // No riscv_vls_cc attribute. |
| 397 | if (ABIVLen == 0) |
| 398 | return false; |
| 399 | |
| 400 | // Legal struct for VLS calling convention should fulfill following rules: |
| 401 | // 1. Struct element should be either "homogeneous fixed-length vectors" or "a |
| 402 | // fixed-length vector array". |
| 403 | // 2. Number of struct elements or array elements should be greater or equal |
| 404 | // to 1 and less or equal to 8 |
| 405 | // 3. Total number of vector registers needed should not exceed 8. |
| 406 | // |
| 407 | // Examples: Assume ABI_VLEN = 128. |
| 408 | // These are legal structs: |
| 409 | // a. Structs with 1~8 "same" fixed-length vectors, e.g. |
| 410 | // struct { |
| 411 | // __attribute__((vector_size(16))) int a; |
| 412 | // __attribute__((vector_size(16))) int b; |
| 413 | // } |
| 414 | // |
| 415 | // b. Structs with "single" fixed-length vector array with lengh 1~8, e.g. |
| 416 | // struct { |
| 417 | // __attribute__((vector_size(16))) int a[3]; |
| 418 | // } |
| 419 | // These are illegal structs: |
| 420 | // a. Structs with 9 fixed-length vectors, e.g. |
| 421 | // struct { |
| 422 | // __attribute__((vector_size(16))) int a; |
| 423 | // __attribute__((vector_size(16))) int b; |
| 424 | // __attribute__((vector_size(16))) int c; |
| 425 | // __attribute__((vector_size(16))) int d; |
| 426 | // __attribute__((vector_size(16))) int e; |
| 427 | // __attribute__((vector_size(16))) int f; |
| 428 | // __attribute__((vector_size(16))) int g; |
| 429 | // __attribute__((vector_size(16))) int h; |
| 430 | // __attribute__((vector_size(16))) int i; |
| 431 | // } |
| 432 | // |
| 433 | // b. Structs with "multiple" fixed-length vector array, e.g. |
| 434 | // struct { |
| 435 | // __attribute__((vector_size(16))) int a[2]; |
| 436 | // __attribute__((vector_size(16))) int b[2]; |
| 437 | // } |
| 438 | // |
| 439 | // c. Vector registers needed exceeds 8, e.g. |
| 440 | // struct { |
| 441 | // // Registers needed for single fixed-length element: |
| 442 | // // 64 * 8 / ABI_VLEN = 4 |
| 443 | // __attribute__((vector_size(64))) int a; |
| 444 | // __attribute__((vector_size(64))) int b; |
| 445 | // __attribute__((vector_size(64))) int c; |
| 446 | // __attribute__((vector_size(64))) int d; |
| 447 | // } |
| 448 | // |
| 449 | // 1. Struct of 1 fixed-length vector is passed as a scalable vector. |
| 450 | // 2. Struct of >1 fixed-length vectors are passed as vector tuple. |
| 451 | // 3. Struct of an array with 1 element of fixed-length vectors is passed as a |
| 452 | // scalable vector. |
| 453 | // 4. Struct of an array with >1 elements of fixed-length vectors is passed as |
| 454 | // vector tuple. |
| 455 | // 5. Otherwise, pass the struct indirectly. |
| 456 | |
| 457 | llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: CGT.ConvertType(T: Ty)); |
| 458 | if (!STy) |
| 459 | return false; |
| 460 | |
| 461 | unsigned NumElts = STy->getStructNumElements(); |
| 462 | if (NumElts > 8) |
| 463 | return false; |
| 464 | |
| 465 | auto *FirstEltTy = STy->getElementType(N: 0); |
| 466 | if (!STy->containsHomogeneousTypes()) |
| 467 | return false; |
| 468 | |
| 469 | if (auto *ArrayTy = dyn_cast<llvm::ArrayType>(Val: FirstEltTy)) { |
| 470 | // Only struct of single array is accepted |
| 471 | if (NumElts != 1) |
| 472 | return false; |
| 473 | FirstEltTy = ArrayTy->getArrayElementType(); |
| 474 | NumElts = ArrayTy->getNumElements(); |
| 475 | } |
| 476 | |
| 477 | auto *FixedVecTy = dyn_cast<llvm::FixedVectorType>(Val: FirstEltTy); |
| 478 | if (!FixedVecTy) |
| 479 | return false; |
| 480 | |
| 481 | // Check registers needed <= 8. |
| 482 | if (NumElts * llvm::divideCeil( |
| 483 | Numerator: FixedVecTy->getNumElements() * |
| 484 | FixedVecTy->getElementType()->getScalarSizeInBits(), |
| 485 | Denominator: ABIVLen) > |
| 486 | 8) |
| 487 | return false; |
| 488 | |
| 489 | // Turn them into scalable vector type or vector tuple type if legal. |
| 490 | if (NumElts == 1) { |
| 491 | // Handle single fixed-length vector. |
| 492 | VLSType = llvm::ScalableVectorType::get( |
| 493 | ElementType: FixedVecTy->getElementType(), |
| 494 | MinNumElts: llvm::divideCeil(Numerator: FixedVecTy->getNumElements() * |
| 495 | llvm::RISCV::RVVBitsPerBlock, |
| 496 | Denominator: ABIVLen)); |
| 497 | return true; |
| 498 | } |
| 499 | |
| 500 | // LMUL |
| 501 | // = fixed-length vector size / ABIVLen |
| 502 | // = 8 * I8EltCount / RVVBitsPerBlock |
| 503 | // => |
| 504 | // I8EltCount |
| 505 | // = (fixed-length vector size * RVVBitsPerBlock) / (ABIVLen * 8) |
| 506 | unsigned I8EltCount = |
| 507 | llvm::divideCeil(Numerator: FixedVecTy->getNumElements() * |
| 508 | FixedVecTy->getElementType()->getScalarSizeInBits() * |
| 509 | llvm::RISCV::RVVBitsPerBlock, |
| 510 | Denominator: ABIVLen * 8); |
| 511 | VLSType = llvm::TargetExtType::get( |
| 512 | Context&: getVMContext(), Name: "riscv.vector.tuple" , |
| 513 | Types: llvm::ScalableVectorType::get(ElementType: llvm::Type::getInt8Ty(C&: getVMContext()), |
| 514 | MinNumElts: I8EltCount), |
| 515 | Ints: NumElts); |
| 516 | return true; |
| 517 | } |
| 518 | |
| 519 | // Fixed-length RVV vectors are represented as scalable vectors in function |
| 520 | // args/return and must be coerced from fixed vectors. |
| 521 | ABIArgInfo RISCVABIInfo::coerceVLSVector(QualType Ty, unsigned ABIVLen) const { |
| 522 | assert(Ty->isVectorType() && "expected vector type!" ); |
| 523 | |
| 524 | const auto *VT = Ty->castAs<VectorType>(); |
| 525 | assert(VT->getElementType()->isBuiltinType() && "expected builtin type!" ); |
| 526 | |
| 527 | auto VScale = getContext().getTargetInfo().getVScaleRange( |
| 528 | LangOpts: getContext().getLangOpts(), Mode: TargetInfo::ArmStreamingKind::NotStreaming); |
| 529 | |
| 530 | unsigned NumElts = VT->getNumElements(); |
| 531 | llvm::Type *EltType = llvm::Type::getInt1Ty(C&: getVMContext()); |
| 532 | switch (VT->getVectorKind()) { |
| 533 | case VectorKind::RVVFixedLengthMask_1: |
| 534 | break; |
| 535 | case VectorKind::RVVFixedLengthMask_2: |
| 536 | NumElts *= 2; |
| 537 | break; |
| 538 | case VectorKind::RVVFixedLengthMask_4: |
| 539 | NumElts *= 4; |
| 540 | break; |
| 541 | case VectorKind::RVVFixedLengthMask: |
| 542 | NumElts *= 8; |
| 543 | break; |
| 544 | default: |
| 545 | assert((VT->getVectorKind() == VectorKind::Generic || |
| 546 | VT->getVectorKind() == VectorKind::RVVFixedLengthData) && |
| 547 | "Unexpected vector kind" ); |
| 548 | EltType = CGT.ConvertType(T: VT->getElementType()); |
| 549 | } |
| 550 | |
| 551 | llvm::ScalableVectorType *ResType; |
| 552 | |
| 553 | if (ABIVLen == 0) { |
| 554 | // The MinNumElts is simplified from equation: |
| 555 | // NumElts / VScale = |
| 556 | // (EltSize * NumElts / (VScale * RVVBitsPerBlock)) |
| 557 | // * (RVVBitsPerBlock / EltSize) |
| 558 | ResType = llvm::ScalableVectorType::get(ElementType: EltType, MinNumElts: NumElts / VScale->first); |
| 559 | } else { |
| 560 | // Check registers needed <= 8. |
| 561 | if ((EltType->getScalarSizeInBits() * NumElts / ABIVLen) > 8) |
| 562 | return getNaturalAlignIndirect( |
| 563 | Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(), |
| 564 | /*ByVal=*/false); |
| 565 | |
| 566 | // Generic vector |
| 567 | // The number of elements needs to be at least 1. |
| 568 | ResType = llvm::ScalableVectorType::get( |
| 569 | ElementType: EltType, |
| 570 | MinNumElts: llvm::divideCeil(Numerator: NumElts * llvm::RISCV::RVVBitsPerBlock, Denominator: ABIVLen)); |
| 571 | |
| 572 | // If the corresponding extension is not supported, just make it an i8 |
| 573 | // vector with same LMUL. |
| 574 | const TargetInfo &TI = getContext().getTargetInfo(); |
| 575 | if ((EltType->isHalfTy() && !TI.hasFeature(Feature: "zvfhmin" )) || |
| 576 | (EltType->isBFloatTy() && !TI.hasFeature(Feature: "zvfbfmin" )) || |
| 577 | (EltType->isFloatTy() && !TI.hasFeature(Feature: "zve32f" )) || |
| 578 | (EltType->isDoubleTy() && !TI.hasFeature(Feature: "zve64d" )) || |
| 579 | (EltType->isIntegerTy(Bitwidth: 64) && !TI.hasFeature(Feature: "zve64x" )) || |
| 580 | EltType->isIntegerTy(Bitwidth: 128)) { |
| 581 | // The number of elements needs to be at least 1. |
| 582 | ResType = llvm::ScalableVectorType::get( |
| 583 | ElementType: llvm::Type::getInt8Ty(C&: getVMContext()), |
| 584 | MinNumElts: llvm::divideCeil(Numerator: EltType->getScalarSizeInBits() * NumElts * |
| 585 | llvm::RISCV::RVVBitsPerBlock, |
| 586 | Denominator: 8 * ABIVLen)); |
| 587 | } |
| 588 | } |
| 589 | |
| 590 | return ABIArgInfo::getDirect(T: ResType); |
| 591 | } |
| 592 | |
| 593 | ABIArgInfo RISCVABIInfo::classifyArgumentType(QualType Ty, bool IsFixed, |
| 594 | int &ArgGPRsLeft, |
| 595 | int &ArgFPRsLeft, |
| 596 | unsigned ABIVLen) const { |
| 597 | assert(ArgGPRsLeft <= NumArgGPRs && "Arg GPR tracking underflow" ); |
| 598 | Ty = useFirstFieldIfTransparentUnion(Ty); |
| 599 | |
| 600 | // Structures with either a non-trivial destructor or a non-trivial |
| 601 | // copy constructor are always passed indirectly. |
| 602 | if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(T: Ty, CXXABI&: getCXXABI())) { |
| 603 | if (ArgGPRsLeft) |
| 604 | ArgGPRsLeft -= 1; |
| 605 | return getNaturalAlignIndirect( |
| 606 | Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(), |
| 607 | /*ByVal=*/RAA == CGCXXABI::RAA_DirectInMemory); |
| 608 | } |
| 609 | |
| 610 | uint64_t Size = getContext().getTypeSize(T: Ty); |
| 611 | |
| 612 | // Ignore empty structs/unions whose size is zero. According to the calling |
| 613 | // convention empty structs/unions are required to be sized types in C++. |
| 614 | if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true) && Size == 0) |
| 615 | return ABIArgInfo::getIgnore(); |
| 616 | |
| 617 | // Pass floating point values via FPRs if possible. |
| 618 | if (IsFixed && Ty->isFloatingType() && !Ty->isComplexType() && |
| 619 | FLen >= Size && ArgFPRsLeft) { |
| 620 | ArgFPRsLeft--; |
| 621 | return ABIArgInfo::getDirect(); |
| 622 | } |
| 623 | |
| 624 | // Complex types for the hard float ABI must be passed direct rather than |
| 625 | // using CoerceAndExpand. |
| 626 | if (IsFixed && Ty->isComplexType() && FLen && ArgFPRsLeft >= 2) { |
| 627 | QualType EltTy = Ty->castAs<ComplexType>()->getElementType(); |
| 628 | if (getContext().getTypeSize(T: EltTy) <= FLen) { |
| 629 | ArgFPRsLeft -= 2; |
| 630 | return ABIArgInfo::getDirect(); |
| 631 | } |
| 632 | } |
| 633 | |
| 634 | if (IsFixed && FLen && Ty->isStructureOrClassType()) { |
| 635 | llvm::Type *Field1Ty = nullptr; |
| 636 | llvm::Type *Field2Ty = nullptr; |
| 637 | CharUnits Field1Off = CharUnits::Zero(); |
| 638 | CharUnits Field2Off = CharUnits::Zero(); |
| 639 | int NeededArgGPRs = 0; |
| 640 | int NeededArgFPRs = 0; |
| 641 | bool IsCandidate = |
| 642 | detectFPCCEligibleStruct(Ty, Field1Ty, Field1Off, Field2Ty, Field2Off, |
| 643 | NeededArgGPRs, NeededArgFPRs); |
| 644 | if (IsCandidate && NeededArgGPRs <= ArgGPRsLeft && |
| 645 | NeededArgFPRs <= ArgFPRsLeft) { |
| 646 | ArgGPRsLeft -= NeededArgGPRs; |
| 647 | ArgFPRsLeft -= NeededArgFPRs; |
| 648 | return coerceAndExpandFPCCEligibleStruct(Field1Ty, Field1Off, Field2Ty, |
| 649 | Field2Off); |
| 650 | } |
| 651 | } |
| 652 | |
| 653 | if (IsFixed && Ty->isStructureOrClassType()) { |
| 654 | llvm::Type *VLSType = nullptr; |
| 655 | if (detectVLSCCEligibleStruct(Ty, ABIVLen, VLSType)) |
| 656 | return ABIArgInfo::getTargetSpecific(T: VLSType); |
| 657 | } |
| 658 | |
| 659 | uint64_t NeededAlign = getContext().getTypeAlign(T: Ty); |
| 660 | // Determine the number of GPRs needed to pass the current argument |
| 661 | // according to the ABI. 2*XLen-aligned varargs are passed in "aligned" |
| 662 | // register pairs, so may consume 3 registers. |
| 663 | // TODO: To be compatible with GCC's behaviors, we don't align registers |
| 664 | // currently if we are using ILP32E calling convention. This behavior may be |
| 665 | // changed when RV32E/ILP32E is ratified. |
| 666 | int NeededArgGPRs = 1; |
| 667 | if (!IsFixed && NeededAlign == 2 * XLen) |
| 668 | NeededArgGPRs = 2 + (EABI && XLen == 32 ? 0 : (ArgGPRsLeft % 2)); |
| 669 | else if (Size > XLen && Size <= 2 * XLen) |
| 670 | NeededArgGPRs = 2; |
| 671 | |
| 672 | if (NeededArgGPRs > ArgGPRsLeft) { |
| 673 | NeededArgGPRs = ArgGPRsLeft; |
| 674 | } |
| 675 | |
| 676 | ArgGPRsLeft -= NeededArgGPRs; |
| 677 | |
| 678 | if (!isAggregateTypeForABI(T: Ty) && !Ty->isVectorType()) { |
| 679 | // Treat an enum type as its underlying type. |
| 680 | if (const auto *ED = Ty->getAsEnumDecl()) |
| 681 | Ty = ED->getIntegerType(); |
| 682 | |
| 683 | if (const auto *EIT = Ty->getAs<BitIntType>()) { |
| 684 | |
| 685 | if (XLen == 64 && EIT->getNumBits() == 32) |
| 686 | return extendType(Ty, CoerceTy: CGT.ConvertType(T: Ty)); |
| 687 | |
| 688 | if (EIT->getNumBits() <= 2 * XLen) |
| 689 | return ABIArgInfo::getExtend(Ty, T: CGT.ConvertType(T: Ty)); |
| 690 | return getNaturalAlignIndirect( |
| 691 | Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(), |
| 692 | /*ByVal=*/false); |
| 693 | } |
| 694 | |
| 695 | // All integral types are promoted to XLen width |
| 696 | if (Size < XLen && Ty->isIntegralOrEnumerationType()) |
| 697 | return extendType(Ty, CoerceTy: CGT.ConvertType(T: Ty)); |
| 698 | |
| 699 | return ABIArgInfo::getDirect(); |
| 700 | } |
| 701 | |
| 702 | // TODO: _BitInt is not handled yet in VLS calling convention since _BitInt |
| 703 | // ABI is also not merged yet in RISC-V: |
| 704 | // https://github.com/riscv-non-isa/riscv-elf-psabi-doc/pull/419 |
| 705 | if (const VectorType *VT = Ty->getAs<VectorType>(); |
| 706 | VT && !VT->getElementType()->isBitIntType()) { |
| 707 | if (VT->getVectorKind() == VectorKind::RVVFixedLengthData || |
| 708 | VT->getVectorKind() == VectorKind::RVVFixedLengthMask || |
| 709 | VT->getVectorKind() == VectorKind::RVVFixedLengthMask_1 || |
| 710 | VT->getVectorKind() == VectorKind::RVVFixedLengthMask_2 || |
| 711 | VT->getVectorKind() == VectorKind::RVVFixedLengthMask_4) |
| 712 | return coerceVLSVector(Ty); |
| 713 | if (VT->getVectorKind() == VectorKind::Generic && ABIVLen != 0) |
| 714 | // Generic vector without riscv_vls_cc should fall through and pass by |
| 715 | // reference. |
| 716 | return coerceVLSVector(Ty, ABIVLen); |
| 717 | } |
| 718 | |
| 719 | // Aggregates which are <= 2*XLen will be passed in registers if possible, |
| 720 | // so coerce to integers. |
| 721 | if (Size <= 2 * XLen) { |
| 722 | unsigned Alignment = getContext().getTypeAlign(T: Ty); |
| 723 | |
| 724 | // Use a single XLen int if possible, 2*XLen if 2*XLen alignment is |
| 725 | // required, and a 2-element XLen array if only XLen alignment is required. |
| 726 | if (Size <= XLen) { |
| 727 | return ABIArgInfo::getDirect( |
| 728 | T: llvm::IntegerType::get(C&: getVMContext(), NumBits: XLen)); |
| 729 | } else if (Alignment == 2 * XLen) { |
| 730 | return ABIArgInfo::getDirect( |
| 731 | T: llvm::IntegerType::get(C&: getVMContext(), NumBits: 2 * XLen)); |
| 732 | } else { |
| 733 | return ABIArgInfo::getDirect(T: llvm::ArrayType::get( |
| 734 | ElementType: llvm::IntegerType::get(C&: getVMContext(), NumBits: XLen), NumElements: 2)); |
| 735 | } |
| 736 | } |
| 737 | return getNaturalAlignIndirect( |
| 738 | Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(), |
| 739 | /*ByVal=*/false); |
| 740 | } |
| 741 | |
| 742 | ABIArgInfo RISCVABIInfo::classifyReturnType(QualType RetTy, |
| 743 | unsigned ABIVLen) const { |
| 744 | if (RetTy->isVoidType()) |
| 745 | return ABIArgInfo::getIgnore(); |
| 746 | |
| 747 | int ArgGPRsLeft = 2; |
| 748 | int ArgFPRsLeft = FLen ? 2 : 0; |
| 749 | |
| 750 | // The rules for return and argument types are the same, so defer to |
| 751 | // classifyArgumentType. |
| 752 | return classifyArgumentType(Ty: RetTy, /*IsFixed=*/true, ArgGPRsLeft, ArgFPRsLeft, |
| 753 | ABIVLen); |
| 754 | } |
| 755 | |
| 756 | RValue RISCVABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, |
| 757 | QualType Ty, AggValueSlot Slot) const { |
| 758 | CharUnits SlotSize = CharUnits::fromQuantity(Quantity: XLen / 8); |
| 759 | |
| 760 | // Empty records are ignored for parameter passing purposes. |
| 761 | if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true)) |
| 762 | return Slot.asRValue(); |
| 763 | |
| 764 | auto TInfo = getContext().getTypeInfoInChars(T: Ty); |
| 765 | |
| 766 | // TODO: To be compatible with GCC's behaviors, we force arguments with |
| 767 | // 2×XLEN-bit alignment and size at most 2×XLEN bits like `long long`, |
| 768 | // `unsigned long long` and `double` to have 4-byte alignment. This |
| 769 | // behavior may be changed when RV32E/ILP32E is ratified. |
| 770 | if (EABI && XLen == 32) |
| 771 | TInfo.Align = std::min(a: TInfo.Align, b: CharUnits::fromQuantity(Quantity: 4)); |
| 772 | |
| 773 | // Arguments bigger than 2*Xlen bytes are passed indirectly. |
| 774 | bool IsIndirect = TInfo.Width > 2 * SlotSize; |
| 775 | |
| 776 | return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect, ValueInfo: TInfo, SlotSizeAndAlign: SlotSize, |
| 777 | /*AllowHigherAlign=*/true, Slot); |
| 778 | } |
| 779 | |
| 780 | ABIArgInfo RISCVABIInfo::extendType(QualType Ty, llvm::Type *CoerceTy) const { |
| 781 | int TySize = getContext().getTypeSize(T: Ty); |
| 782 | // RV64 ABI requires unsigned 32 bit integers to be sign extended. |
| 783 | if (XLen == 64 && Ty->isUnsignedIntegerOrEnumerationType() && TySize == 32) |
| 784 | return ABIArgInfo::getSignExtend(Ty, T: CoerceTy); |
| 785 | return ABIArgInfo::getExtend(Ty, T: CoerceTy); |
| 786 | } |
| 787 | |
| 788 | llvm::Value *RISCVABIInfo::createCoercedLoad(Address Src, const ABIArgInfo &AI, |
| 789 | CodeGenFunction &CGF) const { |
| 790 | llvm::Type *Ty = AI.getCoerceToType(); |
| 791 | llvm::Type *SrcTy = Src.getElementType(); |
| 792 | llvm::StructType *SrcSTy = cast<llvm::StructType>(Val: SrcTy); |
| 793 | assert((Ty->isScalableTy() || Ty->isTargetExtTy()) && |
| 794 | "Only scalable vector type and vector tuple type are allowed for load " |
| 795 | "type." ); |
| 796 | if (llvm::TargetExtType *TupTy = dyn_cast<llvm::TargetExtType>(Val: Ty)) { |
| 797 | // In RISC-V VLS calling convention, struct of fixed vectors or struct of |
| 798 | // array of fixed vector of length >1 might be lowered using vector tuple |
| 799 | // type, we consider it as a valid load, e.g. |
| 800 | // struct i32x4x2 { |
| 801 | // __attribute__((vector_size(16))) int i; |
| 802 | // __attribute__((vector_size(16))) int i; |
| 803 | // }; |
| 804 | // or |
| 805 | // struct i32x4 { |
| 806 | // __attribute__((vector_size(16))) int i[2]; |
| 807 | // }; |
| 808 | // is lowered to target("riscv.vector.tuple", <vscale x 8 x i8>, 2) |
| 809 | // when ABI_VLEN = 128 bits, please checkout |
| 810 | // clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c |
| 811 | // for more information. |
| 812 | assert(TupTy->getName() == "riscv.vector.tuple" ); |
| 813 | llvm::Type *EltTy = TupTy->getTypeParameter(i: 0); |
| 814 | unsigned NumElts = TupTy->getIntParameter(i: 0); |
| 815 | |
| 816 | if (auto *ArrayTy = dyn_cast<llvm::ArrayType>(Val: SrcSTy->getElementType(N: 0))) |
| 817 | Src = Src.withElementType(ElemTy: ArrayTy); |
| 818 | |
| 819 | // Perform extract element and load |
| 820 | llvm::Value *TupleVal = llvm::PoisonValue::get(T: Ty); |
| 821 | auto *Load = CGF.Builder.CreateLoad(Addr: Src); |
| 822 | for (unsigned i = 0; i < NumElts; ++i) { |
| 823 | // Extract from struct |
| 824 | llvm::Value * = CGF.Builder.CreateExtractValue(Agg: Load, Idxs: i); |
| 825 | // Element in vector tuple type is always i8, so we need to cast back to |
| 826 | // it's original element type. |
| 827 | EltTy = |
| 828 | cast<llvm::ScalableVectorType>(Val: llvm::VectorType::getWithSizeAndScalar( |
| 829 | SizeTy: cast<llvm::VectorType>(Val: EltTy), EltTy: ExtractFromLoad->getType())); |
| 830 | llvm::Value *VectorVal = llvm::PoisonValue::get(T: EltTy); |
| 831 | // Insert to scalable vector |
| 832 | VectorVal = CGF.Builder.CreateInsertVector( |
| 833 | DstType: EltTy, SrcVec: VectorVal, SubVec: ExtractFromLoad, Idx: uint64_t(0), Name: "cast.scalable" ); |
| 834 | // Insert scalable vector to vector tuple |
| 835 | llvm::Value *Idx = CGF.Builder.getInt32(C: i); |
| 836 | TupleVal = |
| 837 | CGF.Builder.CreateIntrinsic(ID: llvm::Intrinsic::riscv_tuple_insert, |
| 838 | Types: {Ty, EltTy}, Args: {TupleVal, VectorVal, Idx}); |
| 839 | } |
| 840 | return TupleVal; |
| 841 | } |
| 842 | |
| 843 | // In RISC-V VLS calling convention, struct of fixed vector or struct of |
| 844 | // fixed vector array of length 1 might be lowered using scalable vector, |
| 845 | // we consider it as a valid load, e.g. |
| 846 | // struct i32x4 { |
| 847 | // __attribute__((vector_size(16))) int i; |
| 848 | // }; |
| 849 | // or |
| 850 | // struct i32x4 { |
| 851 | // __attribute__((vector_size(16))) int i[1]; |
| 852 | // }; |
| 853 | // is lowered to <vscale x 2 x i32> |
| 854 | // when ABI_VLEN = 128 bits, please checkout |
| 855 | // clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c |
| 856 | // for more information. |
| 857 | auto *ScalableDstTy = cast<llvm::ScalableVectorType>(Val: Ty); |
| 858 | SrcTy = SrcSTy->getElementType(N: 0); |
| 859 | if (auto *ArrayTy = dyn_cast<llvm::ArrayType>(Val: SrcTy)) |
| 860 | SrcTy = ArrayTy->getElementType(); |
| 861 | Src = Src.withElementType(ElemTy: SrcTy); |
| 862 | [[maybe_unused]] auto *FixedSrcTy = cast<llvm::FixedVectorType>(Val: SrcTy); |
| 863 | assert(ScalableDstTy->getElementType() == FixedSrcTy->getElementType()); |
| 864 | auto *Load = CGF.Builder.CreateLoad(Addr: Src); |
| 865 | auto *VectorVal = llvm::PoisonValue::get(T: ScalableDstTy); |
| 866 | llvm::Value *Result = CGF.Builder.CreateInsertVector( |
| 867 | DstType: ScalableDstTy, SrcVec: VectorVal, SubVec: Load, Idx: uint64_t(0), Name: "cast.scalable" ); |
| 868 | return Result; |
| 869 | } |
| 870 | |
| 871 | void RISCVABIInfo::createCoercedStore(llvm::Value *Val, Address Dst, |
| 872 | const ABIArgInfo &AI, bool DestIsVolatile, |
| 873 | CodeGenFunction &CGF) const { |
| 874 | llvm::Type *SrcTy = Val->getType(); |
| 875 | llvm::StructType *DstSTy = cast<llvm::StructType>(Val: Dst.getElementType()); |
| 876 | assert((SrcTy->isScalableTy() || SrcTy->isTargetExtTy()) && |
| 877 | "Only scalable vector type and vector tuple type are allowed for " |
| 878 | "store value." ); |
| 879 | if (llvm::TargetExtType *TupTy = dyn_cast<llvm::TargetExtType>(Val: SrcTy)) { |
| 880 | // In RISC-V VLS calling convention, struct of fixed vectors or struct |
| 881 | // of array of fixed vector of length >1 might be lowered using vector |
| 882 | // tuple type, we consider it as a valid load, e.g. |
| 883 | // struct i32x4x2 { |
| 884 | // __attribute__((vector_size(16))) int i; |
| 885 | // __attribute__((vector_size(16))) int i; |
| 886 | // }; |
| 887 | // or |
| 888 | // struct i32x4 { |
| 889 | // __attribute__((vector_size(16))) int i[2]; |
| 890 | // }; |
| 891 | // is lowered to target("riscv.vector.tuple", <vscale x 8 x i8>, 2) |
| 892 | // when ABI_VLEN = 128 bits, please checkout |
| 893 | // clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c |
| 894 | // for more information. |
| 895 | assert(TupTy->getName() == "riscv.vector.tuple" ); |
| 896 | llvm::Type *EltTy = TupTy->getTypeParameter(i: 0); |
| 897 | unsigned NumElts = TupTy->getIntParameter(i: 0); |
| 898 | |
| 899 | llvm::Type *FixedVecTy = DstSTy->getElementType(N: 0); |
| 900 | if (auto *ArrayTy = dyn_cast<llvm::ArrayType>(Val: DstSTy->getElementType(N: 0))) { |
| 901 | Dst = Dst.withElementType(ElemTy: ArrayTy); |
| 902 | FixedVecTy = ArrayTy->getArrayElementType(); |
| 903 | } |
| 904 | |
| 905 | // Perform extract element and store |
| 906 | for (unsigned i = 0; i < NumElts; ++i) { |
| 907 | // Element in vector tuple type is always i8, so we need to cast back |
| 908 | // to it's original element type. |
| 909 | EltTy = |
| 910 | cast<llvm::ScalableVectorType>(Val: llvm::VectorType::getWithSizeAndScalar( |
| 911 | SizeTy: cast<llvm::VectorType>(Val: EltTy), EltTy: FixedVecTy)); |
| 912 | // Extract scalable vector from tuple |
| 913 | llvm::Value *Idx = CGF.Builder.getInt32(C: i); |
| 914 | auto *TupleElement = CGF.Builder.CreateIntrinsic( |
| 915 | ID: llvm::Intrinsic::riscv_tuple_extract, Types: {EltTy, TupTy}, Args: {Val, Idx}); |
| 916 | |
| 917 | // Extract fixed vector from scalable vector |
| 918 | auto * = CGF.Builder.CreateExtractVector( |
| 919 | DstType: FixedVecTy, SrcVec: TupleElement, Idx: uint64_t(0)); |
| 920 | // Store fixed vector to corresponding address |
| 921 | Address EltPtr = Address::invalid(); |
| 922 | if (Dst.getElementType()->isStructTy()) |
| 923 | EltPtr = CGF.Builder.CreateStructGEP(Addr: Dst, Index: i); |
| 924 | else |
| 925 | EltPtr = CGF.Builder.CreateConstArrayGEP(Addr: Dst, Index: i); |
| 926 | auto *I = CGF.Builder.CreateStore(Val: ExtractVec, Addr: EltPtr, IsVolatile: DestIsVolatile); |
| 927 | CGF.addInstToCurrentSourceAtom(KeyInstruction: I, Backup: ExtractVec); |
| 928 | } |
| 929 | return; |
| 930 | } |
| 931 | |
| 932 | // In RISC-V VLS calling convention, struct of fixed vector or struct of |
| 933 | // fixed vector array of length 1 might be lowered using scalable |
| 934 | // vector, we consider it as a valid load, e.g. |
| 935 | // struct i32x4 { |
| 936 | // __attribute__((vector_size(16))) int i; |
| 937 | // }; |
| 938 | // or |
| 939 | // struct i32x4 { |
| 940 | // __attribute__((vector_size(16))) int i[1]; |
| 941 | // }; |
| 942 | // is lowered to <vscale x 2 x i32> |
| 943 | // when ABI_VLEN = 128 bits, please checkout |
| 944 | // clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c |
| 945 | // for more information. |
| 946 | llvm::Type *EltTy = DstSTy->getElementType(N: 0); |
| 947 | if (auto *ArrayTy = dyn_cast<llvm::ArrayType>(Val: EltTy)) { |
| 948 | assert(ArrayTy->getNumElements() == 1); |
| 949 | EltTy = ArrayTy->getElementType(); |
| 950 | } |
| 951 | auto *Coerced = CGF.Builder.CreateExtractVector( |
| 952 | DstType: cast<llvm::FixedVectorType>(Val: EltTy), SrcVec: Val, Idx: uint64_t(0)); |
| 953 | auto *I = CGF.Builder.CreateStore(Val: Coerced, Addr: Dst, IsVolatile: DestIsVolatile); |
| 954 | CGF.addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Val); |
| 955 | } |
| 956 | |
| 957 | namespace { |
| 958 | class RISCVTargetCodeGenInfo : public TargetCodeGenInfo { |
| 959 | public: |
| 960 | RISCVTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, unsigned XLen, |
| 961 | unsigned FLen, bool EABI) |
| 962 | : TargetCodeGenInfo( |
| 963 | std::make_unique<RISCVABIInfo>(args&: CGT, args&: XLen, args&: FLen, args&: EABI)) { |
| 964 | SwiftInfo = |
| 965 | std::make_unique<SwiftABIInfo>(args&: CGT, /*SwiftErrorInRegister=*/args: false); |
| 966 | } |
| 967 | |
| 968 | void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, |
| 969 | CodeGen::CodeGenModule &CGM) const override { |
| 970 | const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: D); |
| 971 | if (!FD) return; |
| 972 | |
| 973 | auto *Fn = cast<llvm::Function>(Val: GV); |
| 974 | |
| 975 | if (CGM.getCodeGenOpts().CFProtectionReturn) |
| 976 | Fn->addFnAttr(Kind: "hw-shadow-stack" ); |
| 977 | |
| 978 | const auto *Attr = FD->getAttr<RISCVInterruptAttr>(); |
| 979 | if (!Attr) |
| 980 | return; |
| 981 | |
| 982 | StringRef Kind = "machine" ; |
| 983 | bool HasSiFiveCLICPreemptible = false; |
| 984 | bool HasSiFiveCLICStackSwap = false; |
| 985 | for (RISCVInterruptAttr::InterruptType type : Attr->interrupt()) { |
| 986 | switch (type) { |
| 987 | case RISCVInterruptAttr::machine: |
| 988 | // Do not update `Kind` because `Kind` is already "machine", or the |
| 989 | // kinds also contains SiFive types which need to be applied. |
| 990 | break; |
| 991 | case RISCVInterruptAttr::supervisor: |
| 992 | Kind = "supervisor" ; |
| 993 | break; |
| 994 | case RISCVInterruptAttr::rnmi: |
| 995 | Kind = "rnmi" ; |
| 996 | break; |
| 997 | case RISCVInterruptAttr::qcinest: |
| 998 | Kind = "qci-nest" ; |
| 999 | break; |
| 1000 | case RISCVInterruptAttr::qcinonest: |
| 1001 | Kind = "qci-nonest" ; |
| 1002 | break; |
| 1003 | // There are three different LLVM IR attribute values for SiFive CLIC |
| 1004 | // interrupt kinds, one for each kind and one extra for their combination. |
| 1005 | case RISCVInterruptAttr::SiFiveCLICPreemptible: { |
| 1006 | HasSiFiveCLICPreemptible = true; |
| 1007 | Kind = HasSiFiveCLICStackSwap ? "SiFive-CLIC-preemptible-stack-swap" |
| 1008 | : "SiFive-CLIC-preemptible" ; |
| 1009 | break; |
| 1010 | } |
| 1011 | case RISCVInterruptAttr::SiFiveCLICStackSwap: { |
| 1012 | HasSiFiveCLICStackSwap = true; |
| 1013 | Kind = HasSiFiveCLICPreemptible ? "SiFive-CLIC-preemptible-stack-swap" |
| 1014 | : "SiFive-CLIC-stack-swap" ; |
| 1015 | break; |
| 1016 | } |
| 1017 | } |
| 1018 | } |
| 1019 | |
| 1020 | Fn->addFnAttr(Kind: "interrupt" , Val: Kind); |
| 1021 | } |
| 1022 | }; |
| 1023 | } // namespace |
| 1024 | |
| 1025 | std::unique_ptr<TargetCodeGenInfo> |
| 1026 | CodeGen::createRISCVTargetCodeGenInfo(CodeGenModule &CGM, unsigned XLen, |
| 1027 | unsigned FLen, bool EABI) { |
| 1028 | return std::make_unique<RISCVTargetCodeGenInfo>(args&: CGM.getTypes(), args&: XLen, args&: FLen, |
| 1029 | args&: EABI); |
| 1030 | } |
| 1031 | |