| 1 | //===- CIRABIClassifier.cpp - ABI classification for CIR ------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements the classification shared by the targets that classify |
| 10 | // through the LLVM ABI Lowering Library, and the two test drivers that stand |
| 11 | // in for a target. |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #include "CIRABIClassifier.h" |
| 16 | #include "TargetLowering/CIRABIRewriteContext.h" |
| 17 | |
| 18 | #include "mlir/ABI/Targets/Test/TestTarget.h" |
| 19 | #include "clang/CIR/Dialect/IR/CIRDialect.h" |
| 20 | #include "clang/CIR/MissingFeatures.h" |
| 21 | #include "llvm/ABI/Types.h" |
| 22 | #include "llvm/ADT/TypeSwitch.h" |
| 23 | #include "llvm/Support/MathExtras.h" |
| 24 | |
| 25 | using namespace mlir; |
| 26 | using namespace mlir::abi; |
| 27 | using namespace cir; |
| 28 | |
| 29 | //===----------------------------------------------------------------------===// |
| 30 | // LLVM ABI Lowering Library classifier bridge |
| 31 | // |
| 32 | // Maps CIR types to llvm::abi::Type, runs the target's classifier from the |
| 33 | // LLVM ABI Lowering Library, and converts the result back into the |
| 34 | // dialect-agnostic mlir::abi::FunctionClassification that CIRABIRewriteContext |
| 35 | // consumes. |
| 36 | // |
| 37 | // isSupportedType says which CIR types the bridge handles, and a signature |
| 38 | // naming any other fails the pass instead of being misclassified. |
| 39 | //===----------------------------------------------------------------------===// |
| 40 | |
| 41 | /// Whether a struct's declared argument-passing kind (from the module's |
| 42 | /// record-layout metadata) allows it to be passed in registers. A record with |
| 43 | /// no layout entry (e.g. an anonymous struct) has no C++ non-trivial reason to |
| 44 | /// be forced to memory, so it defaults to can-pass-in-registers. |
| 45 | static bool recordCanPassInRegs(ModuleOp modOp, cir::RecordType recTy) { |
| 46 | auto layout = cir::tryGetRecordLayout(mod: modOp, name: recTy.getName()); |
| 47 | if (!layout) |
| 48 | return true; |
| 49 | return layout.getArgPassingKind() == cir::ArgPassingKind::CanPassInRegs; |
| 50 | } |
| 51 | |
| 52 | static bool recordIsTransparentUnion(ModuleOp modOp, cir::RecordType recTy) { |
| 53 | auto layout = cir::tryGetRecordLayout(mod: modOp, name: recTy.getName()); |
| 54 | return layout && layout.getTransparentUnion(); |
| 55 | } |
| 56 | |
| 57 | /// Read from the layout, which also covers a last field that has one. |
| 58 | static bool recordHasFlexibleArrayMember(ModuleOp modOp, |
| 59 | cir::RecordType recTy) { |
| 60 | auto layout = cir::tryGetRecordLayout(mod: modOp, name: recTy.getName()); |
| 61 | return layout && layout.getFlexibleArrayMember(); |
| 62 | } |
| 63 | |
| 64 | /// Whether the classifier could give this type the SSEUP class, looking |
| 65 | /// through arrays and records at the types they hold. |
| 66 | static bool mayReachSseUp(mlir::Type ty, const DataLayout &dl) { |
| 67 | // The classifier sizes a vector with its width rounded up to a power of two. |
| 68 | if (isa<cir::VectorType>(Val: ty)) |
| 69 | return llvm::PowerOf2Ceil(A: dl.getTypeSizeInBits(t: ty).getFixedValue()) >= 128; |
| 70 | if (auto fpTy = dyn_cast<cir::FPTypeInterface>(Val&: ty)) |
| 71 | return &fpTy.getFloatSemantics() == &llvm::APFloat::IEEEquad(); |
| 72 | if (auto arrTy = dyn_cast<cir::ArrayType>(Val&: ty)) |
| 73 | return mayReachSseUp(ty: arrTy.getElementType(), dl); |
| 74 | if (auto recTy = dyn_cast<cir::RecordType>(Val&: ty)) |
| 75 | return recTy.isComplete() && |
| 76 | llvm::any_of(Range: recTy.getMembers(), |
| 77 | P: [&](mlir::Type m) { return mayReachSseUp(ty: m, dl); }); |
| 78 | // The rest classify integer, SSE or memory. A complex is SSE at float and |
| 79 | // double width and memory above that, so it needs no walk of its own. |
| 80 | assert((isa<cir::IntType, cir::BoolType, cir::PointerType, cir::VPtrType, |
| 81 | cir::VoidType, cir::ComplexType, cir::BitFieldType>(ty)) && |
| 82 | "unhandled type in the SSEUP walk" ); |
| 83 | return false; |
| 84 | } |
| 85 | |
| 86 | /// Whether the record's size has no SSEUP coerce. That coerce is a vector as |
| 87 | /// wide as the record, so only 128, 256 and 512 bits have one. A record past |
| 88 | /// 512 bits classifies memory before any coerce is asked for. |
| 89 | static bool sseUpCoerceSizeUnsupported(uint64_t recordBits, |
| 90 | llvm::ArrayRef<mlir::Type> members, |
| 91 | const DataLayout &dl) { |
| 92 | if (recordBits == 128 || recordBits == 256 || recordBits == 512) |
| 93 | return false; |
| 94 | if (recordBits < 128 || recordBits > 512) |
| 95 | return false; |
| 96 | return llvm::any_of(Range&: members, |
| 97 | P: [&](mlir::Type m) { return mayReachSseUp(ty: m, dl); }); |
| 98 | } |
| 99 | |
| 100 | /// Whether a member is an empty record, looking through arrays, since an array |
| 101 | /// of empty records supplies no bytes either. |
| 102 | static bool memberIsEmptyRecord(mlir::Type ty) { |
| 103 | while (auto arrTy = dyn_cast<cir::ArrayType>(Val&: ty)) |
| 104 | ty = arrTy.getElementType(); |
| 105 | auto recTy = dyn_cast<cir::RecordType>(Val&: ty); |
| 106 | return recTy && recTy.isEmptyForABI(); |
| 107 | } |
| 108 | |
| 109 | /// A record's unadjusted alignment, the alignment its members give it before |
| 110 | /// an alignment attribute on the record raises it. It comes from the module's |
| 111 | /// record-layout metadata, and the computed fallback only serves hand-written |
| 112 | /// CIR. |
| 113 | static llvm::Align recordUnadjustedAlign(ModuleOp modOp, cir::RecordType recTy, |
| 114 | const cir::CIRDataLayout &cirDL) { |
| 115 | auto layout = cir::tryGetRecordLayout(mod: modOp, name: recTy.getName()); |
| 116 | if (!layout) |
| 117 | return llvm::Align(cirDL.getTypeLanguageAlignment(ty: recTy)); |
| 118 | return llvm::Align(layout.getUnadjustedAlign()); |
| 119 | } |
| 120 | |
| 121 | /// The CIR types the bridge handles. Scalars: an integer up to 128 |
| 122 | /// bits (including `__int128`), a `_BitInt` of any width, pointer, vtable |
| 123 | /// pointer, bool, void, or any floating-point type. Aggregates: a complete |
| 124 | /// struct or union whose members are all themselves supported, or an array |
| 125 | /// of a supported element type. Also a `_Complex`, or a fixed-width vector, |
| 126 | /// of a supported element type. Everything else is reported NYI at the |
| 127 | /// reject() choke point in classifySignature. |
| 128 | static bool isSupportedType(mlir::Type ty, const DataLayout &dl) { |
| 129 | // A pointer is only handled in the default address space (null) or an |
| 130 | // already-lowered target address space. A LangAddressSpaceAttr must be |
| 131 | // lowered before this pass, so reject it rather than silently dropping it. |
| 132 | if (auto ptrTy = dyn_cast<cir::PointerType>(Val&: ty)) |
| 133 | return !ptrTy.getAddrSpace() || |
| 134 | mlir::isa<cir::TargetAddressSpaceAttr>(Val: ptrTy.getAddrSpace()); |
| 135 | // cir::VPtrType carries no address-space parameter yet, so its target |
| 136 | // address space cannot be checked here even though it is not always the |
| 137 | // default one. |
| 138 | if (isa<cir::VPtrType>(Val: ty)) { |
| 139 | assert(!cir::MissingFeatures::addressSpace()); |
| 140 | return true; |
| 141 | } |
| 142 | if (isa<cir::VoidType, cir::BoolType>(Val: ty)) |
| 143 | return true; |
| 144 | // Every CIR floating-point type carries the semantics the classifier |
| 145 | // switches on, so all of them are handled. |
| 146 | if (isa<cir::FPTypeInterface>(Val: ty)) |
| 147 | return true; |
| 148 | if (auto intTy = dyn_cast<cir::IntType>(Val&: ty)) { |
| 149 | // Integers up to 64 bits, __int128, and _BitInt of any width are |
| 150 | // handled: the classifier extends a width below 32, widens 33 through 63 |
| 151 | // to i64, coerces 65 through 127 to a {i64, i64} pair, and passes 32, 64, |
| 152 | // and 128 in the natural type. A _BitInt wider than 128 bits classifies |
| 153 | // Indirect, the same path an oversized aggregate already takes. |
| 154 | // Non-_BitInt intermediate widths (65..127) do not arise from C and stay |
| 155 | // rejected. |
| 156 | if (intTy.getIsBitInt()) |
| 157 | return true; |
| 158 | return intTy.getWidth() <= 64 || intTy.getWidth() == 128; |
| 159 | } |
| 160 | if (auto complexTy = dyn_cast<cir::ComplexType>(Val&: ty)) |
| 161 | return isSupportedType(ty: complexTy.getElementType(), dl); |
| 162 | if (auto vecTy = dyn_cast<cir::VectorType>(Val&: ty)) { |
| 163 | // A scalable vector has no fixed size, and no target that classifies |
| 164 | // through the bridge passes one yet. |
| 165 | if (vecTy.isScalable()) |
| 166 | return false; |
| 167 | // mapCIRType maps a bool as a one-bit integer, so a bool vector is sized |
| 168 | // one bit per element, as clang packs it. Vectors of a _BitInt whose |
| 169 | // width is not a whole number of bytes are not handled yet. |
| 170 | mlir::Type elemTy = vecTy.getElementType(); |
| 171 | if (auto elemInt = dyn_cast<cir::IntType>(Val&: elemTy)) { |
| 172 | if (elemInt.getWidth() % 8) |
| 173 | return false; |
| 174 | } else if (!isa<cir::FPTypeInterface, cir::BoolType>(Val: elemTy)) { |
| 175 | return false; |
| 176 | } |
| 177 | return isSupportedType(ty: elemTy, dl); |
| 178 | } |
| 179 | if (auto arrTy = dyn_cast<cir::ArrayType>(Val&: ty)) |
| 180 | return isSupportedType(ty: arrTy.getElementType(), dl); |
| 181 | // A bit-field is classified from the type it was declared with and the bits |
| 182 | // it holds, and is loaded through its access unit, so both types have to be |
| 183 | // ones the bridge handles. A zero-width bit-field has no unit of its own. |
| 184 | if (auto bfTy = dyn_cast<cir::BitFieldType>(Val&: ty)) { |
| 185 | if (mlir::Type storageTy = bfTy.getStorageType()) |
| 186 | if (!isSupportedType(ty: storageTy, dl)) |
| 187 | return false; |
| 188 | return llvm::all_of(Range: bfTy.getFields(), P: [&](cir::BitFieldDeclAttr decl) { |
| 189 | return isSupportedType(ty: decl.getDeclaredType(), dl); |
| 190 | }); |
| 191 | } |
| 192 | if (auto recTy = dyn_cast<cir::RecordType>(Val&: ty)) { |
| 193 | // An incomplete record has no layout to classify. |
| 194 | if (!recTy.isComplete()) |
| 195 | return false; |
| 196 | // The members are checked first so that everything below reasons only |
| 197 | // about types the bridge handles. |
| 198 | if (!llvm::all_of(Range: recTy.getMembers(), |
| 199 | P: [&](mlir::Type m) { return isSupportedType(ty: m, dl); })) |
| 200 | return false; |
| 201 | if (recTy.isUnion()) { |
| 202 | llvm::ArrayRef<mlir::Type> members = recTy.getMembers(); |
| 203 | uint64_t recordBits = dl.getTypeSizeInBits(t: recTy).getFixedValue(); |
| 204 | // Refuse a union that could reach SSEUP at a size no coerce exists for. |
| 205 | // The classifier asserts on those rather than returning a coerce. |
| 206 | if (sseUpCoerceSizeUnsupported(recordBits, members, dl)) |
| 207 | return false; |
| 208 | // A member-less union is all padding, which classifies Ignore up to two |
| 209 | // eightbytes. Past that SysV says MEMORY regardless of content, and |
| 210 | // there is no member here to build the Indirect coercion from. |
| 211 | if (members.empty() && recordBits > 128) |
| 212 | return false; |
| 213 | // A `_BitInt` access unit past an eightbyte has byte-array storage in |
| 214 | // CIR and integer storage in classic CodeGen, so the narrowing walk |
| 215 | // finds an i8 in the second eightbyte where classic keeps an i64. |
| 216 | if (llvm::any_of(Range&: members, P: [&](mlir::Type m) { |
| 217 | auto bfTy = dyn_cast<cir::BitFieldType>(Val&: m); |
| 218 | if (!bfTy || dl.getTypeSizeInBits(t: bfTy).getFixedValue() <= 64) |
| 219 | return false; |
| 220 | return llvm::any_of(Range: bfTy.getFields(), P: [](cir::BitFieldDeclAttr d) { |
| 221 | auto intTy = dyn_cast<cir::IntType>(Val: d.getDeclaredType()); |
| 222 | return intTy && intTy.getIsBitInt(); |
| 223 | }); |
| 224 | })) |
| 225 | return false; |
| 226 | } else { |
| 227 | // A struct's `empty` member that occupies bytes is later read as an |
| 228 | // unnamed bit-field. An empty-for-ABI record occupies its own padding, |
| 229 | // and reading that as a unit would hand the classifier storage classic |
| 230 | // CodeGen ignores. A zero-sized one is dropped before classification, |
| 231 | // so a misaligned one never reaches the rule that sends its record to |
| 232 | // memory. A union needs neither rule: mapCIRType skips such a variant |
| 233 | // instead of reading it as a unit, and every variant sits at offset |
| 234 | // zero. |
| 235 | for (auto [idx, memberTy, kind] : |
| 236 | llvm::enumerate(First: recTy.getMembers(), Rest: recTy.getMemberKinds())) { |
| 237 | if (kind != cir::RecordMemberKind::Empty) |
| 238 | continue; |
| 239 | if (dl.getTypeSizeInBits(t: memberTy).getFixedValue()) { |
| 240 | if (memberIsEmptyRecord(ty: memberTy)) |
| 241 | return false; |
| 242 | } else if (recTy.getElementOffset(dataLayout: dl, idx) % |
| 243 | dl.getTypeABIAlignment(t: memberTy)) { |
| 244 | return false; |
| 245 | } |
| 246 | } |
| 247 | } |
| 248 | return true; |
| 249 | } |
| 250 | return false; |
| 251 | } |
| 252 | |
| 253 | /// Convert an llvm::abi::Type coercion type back to a CIR type. |
| 254 | static mlir::Type abiTypeToCIR(const llvm::abi::Type *ty, MLIRContext *ctx) { |
| 255 | if (!ty) |
| 256 | return nullptr; |
| 257 | return llvm::TypeSwitch<const llvm::abi::Type *, mlir::Type>(ty) |
| 258 | .Case( |
| 259 | caseFn: [&](const llvm::abi::VoidType *) { return cir::VoidType::get(ctx); }) |
| 260 | .Case(caseFn: [&](const llvm::abi::IntegerType *intTy) -> mlir::Type { |
| 261 | // mapCIRType maps a bool as a one-bit integer. |
| 262 | if (intTy->isBool()) |
| 263 | return cir::BoolType::get(ctx); |
| 264 | return cir::IntType::get(context: ctx, width: intTy->getSizeInBits().getFixedValue(), |
| 265 | is_signed: intTy->isSigned(), is_bit_int: intTy->isBitInt()); |
| 266 | }) |
| 267 | .Case(caseFn: [&](const llvm::abi::FloatType *fltTy) { |
| 268 | return cir::getFloatingPointType(sem: *fltTy->getSemantics(), ctx); |
| 269 | }) |
| 270 | .Case(caseFn: [&](const llvm::abi::PointerType *ptrTy) { |
| 271 | auto voidTy = cir::VoidType::get(ctx); |
| 272 | if (unsigned as = ptrTy->getAddrSpace()) |
| 273 | return cir::PointerType::get( |
| 274 | pointee: voidTy, addrSpace: cir::TargetAddressSpaceAttr::get(context: ctx, value: as)); |
| 275 | return cir::PointerType::get(pointee: voidTy); |
| 276 | }) |
| 277 | .Case(caseFn: [&](const llvm::abi::VectorType *vecTy) -> mlir::Type { |
| 278 | mlir::Type elemCIR = abiTypeToCIR(ty: vecTy->getElementType(), ctx); |
| 279 | if (!elemCIR) |
| 280 | return nullptr; |
| 281 | return cir::VectorType::getFixed( |
| 282 | elementType: elemCIR, size: vecTy->getNumElements().getFixedValue()); |
| 283 | }) |
| 284 | .Case(caseFn: [&](const llvm::abi::ArrayType *arrTy) -> mlir::Type { |
| 285 | mlir::Type elemCIR = abiTypeToCIR(ty: arrTy->getElementType(), ctx); |
| 286 | if (!elemCIR) |
| 287 | return nullptr; |
| 288 | return cir::ArrayType::get(elementType: elemCIR, size: arrTy->getNumElements()); |
| 289 | }) |
| 290 | .Case(caseFn: [&](const llvm::abi::RecordType *recTy) -> mlir::Type { |
| 291 | SmallVector<mlir::Type> fieldTypes; |
| 292 | fieldTypes.reserve(N: recTy->getFields().size()); |
| 293 | for (const auto &field : recTy->getFields()) { |
| 294 | mlir::Type fieldCIR = abiTypeToCIR(ty: field.FieldType, ctx); |
| 295 | if (!fieldCIR) |
| 296 | return nullptr; |
| 297 | fieldTypes.push_back(Elt: fieldCIR); |
| 298 | } |
| 299 | // Coercion types are plain register tuples, not the source record. |
| 300 | return cir::StructType::get( |
| 301 | context: ctx, members: fieldTypes, /*packed=*/false, |
| 302 | /*is_class=*/false, member_kinds: cir::RecordType::getAllDataKinds(members: fieldTypes)); |
| 303 | }) |
| 304 | .Default(defaultFn: [](const llvm::abi::Type *) -> mlir::Type { return nullptr; }); |
| 305 | } |
| 306 | |
| 307 | /// Returns \p ty with a LongDoubleType vector element replaced by its |
| 308 | /// underlying format, which is the element abiTypeToCIR builds for the same |
| 309 | /// float. |
| 310 | static mlir::Type unwrapLongDoubleVectorElement(mlir::Type ty) { |
| 311 | if (auto vecTy = dyn_cast<cir::VectorType>(Val&: ty)) { |
| 312 | if (auto longDoubleTy = |
| 313 | dyn_cast<cir::LongDoubleType>(Val: vecTy.getElementType())) |
| 314 | return cir::VectorType::get(element_type: longDoubleTy.getUnderlying(), |
| 315 | size: vecTy.getSize()); |
| 316 | } |
| 317 | return ty; |
| 318 | } |
| 319 | |
| 320 | /// Map a CIR type to an llvm::abi::Type. classifySignature pre-filters |
| 321 | /// the signature, so only the scalar and struct/array types handled here can |
| 322 | /// reach this function. |
| 323 | static const llvm::abi::Type *mapCIRType(mlir::Type type, |
| 324 | mlir::abi::ABITypeMapper &typeMapper, |
| 325 | const DataLayout &dl, |
| 326 | const cir::CIRDataLayout &cirDL, |
| 327 | ModuleOp modOp) { |
| 328 | llvm::abi::TypeBuilder &tb = typeMapper.getTypeBuilder(); |
| 329 | return llvm::TypeSwitch<mlir::Type, const llvm::abi::Type *>(type) |
| 330 | .Case(caseFn: [&](cir::IntType intTy) { |
| 331 | return tb.getIntegerType( |
| 332 | BitWidth: intTy.getWidth(), Align: llvm::Align(cirDL.getTypeLanguageAlignment(ty: type)), |
| 333 | Signed: intTy.isSigned(), IsBitInt: intTy.getIsBitInt()); |
| 334 | }) |
| 335 | .Case(caseFn: [&](cir::PointerType ptrTy) { |
| 336 | unsigned addrSpace = 0; |
| 337 | if (auto targetAsAttr = |
| 338 | dyn_cast_if_present<cir::TargetAddressSpaceAttr>( |
| 339 | Val: ptrTy.getAddrSpace())) |
| 340 | addrSpace = targetAsAttr.getValue(); |
| 341 | // Any address-space attribute is a qualifier. |
| 342 | bool hasAddressSpaceQualifier = ptrTy.getAddrSpace() != nullptr; |
| 343 | return tb.getPointerType( |
| 344 | Size: dl.getTypeSizeInBits(t: type), |
| 345 | Align: llvm::Align(cirDL.getTypeLanguageAlignment(ty: type)), Addrspace: addrSpace, |
| 346 | HasAddressSpaceQualifier: hasAddressSpaceQualifier); |
| 347 | }) |
| 348 | .Case(caseFn: [&](cir::VPtrType) { |
| 349 | // cir::VPtrType carries no address-space parameter yet, so this |
| 350 | // always maps into the default one until that gap closes. |
| 351 | assert(!cir::MissingFeatures::addressSpace()); |
| 352 | return tb.getPointerType( |
| 353 | Size: dl.getTypeSizeInBits(t: type), |
| 354 | Align: llvm::Align(cirDL.getTypeLanguageAlignment(ty: type))); |
| 355 | }) |
| 356 | .Case(caseFn: [&](cir::BoolType) { |
| 357 | // A bool is a one-bit integer to the classifier, as QualTypeMapper |
| 358 | // maps it, so a bool vector is sized one bit per element. |
| 359 | return tb.getIntegerType( |
| 360 | BitWidth: 1, Align: llvm::Align(cirDL.getTypeLanguageAlignment(ty: type)), |
| 361 | /*Signed=*/false); |
| 362 | }) |
| 363 | .Case(caseFn: [&](cir::VoidType) { return tb.getVoidType(); }) |
| 364 | .Case(caseFn: [&](cir::FPTypeInterface fpTy) { |
| 365 | // LongDoubleType reports its underlying format's semantics, so the |
| 366 | // classifier sees x87 or IEEE quad rather than the wrapper. |
| 367 | return tb.getFloatType( |
| 368 | Semantics: fpTy.getFloatSemantics(), |
| 369 | Align: llvm::Align(cirDL.getTypeLanguageAlignment(ty: type))); |
| 370 | }) |
| 371 | .Case(caseFn: [&](cir::ComplexType complexTy) { |
| 372 | return tb.getComplexType( |
| 373 | ElementType: mapCIRType(type: complexTy.getElementType(), typeMapper, dl, cirDL, |
| 374 | modOp), |
| 375 | Align: llvm::Align(cirDL.getTypeLanguageAlignment(ty: type))); |
| 376 | }) |
| 377 | .Case(caseFn: [&](cir::VectorType vecTy) { |
| 378 | // isSupportedType rejects a scalable vector, so the element count is |
| 379 | // always fixed here. |
| 380 | return tb.getVectorType( |
| 381 | ElementType: mapCIRType(type: vecTy.getElementType(), typeMapper, dl, cirDL, modOp), |
| 382 | NumElements: vecTy.getSize(), Align: llvm::Align(cirDL.getTypeLanguageAlignment(ty: type))); |
| 383 | }) |
| 384 | .Case(caseFn: [&](cir::ArrayType arrTy) { |
| 385 | const llvm::abi::Type *elemAbi = |
| 386 | mapCIRType(type: arrTy.getElementType(), typeMapper, dl, cirDL, modOp); |
| 387 | return tb.getArrayType(ElementType: elemAbi, NumElements: arrTy.getSize(), |
| 388 | SizeInBits: dl.getTypeSizeInBits(t: type).getFixedValue()); |
| 389 | }) |
| 390 | .Case(caseFn: [&](cir::RecordType recTy) -> const llvm::abi::Type * { |
| 391 | llvm::abi::RecordFlags flags = llvm::abi::RecordFlags::None; |
| 392 | if (recordCanPassInRegs(modOp, recTy)) |
| 393 | flags = flags | llvm::abi::RecordFlags::CanPassInRegisters; |
| 394 | if (recordIsTransparentUnion(modOp, recTy)) |
| 395 | flags = flags | llvm::abi::RecordFlags::IsTransparent; |
| 396 | if (recordHasFlexibleArrayMember(modOp, recTy)) |
| 397 | flags = flags | llvm::abi::RecordFlags::HasFlexibleArrayMember; |
| 398 | llvm::TypeSize sizeBits = llvm::TypeSize::getFixed( |
| 399 | ExactSize: dl.getTypeSizeInBits(t: type).getFixedValue()); |
| 400 | llvm::Align align(cirDL.getTypeLanguageAlignment(ty: recTy)); |
| 401 | llvm::Align unadjustedAlign = |
| 402 | recordUnadjustedAlign(modOp, recTy, cirDL); |
| 403 | |
| 404 | // Mapped with no fields, an empty record reaches Ignore on its own. |
| 405 | // The size still matters, since a target may pass a large empty |
| 406 | // record in memory whatever its content. |
| 407 | if (recTy.isEmptyForABI()) |
| 408 | return tb.getRecordType( |
| 409 | /*Fields=*/{}, Size: sizeBits, ABIAlign: align, UnadjustedAlign: unadjustedAlign, |
| 410 | Pack: llvm::abi::StructPacking::Default, |
| 411 | /*BaseClasses=*/{}, /*VirtualBaseClasses=*/{}, RecFlags: flags); |
| 412 | |
| 413 | SmallVector<llvm::abi::FieldInfo> fields; |
| 414 | fields.reserve(N: recTy.getMembers().size()); |
| 415 | |
| 416 | // The classifier reads one entry per source bit-field, so a unit is |
| 417 | // spread back out into the fields it holds. The targets classified |
| 418 | // here are little-endian, so the unit's first byte holds the first of |
| 419 | // them, and the rest follow at the widths ahead of them. |
| 420 | // |
| 421 | // A coercion for a register is built from the type at the first byte |
| 422 | // the register holds, where the unit's first field sits, so that |
| 423 | // field's entry carries the unit rather than its own declared type. |
| 424 | // The bits holding user data are a separate question, and the answer |
| 425 | // is the extent of the type each field was declared with, which the |
| 426 | // classifier reads through to decide whether a narrower coercion |
| 427 | // would drop something. One entry cannot answer both, so a unit |
| 428 | // narrower than its first field's declaration takes an extra entry for |
| 429 | // that declaration, holding no bits of its own. A zero-width |
| 430 | // bit-field has no unit and so contributes only the declaration. |
| 431 | // |
| 432 | // A width of zero is how the classifier recognizes the entries that |
| 433 | // take no part in assigning register classes, so the width goes |
| 434 | // through as declared. |
| 435 | auto addAccessUnit = [&](cir::BitFieldType bfTy, |
| 436 | uint64_t unitOffsetBits) { |
| 437 | mlir::Type unitTy = bfTy.getStorageType(); |
| 438 | uint64_t unitBits = |
| 439 | unitTy ? dl.getTypeSizeInBits(t: unitTy).getFixedValue() : 0; |
| 440 | for (auto [idx, decl] : llvm::enumerate(First: bfTy.getFields())) { |
| 441 | const uint64_t offsetBits = |
| 442 | unitOffsetBits + bfTy.getFieldBitOffset(idx); |
| 443 | const bool holdsUnit = idx == 0 && unitTy; |
| 444 | if (holdsUnit) |
| 445 | fields.push_back(Elt: llvm::abi::FieldInfo( |
| 446 | mapCIRType(type: unitTy, typeMapper, dl, cirDL, modOp), offsetBits, |
| 447 | /*IsBitField=*/true, decl.getWidth(), |
| 448 | /*IsUnnamedBitField=*/decl.getIsUnnamed())); |
| 449 | mlir::Type declaredTy = decl.getDeclaredType(); |
| 450 | if (!holdsUnit || |
| 451 | dl.getTypeSizeInBits(t: declaredTy).getFixedValue() > unitBits) |
| 452 | fields.push_back(Elt: llvm::abi::FieldInfo( |
| 453 | mapCIRType(type: declaredTy, typeMapper, dl, cirDL, modOp), |
| 454 | offsetBits, |
| 455 | /*IsBitField=*/true, holdsUnit ? 0 : decl.getWidth(), |
| 456 | /*IsUnnamedBitField=*/holdsUnit || decl.getIsUnnamed())); |
| 457 | } |
| 458 | }; |
| 459 | |
| 460 | // The size passed here spans the tail padding, so the classifier |
| 461 | // covers the whole union rather than just the member it reduces the |
| 462 | // union to. |
| 463 | if (recTy.isUnion()) { |
| 464 | // Classify only the variants that supply data, judged by each |
| 465 | // variant's own storage rather than by its mark. One supplying |
| 466 | // none is skipped so it does not appear as a spurious argument. |
| 467 | // Whether the record is empty at all is settled from the marks |
| 468 | // before this. Every variant starts at offset zero, so a |
| 469 | // bit-field variant needs no offset of its own. |
| 470 | assert(!llvm::is_contained(recTy.getMemberKinds(), |
| 471 | cir::RecordMemberKind::Pad) && |
| 472 | "a union member cannot be marked pad" ); |
| 473 | for (mlir::Type variantTy : recTy.getMembers()) { |
| 474 | if (auto bfTy = dyn_cast<cir::BitFieldType>(Val&: variantTy)) { |
| 475 | // A variant that is an unnamed bit-field holds data all the |
| 476 | // same, so it is flagged rather than skipped, as in the struct |
| 477 | // path below. |
| 478 | addAccessUnit(bfTy, /*unitOffsetBits=*/0); |
| 479 | continue; |
| 480 | } |
| 481 | if (dl.getTypeSizeInBits(t: variantTy).getFixedValue() == 0 || |
| 482 | memberIsEmptyRecord(ty: variantTy)) |
| 483 | continue; |
| 484 | fields.push_back(Elt: llvm::abi::FieldInfo( |
| 485 | mapCIRType(type: variantTy, typeMapper, dl, cirDL, modOp))); |
| 486 | } |
| 487 | return tb.getUnionType(Fields: fields, Size: sizeBits, ABIAlign: align, UnadjustedAlign: unadjustedAlign, |
| 488 | Pack: llvm::abi::StructPacking::Default, RecFlags: flags); |
| 489 | } |
| 490 | |
| 491 | // Padding is dropped, so the classifier sees its bytes as holding |
| 492 | // nothing, while the record's full size set above still spans them. An |
| 493 | // unnamed bit-field is flagged rather than dropped, since classic |
| 494 | // CodeGen ignores it when assigning register classes but counts it |
| 495 | // when choosing the coerce type, and a member's presence alone cannot |
| 496 | // say both. |
| 497 | for (auto [idx, fieldTy, kind] : |
| 498 | llvm::enumerate(First: recTy.getMembers(), Rest: recTy.getMemberKinds())) { |
| 499 | if (kind == cir::RecordMemberKind::Pad) |
| 500 | continue; |
| 501 | if (auto bfTy = dyn_cast<cir::BitFieldType>(Val: fieldTy)) { |
| 502 | addAccessUnit(bfTy, recTy.getElementOffset(dataLayout: dl, idx) * 8); |
| 503 | continue; |
| 504 | } |
| 505 | // An `empty` member that is not a bit-field still stands for bytes |
| 506 | // no field of the source reads, so it is flagged the same way. |
| 507 | bool isUnnamed = kind == cir::RecordMemberKind::Empty; |
| 508 | uint64_t widthBits = dl.getTypeSizeInBits(t: fieldTy).getFixedValue(); |
| 509 | if (isUnnamed && widthBits == 0) |
| 510 | continue; |
| 511 | assert((!isUnnamed || !memberIsEmptyRecord(fieldTy)) && |
| 512 | "an empty-for-ABI member must not reach the classifier as an " |
| 513 | "unnamed bit-field" ); |
| 514 | fields.push_back(Elt: llvm::abi::FieldInfo( |
| 515 | mapCIRType(type: fieldTy, typeMapper, dl, cirDL, modOp), |
| 516 | recTy.getElementOffset(dataLayout: dl, idx) * 8, |
| 517 | /*IsBitField=*/isUnnamed, isUnnamed ? widthBits : 0, |
| 518 | /*IsUnnamedBitField=*/isUnnamed)); |
| 519 | } |
| 520 | |
| 521 | return tb.getRecordType(Fields: fields, Size: sizeBits, ABIAlign: align, UnadjustedAlign: unadjustedAlign, |
| 522 | Pack: llvm::abi::StructPacking::Default, |
| 523 | /*BaseClasses=*/{}, /*VirtualBaseClasses=*/{}, |
| 524 | RecFlags: flags); |
| 525 | }) |
| 526 | .Default(defaultFn: [](mlir::Type) -> const llvm::abi::Type * { |
| 527 | llvm_unreachable( |
| 528 | "mapCIRType: type not pre-filtered by classifySignature" ); |
| 529 | }); |
| 530 | } |
| 531 | |
| 532 | #ifndef NDEBUG |
| 533 | /// Whether \p coercedAbiTy holds the same value as the scalar \p origTy: an |
| 534 | /// integer of the same width, a float with the same semantics, or a pointer. |
| 535 | /// Used only in an assert in convertDirectArgInfo. |
| 536 | static bool coerceKeepsScalarValue(mlir::Type origTy, |
| 537 | const llvm::abi::Type *coercedAbiTy) { |
| 538 | if (auto intTy = dyn_cast<cir::IntType>(origTy)) { |
| 539 | const auto *abiInt = dyn_cast<llvm::abi::IntegerType>(coercedAbiTy); |
| 540 | return abiInt && |
| 541 | abiInt->getSizeInBits().getFixedValue() == intTy.getWidth(); |
| 542 | } |
| 543 | if (isa<cir::BoolType>(origTy)) |
| 544 | return isa<llvm::abi::IntegerType>(coercedAbiTy); |
| 545 | if (auto fpTy = dyn_cast<cir::FPTypeInterface>(origTy)) { |
| 546 | const auto *abiFloat = dyn_cast<llvm::abi::FloatType>(coercedAbiTy); |
| 547 | return abiFloat && abiFloat->getSemantics() == &fpTy.getFloatSemantics(); |
| 548 | } |
| 549 | if (isa<cir::PointerType, cir::VPtrType>(origTy)) |
| 550 | return isa<llvm::abi::PointerType>(coercedAbiTy); |
| 551 | return false; |
| 552 | } |
| 553 | #endif |
| 554 | |
| 555 | /// Convert a Direct llvm::abi::ArgInfo. A null coercedType in the result |
| 556 | /// means the value is passed unchanged in its own type. Otherwise the |
| 557 | /// rewriter converts the value to coercedType, and when that is a struct and |
| 558 | /// canFlatten is set, passes each field of the struct as a separate argument. |
| 559 | /// When the classifier names no coerce type, a struct or _Complex that can be |
| 560 | /// flattened is coerced to a struct of its own fields. |
| 561 | static mlir::FailureOr<ArgClassification> |
| 562 | convertDirectArgInfo(const llvm::abi::ArgInfo &info, MLIRContext *ctx, |
| 563 | mlir::Type origTy) { |
| 564 | assert(origTy && "a Direct value must have a type" ); |
| 565 | unsigned offset = info.getDirectOffset(); |
| 566 | llvm::MaybeAlign align = info.getDirectAlign(); |
| 567 | auto passUnchanged = [&] { |
| 568 | return ArgClassification::getDirect(/*coerced=*/nullptr, /*offset=*/0, |
| 569 | align); |
| 570 | }; |
| 571 | auto passCoerced = [&](mlir::Type coerced, unsigned coercedOffset) { |
| 572 | ArgClassification classified = |
| 573 | ArgClassification::getDirect(coerced, offset: coercedOffset, align); |
| 574 | classified.canFlatten = info.getCanBeFlattened(); |
| 575 | return classified; |
| 576 | }; |
| 577 | |
| 578 | const llvm::abi::Type *coercedAbiTy = info.getCoerceToType(); |
| 579 | if (!coercedAbiTy) { |
| 580 | // A record the classifier allows to be flattened is coerced to its own |
| 581 | // type, so that its fields are passed as separate arguments. A union is |
| 582 | // flattened as its LLVM type, the storage member and the tail padding. |
| 583 | auto recTy = dyn_cast<cir::RecordType>(Val&: origTy); |
| 584 | if (recTy && (recTy.isStruct() || recTy.isUnion()) && |
| 585 | info.getCanBeFlattened()) |
| 586 | return passCoerced(recTy, /*offset=*/0); |
| 587 | // A _Complex is flattened the same way, as its {real, imag} pair. |
| 588 | if (auto complexTy = dyn_cast<cir::ComplexType>(Val&: origTy); |
| 589 | complexTy && info.getCanBeFlattened()) { |
| 590 | SmallVector<mlir::Type, 2> parts(2, complexTy.getElementType()); |
| 591 | return passCoerced( |
| 592 | cir::StructType::get(context: ctx, members: parts, /*packed=*/false, |
| 593 | /*is_class=*/false, |
| 594 | member_kinds: cir::RecordType::getAllDataKinds(members: parts)), |
| 595 | /*offset=*/0); |
| 596 | } |
| 597 | return passUnchanged(); |
| 598 | } |
| 599 | |
| 600 | // A nonzero offset means the value's data starts that many bytes into its |
| 601 | // storage. The rewriter has no implementation of an offset for a coerced |
| 602 | // struct, either when it passes the struct's fields as separate arguments |
| 603 | // or when it fetches the struct in va_arg, so this is reported as NYI. |
| 604 | bool coercedIsStruct = isa<llvm::abi::RecordType>(Val: coercedAbiTy); |
| 605 | if (offset && coercedIsStruct) |
| 606 | return mlir::failure(); |
| 607 | |
| 608 | // A scalar at byte 0 that is not split into a struct is coerced when the |
| 609 | // classifier names a wider integer or a pointer in another address space. |
| 610 | // Otherwise the classifier names a type of the same kind and width, and the |
| 611 | // scalar passes unchanged. The coerce type is not compared with the |
| 612 | // scalar's own in CIR, since converting it to CIR loses a long double's |
| 613 | // wrapper and a pointer's pointee. |
| 614 | bool isScalar = |
| 615 | !isa<cir::VectorType, cir::RecordType, cir::ArrayType, cir::ComplexType>( |
| 616 | Val: origTy); |
| 617 | if (isScalar && !offset && !coercedIsStruct) { |
| 618 | auto origPtr = dyn_cast<cir::PointerType>(Val&: origTy); |
| 619 | const auto *coercedAbiPtr = dyn_cast<llvm::abi::PointerType>(Val: coercedAbiTy); |
| 620 | if (origPtr && coercedAbiPtr) { |
| 621 | // Compare numbers, since an explicit address_space(0) keeps a |
| 622 | // target_address_space(0) attribute that a plain pointer lacks. |
| 623 | auto origAS = dyn_cast_if_present<cir::TargetAddressSpaceAttr>( |
| 624 | Val: origPtr.getAddrSpace()); |
| 625 | if (coercedAbiPtr->getAddrSpace() == (origAS ? origAS.getValue() : 0)) |
| 626 | return passUnchanged(); |
| 627 | auto coercedPtr = cast<cir::PointerType>(Val: abiTypeToCIR(ty: coercedAbiTy, ctx)); |
| 628 | return passCoerced(cir::PointerType::get(pointee: origPtr.getPointee(), |
| 629 | addrSpace: coercedPtr.getAddrSpace()), |
| 630 | /*offset=*/0); |
| 631 | } |
| 632 | auto origInt = dyn_cast<cir::IntType>(Val&: origTy); |
| 633 | const auto *coercedAbiInt = dyn_cast<llvm::abi::IntegerType>(Val: coercedAbiTy); |
| 634 | bool widensInt = |
| 635 | origInt && coercedAbiInt && |
| 636 | coercedAbiInt->getSizeInBits().getFixedValue() > origInt.getWidth(); |
| 637 | if (!widensInt) { |
| 638 | // FIXME: A coerce type that changes the scalar's kind or width should |
| 639 | // be handled like a wider integer. The scalar is coerced to |
| 640 | // abiTypeToCIR(coercedAbiTy), which the rewriter emits as a store of |
| 641 | // the scalar and a load of the coerce type from the same memory. |
| 642 | assert(coerceKeepsScalarValue(origTy, coercedAbiTy) && |
| 643 | "the classifier changed a scalar's kind or width" ); |
| 644 | return passUnchanged(); |
| 645 | } |
| 646 | } |
| 647 | |
| 648 | mlir::Type coerced = abiTypeToCIR(ty: coercedAbiTy, ctx); |
| 649 | if (!coerced) |
| 650 | return mlir::failure(); |
| 651 | if (coerced == unwrapLongDoubleVectorElement(ty: origTy)) { |
| 652 | // ArgClassification has no way to represent a coercion to the value's own |
| 653 | // type at an offset, since a coerced type equal to the value's own type |
| 654 | // is treated as no coercion. This is reported as NYI. |
| 655 | if (offset) |
| 656 | return mlir::failure(); |
| 657 | // A vector the classifier passes as itself needs no rewrite. |
| 658 | if (isa<cir::VectorType>(Val: origTy)) |
| 659 | return passUnchanged(); |
| 660 | } |
| 661 | // An aggregate or _Complex is unpacked into the registers the coerced type |
| 662 | // describes. A vector or scalar is reinterpreted as the coerced type. |
| 663 | return passCoerced(coerced, offset); |
| 664 | } |
| 665 | |
| 666 | /// Convert an llvm::abi::ArgInfo into the ArgClassification consumed by |
| 667 | /// CIRABIRewriteContext. Returns failure when the classifier picks a |
| 668 | /// coercion this bridge cannot represent, which the caller reports as NYI. |
| 669 | static mlir::FailureOr<ArgClassification> |
| 670 | convertABIArgInfo(const llvm::abi::ArgInfo &info, MLIRContext *ctx, |
| 671 | mlir::Type origTy) { |
| 672 | if (info.isDirect()) |
| 673 | return convertDirectArgInfo(info, ctx, origTy); |
| 674 | // An extended value is always read from byte 0 of its own storage, so |
| 675 | // there is no offset to honor here. |
| 676 | if (info.isExtend()) { |
| 677 | assert(origTy && "an extended value must have a type" ); |
| 678 | if (isa<cir::BoolType>(Val: origTy)) |
| 679 | return ArgClassification::getExtend(coerced: nullptr, signExt: info.isSignExt(), |
| 680 | conservative: info.isConservative()); |
| 681 | assert((isa<cir::IntType, cir::RecordType>(origTy)) && |
| 682 | "Extend is only expected for integers, bool, and transparent " |
| 683 | "unions" ); |
| 684 | mlir::Type extendedTy = abiTypeToCIR(ty: info.getCoerceToType(), ctx); |
| 685 | return ArgClassification::getExtend(coerced: extendedTy, signExt: info.isSignExt(), |
| 686 | conservative: info.isConservative()); |
| 687 | } |
| 688 | if (info.isIndirect()) |
| 689 | return ArgClassification::getIndirect(align: info.getIndirectAlign(), |
| 690 | byVal: info.getIndirectByVal(), |
| 691 | addrSpace: info.getIndirectAddrSpace()); |
| 692 | if (info.isIndirectAliased()) |
| 693 | return ArgClassification::getIndirectAliased(align: info.getIndirectAlign(), |
| 694 | addrSpace: info.getIndirectAddrSpace()); |
| 695 | // AArch64 pure scalable aggregates use CoerceAndExpand, but they aren't |
| 696 | // supported yet. |
| 697 | assert(!info.isCoerceAndExpand() && "CoerceAndExpand is not handled yet" ); |
| 698 | assert(info.isIgnore() && "Unexpected classification" ); |
| 699 | return ArgClassification::getIgnore(); |
| 700 | } |
| 701 | |
| 702 | /// Where \p fnTy's declared parameters end and its ellipsis arguments begin. |
| 703 | static llvm::abi::RequiredArgs requiredArgs(cir::FuncType fnTy) { |
| 704 | if (!fnTy.isVarArg()) |
| 705 | return llvm::abi::RequiredArgs::All; |
| 706 | return llvm::abi::RequiredArgs(fnTy.getNumInputs()); |
| 707 | } |
| 708 | |
| 709 | /// Classify a signature (return type + argument types) using the LLVM ABI |
| 710 | /// library. Shared by the cir.func path, the variadic-call path and the |
| 711 | /// indirect-call path (the latter classifies from the callee function |
| 712 | /// pointer's pointee FuncType). \p required marks where the declared |
| 713 | /// parameters in \p inputs end. The classifier treats every argument past that |
| 714 | /// point as passed through an ellipsis. \p callConv is the calling convention |
| 715 | /// the signature is classified under. Returns failure and emits an NYI |
| 716 | /// error via \p emitError if the signature uses a type the bridge does not |
| 717 | /// handle yet. |
| 718 | mlir::FailureOr<FunctionClassification> |
| 719 | CIRABIClassifierCommon::classifySignature( |
| 720 | mlir::Type retCIR, mlir::TypeRange inputs, llvm::abi::RequiredArgs required, |
| 721 | const llvm::abi::TargetInfo &targetInfo, llvm::CallingConv::ID callConv, |
| 722 | llvm::function_ref<mlir::InFlightDiagnostic()> emitError) { |
| 723 | assert(retCIR && "signature return type must be non-null" ); |
| 724 | assert((!required.allowsOptionalArgs() || |
| 725 | required.getNumRequiredArgs() <= inputs.size()) && |
| 726 | "declared parameters cannot outnumber the classified arguments" ); |
| 727 | MLIRContext *ctx = modOp->getContext(); |
| 728 | bool voidRet = isa<cir::VoidType>(Val: retCIR); |
| 729 | |
| 730 | auto reject = [&](mlir::Type t) -> bool { |
| 731 | if (isSupportedType(ty: t, dl)) |
| 732 | return false; |
| 733 | emitError() << "calling-convention lowering not yet implemented for type " |
| 734 | << t; |
| 735 | return true; |
| 736 | }; |
| 737 | if (!voidRet && reject(retCIR)) |
| 738 | return mlir::failure(); |
| 739 | for (mlir::Type a : inputs) |
| 740 | if (reject(a)) |
| 741 | return mlir::failure(); |
| 742 | |
| 743 | const llvm::abi::Type *retAbi = |
| 744 | voidRet ? typeMapper.getTypeBuilder().getVoidType() |
| 745 | : mapCIRType(type: retCIR, typeMapper, dl, cirDL: cirDataLayout, modOp); |
| 746 | SmallVector<const llvm::abi::Type *> argAbi; |
| 747 | for (mlir::Type a : inputs) |
| 748 | argAbi.push_back(Elt: mapCIRType(type: a, typeMapper, dl, cirDL: cirDataLayout, modOp)); |
| 749 | |
| 750 | std::unique_ptr<llvm::abi::FunctionInfo> fi = |
| 751 | llvm::abi::FunctionInfo::create(CC: callConv, ReturnType: retAbi, ArgTypes: argAbi, Required: required); |
| 752 | targetInfo.computeInfo(FI&: *fi); |
| 753 | |
| 754 | // convertABIArgInfo returns failure when the classifier picks a coercion this |
| 755 | // bridge cannot represent. |
| 756 | auto nyiCoercion = [&](mlir::Type t) { |
| 757 | emitError() << "calling-convention lowering not yet " |
| 758 | "implemented for the ABI coercion of type " |
| 759 | << t; |
| 760 | }; |
| 761 | |
| 762 | FunctionClassification fc; |
| 763 | fc.returnsVoid = voidRet; |
| 764 | mlir::Type origRet = voidRet ? mlir::Type() : retCIR; |
| 765 | mlir::FailureOr<ArgClassification> retAc = |
| 766 | convertABIArgInfo(info: fi->getReturnInfo(), ctx, origTy: origRet); |
| 767 | if (failed(Result: retAc)) { |
| 768 | nyiCoercion(retCIR); |
| 769 | return mlir::failure(); |
| 770 | } |
| 771 | fc.returnInfo = *retAc; |
| 772 | for (unsigned i = 0, e = fi->arg_size(); i < e; ++i) { |
| 773 | mlir::Type origArg = inputs[i]; |
| 774 | const llvm::abi::ArgInfo &argInfo = fi->getArgInfo(Index: i).Info; |
| 775 | mlir::FailureOr<ArgClassification> ac = |
| 776 | convertABIArgInfo(info: argInfo, ctx, origTy: origArg); |
| 777 | if (failed(Result: ac)) { |
| 778 | nyiCoercion(origArg); |
| 779 | return mlir::failure(); |
| 780 | } |
| 781 | // The classifier passes a transparent union as its first member, and |
| 782 | // names no coerce type when it passes that member as its own type. The |
| 783 | // union is then left unchanged or coerced to itself above. |
| 784 | auto recTy = dyn_cast<cir::RecordType>(Val&: origArg); |
| 785 | if (ac->kind == ArgKind::Direct && recTy && |
| 786 | (!ac->coercedType || ac->coercedType == recTy) && |
| 787 | recordIsTransparentUnion(modOp, recTy)) |
| 788 | ac->coercedType = recTy.getMembers().front(); |
| 789 | ac->neededIntRegs = argInfo.getNeededIntRegs(); |
| 790 | ac->neededSseRegs = argInfo.getNeededSseRegs(); |
| 791 | fc.argInfos.push_back(Elt: *ac); |
| 792 | } |
| 793 | return fc; |
| 794 | } |
| 795 | |
| 796 | /// FIXME: Implement proper support for other calling conventions. |
| 797 | llvm::CallingConv::ID cir::convertCallingConv(cir::CallingConv cc) { |
| 798 | if (cc == cir::CallingConv::AMDGPUKernel) |
| 799 | return llvm::CallingConv::AMDGPU_KERNEL; |
| 800 | if (cc == cir::CallingConv::SpirKernel) |
| 801 | return llvm::CallingConv::SPIR_KERNEL; |
| 802 | return llvm::CallingConv::C; |
| 803 | } |
| 804 | |
| 805 | /// Classify a cir.func using the LLVM ABI library. Returns failure and |
| 806 | /// emits an NYI error if the signature uses a type the bridge does not handle |
| 807 | /// yet. |
| 808 | mlir::FailureOr<FunctionClassification> |
| 809 | CIRABIClassifierCommon::classifyFunction(cir::FuncOp func) { |
| 810 | cir::FuncType fnTy = func.getFunctionType(); |
| 811 | return classifySignature(retCIR: fnTy.getReturnType(), inputs: fnTy.getInputs(), |
| 812 | required: requiredArgs(fnTy), targetInfo: getTargetInfo(featureSource: func), |
| 813 | callConv: convertCallingConv(cc: func.getCallingConv()), |
| 814 | emitError: [&]() { return func.emitOpError(); }); |
| 815 | } |
| 816 | |
| 817 | /// Classify the single type fetched by a `cir.va_arg` as an unnamed argument |
| 818 | /// (RequiredArgs(0)) with a full register budget. Returns failure and |
| 819 | /// emits an NYI error if the type is not one this bridge handles. |
| 820 | mlir::FailureOr<ArgClassification> CIRABIClassifierCommon::classifyVarArgType( |
| 821 | mlir::Type ty, const llvm::abi::TargetInfo &targetInfo, |
| 822 | llvm::function_ref<mlir::InFlightDiagnostic()> emitError) { |
| 823 | MLIRContext *ctx = modOp->getContext(); |
| 824 | mlir::FailureOr<FunctionClassification> fc = classifySignature( |
| 825 | retCIR: cir::VoidType::get(ctx), inputs: mlir::TypeRange(ty), required: llvm::abi::RequiredArgs(0), |
| 826 | targetInfo, callConv: llvm::CallingConv::C, emitError); |
| 827 | if (failed(Result: fc)) |
| 828 | return mlir::failure(); |
| 829 | return fc->argInfos[0]; |
| 830 | } |
| 831 | |
| 832 | /// Classify a call that passes arguments through an ellipsis. The callee's |
| 833 | /// own classification covers only its declared parameters, but an ellipsis |
| 834 | /// argument competes for the same argument registers as a declared one, so |
| 835 | /// what the ABI does with it depends on the whole argument list: the same |
| 836 | /// small struct is passed in registers early in the list and in memory once |
| 837 | /// the integer registers are gone. Classifying from the call's operands |
| 838 | /// rather than the callee's signature is what makes that accounting right. |
| 839 | mlir::FailureOr<FunctionClassification> |
| 840 | CIRABIClassifierCommon::classifyVariadicCall( |
| 841 | cir::CIRCallOpInterface call, cir::FuncType calleeTy, |
| 842 | const llvm::abi::TargetInfo &targetInfo, llvm::CallingConv::ID callConv) { |
| 843 | assert(calleeTy.isVarArg() && |
| 844 | "only a variadic callee can take more operands than it declares" ); |
| 845 | Operation *op = call.getOperation(); |
| 846 | return classifySignature(retCIR: calleeTy.getReturnType(), |
| 847 | inputs: call.getArgOperands().getTypes(), |
| 848 | required: requiredArgs(fnTy: calleeTy), targetInfo, callConv, |
| 849 | emitError: [&]() { return op->emitOpError(); }); |
| 850 | } |
| 851 | |
| 852 | mlir::FailureOr<FunctionClassification> |
| 853 | CIRABIClassifierCommon::classifyVariadicCall(cir::CIRCallOpInterface call, |
| 854 | cir::FuncOp callee) { |
| 855 | return classifyVariadicCall(call, calleeTy: callee.getFunctionType(), |
| 856 | targetInfo: getTargetInfo(featureSource: callee), |
| 857 | callConv: convertCallingConv(cc: callee.getCallingConv())); |
| 858 | } |
| 859 | |
| 860 | mlir::FailureOr<FunctionClassification> |
| 861 | CIRABIClassifierCommon::classifyIndirectCall(cir::CIRCallOpInterface call) { |
| 862 | cir::FuncType funcTy = getIndirectCalleeType(call); |
| 863 | const llvm::abi::TargetInfo &targetInfo = |
| 864 | getTargetInfo(featureSource: call->getParentOfType<cir::FuncOp>()); |
| 865 | // An indirect callee is an ordinary function pointer, so it uses the C |
| 866 | // convention. |
| 867 | if (call.getNumArgOperands() > funcTy.getNumInputs()) |
| 868 | return classifyVariadicCall(call, calleeTy: funcTy, targetInfo, callConv: llvm::CallingConv::C); |
| 869 | return classifySignature( |
| 870 | retCIR: funcTy.getReturnType(), inputs: funcTy.getInputs(), required: requiredArgs(fnTy: funcTy), |
| 871 | targetInfo, callConv: llvm::CallingConv::C, emitError: [&]() { return call->emitOpError(); }); |
| 872 | } |
| 873 | |
| 874 | /// Record on \p fc whether \p returnType is CIR's void. classifySignature |
| 875 | /// answers this itself, but the other two drivers cannot: the test target is |
| 876 | /// dialect-neutral and has no notion of CIR's void, and the |
| 877 | /// classification-attr schema carries no return type at all. Both route |
| 878 | /// through here so a classification always reaches needsRewrite paired with |
| 879 | /// the return type it was built from. |
| 880 | static mlir::FailureOr<FunctionClassification> |
| 881 | withReturnVoidness(mlir::FailureOr<FunctionClassification> fc, |
| 882 | mlir::Type returnType) { |
| 883 | if (succeeded(Result: fc)) |
| 884 | fc->returnsVoid = mlir::isa<cir::VoidType>(Val: returnType); |
| 885 | return fc; |
| 886 | } |
| 887 | |
| 888 | /// Classify \p func using whichever driver mode is configured. Returns |
| 889 | /// failure and emits an error on the function if classification fails |
| 890 | /// (e.g. injection-driver mode but the function is missing the attribute, |
| 891 | /// or the attribute is malformed). |
| 892 | static mlir::FailureOr<FunctionClassification> |
| 893 | classifyFunction(cir::FuncOp func, const DataLayout &dl, |
| 894 | cir::CallConvTarget target, StringRef classificationAttrName) { |
| 895 | ArrayRef<Type> argTypes = func.getFunctionType().getInputs(); |
| 896 | Type returnType = func.getFunctionType().getReturnType(); |
| 897 | |
| 898 | if (!classificationAttrName.empty()) { |
| 899 | auto attr = func->getAttrOfType<DictionaryAttr>(name: classificationAttrName); |
| 900 | if (!attr) { |
| 901 | func.emitOpError() |
| 902 | << "missing classification attribute '" << classificationAttrName |
| 903 | << "' (CallConvLowering driver mode 'classification-attr')" ; |
| 904 | return mlir::failure(); |
| 905 | } |
| 906 | return withReturnVoidness(fc: mlir::abi::test::parseClassificationAttr( |
| 907 | attr, emitError: [&]() { return func.emitOpError(); }), |
| 908 | returnType); |
| 909 | } |
| 910 | |
| 911 | // The x86_64 and AMDGPU targets classify through CIRABIClassifierCommon, so |
| 912 | // only the test target reaches here. |
| 913 | assert(target == cir::CallConvTarget::Test && |
| 914 | "classifyFunction only handles the test target" ); |
| 915 | return withReturnVoidness(fc: mlir::abi::test::classify(argTypes, returnType, dl), |
| 916 | returnType); |
| 917 | } |
| 918 | |
| 919 | namespace { |
| 920 | |
| 921 | /// Classifies with the MLIR test ABI target: predictable rules that |
| 922 | /// approximate x86_64 SysV, used by tests that need a target-independent |
| 923 | /// driver. |
| 924 | class TestTargetClassifier final : public CIRABIClassifier { |
| 925 | public: |
| 926 | explicit TestTargetClassifier(const mlir::DataLayout &dl) : dl(dl) {} |
| 927 | |
| 928 | mlir::FailureOr<FunctionClassification> |
| 929 | classifyFunction(cir::FuncOp func) final { |
| 930 | return ::classifyFunction(func, dl, target: cir::CallConvTarget::Test, |
| 931 | /*classificationAttrName=*/"" ); |
| 932 | } |
| 933 | |
| 934 | /// The test target classifies only declared parameters. |
| 935 | mlir::FailureOr<FunctionClassification> |
| 936 | classifyVariadicCall(cir::CIRCallOpInterface call, cir::FuncOp) final { |
| 937 | call->emitOpError() |
| 938 | << "variadic arguments not yet implemented in CallConvLowering" ; |
| 939 | return mlir::failure(); |
| 940 | } |
| 941 | |
| 942 | mlir::FailureOr<FunctionClassification> |
| 943 | classifyIndirectCall(cir::CIRCallOpInterface call) final { |
| 944 | cir::FuncType funcTy = getIndirectCalleeType(call); |
| 945 | if (call.getNumArgOperands() > funcTy.getNumInputs()) { |
| 946 | call->emitOpError() |
| 947 | << "variadic arguments not yet implemented in CallConvLowering" ; |
| 948 | return mlir::failure(); |
| 949 | } |
| 950 | return withReturnVoidness(fc: mlir::abi::test::classify(argTypes: funcTy.getInputs(), |
| 951 | returnType: funcTy.getReturnType(), |
| 952 | dl), |
| 953 | returnType: funcTy.getReturnType()); |
| 954 | } |
| 955 | |
| 956 | mlir::FailureOr<ArgClassification> classifyVAArg(cir::VAArgOp vaArg) final { |
| 957 | vaArg->emitOpError() |
| 958 | << "not yet implemented in CallConvLowering for the test target" ; |
| 959 | return mlir::failure(); |
| 960 | } |
| 961 | |
| 962 | private: |
| 963 | const mlir::DataLayout &dl; |
| 964 | }; |
| 965 | |
| 966 | /// Reads a pre-built classification from a function attribute, so a test can |
| 967 | /// inject any classification (including shapes no target produces) without |
| 968 | /// depending on a real ABI target. |
| 969 | class InjectedClassifier final : public CIRABIClassifier { |
| 970 | public: |
| 971 | InjectedClassifier(const mlir::DataLayout &dl, llvm::StringRef attrName) |
| 972 | : dl(dl), attrName(attrName) {} |
| 973 | |
| 974 | mlir::FailureOr<FunctionClassification> |
| 975 | classifyFunction(cir::FuncOp func) final { |
| 976 | return ::classifyFunction(func, dl, target: cir::CallConvTarget::None, classificationAttrName: attrName); |
| 977 | } |
| 978 | |
| 979 | /// The injected classification covers a function's declared parameters, so |
| 980 | /// it cannot describe an operand passed through an ellipsis. |
| 981 | mlir::FailureOr<FunctionClassification> |
| 982 | classifyVariadicCall(cir::CIRCallOpInterface call, cir::FuncOp) final { |
| 983 | call->emitOpError() |
| 984 | << "variadic arguments not yet implemented in CallConvLowering" ; |
| 985 | return mlir::failure(); |
| 986 | } |
| 987 | |
| 988 | /// Nor can it describe a callee resolved at run time, which names no |
| 989 | /// function to read the attribute from. |
| 990 | mlir::FailureOr<FunctionClassification> |
| 991 | classifyIndirectCall(cir::CIRCallOpInterface call) final { |
| 992 | call->emitOpError() << "indirect call cannot be classified in the " |
| 993 | "'classification-attr' driver mode" ; |
| 994 | return mlir::failure(); |
| 995 | } |
| 996 | |
| 997 | mlir::FailureOr<ArgClassification> classifyVAArg(cir::VAArgOp vaArg) final { |
| 998 | vaArg->emitOpError() << "cannot be classified in the " |
| 999 | "'classification-attr' driver mode" ; |
| 1000 | return mlir::failure(); |
| 1001 | } |
| 1002 | |
| 1003 | private: |
| 1004 | const mlir::DataLayout &dl; |
| 1005 | std::string attrName; |
| 1006 | }; |
| 1007 | |
| 1008 | } // namespace |
| 1009 | |
| 1010 | std::unique_ptr<CIRABIClassifier> |
| 1011 | cir::createTestTargetABIClassifier(const mlir::DataLayout &dl) { |
| 1012 | return std::make_unique<TestTargetClassifier>(args: dl); |
| 1013 | } |
| 1014 | |
| 1015 | std::unique_ptr<CIRABIClassifier> |
| 1016 | cir::createInjectedABIClassifier(const mlir::DataLayout &dl, |
| 1017 | llvm::StringRef attrName) { |
| 1018 | return std::make_unique<InjectedClassifier>(args: dl, args&: attrName); |
| 1019 | } |
| 1020 | |