| 1 | //===- CIRABIRewriteContext.cpp - CIR ABI rewrite context ----------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "CIRABIRewriteContext.h" |
| 10 | #include "mlir/Dialect/LLVMIR/LLVMDialectDecl.h" |
| 11 | #include "mlir/IR/Builders.h" |
| 12 | #include "mlir/IR/Dominance.h" |
| 13 | #include "clang/CIR/Dialect/Builder/CIRBaseBuilder.h" |
| 14 | #include "clang/CIR/Dialect/IR/CIRAttrs.h" |
| 15 | #include "clang/CIR/Dialect/IR/CIRDataLayout.h" |
| 16 | #include "clang/CIR/Dialect/IR/CIRDialect.h" |
| 17 | #include "clang/CIR/Dialect/IR/CIRTypes.h" |
| 18 | #include "clang/CIR/MissingFeatures.h" |
| 19 | #include "llvm/ADT/STLExtras.h" |
| 20 | #include "llvm/IR/Attributes.h" |
| 21 | #include "llvm/Support/MathExtras.h" |
| 22 | #include <algorithm> |
| 23 | #include <array> |
| 24 | #include <utility> |
| 25 | |
| 26 | using namespace cir; |
| 27 | using namespace mlir; |
| 28 | using namespace mlir::abi; |
| 29 | |
| 30 | // This rewrite context supports the Direct (with or without coercion), |
| 31 | // Extend, Ignore, Indirect-return (sret), Indirect-argument (byval and |
| 32 | // non-byval), IndirectAliased-argument (byref), and Expand (struct |
| 33 | // flattening) classifications. |
| 34 | // |
| 35 | // An Indirect argument is byval or not, following its classification's |
| 36 | // byVal flag. byval is a by-value parameter the ABI passes in memory rather |
| 37 | // than registers, usually for its size. Non-byval is a by-value parameter |
| 38 | // whose type cannot be copied freely, because it has a non-trivial copy |
| 39 | // constructor, move constructor, or destructor, so the callee works on the |
| 40 | // caller's own object rather than a copy. |
| 41 | // |
| 42 | // At the call site both forward the caller's storage, since byval is copied |
| 43 | // on the caller's side of the call. byval falls back to a slot of its own |
| 44 | // when the operand's storage cannot be handed on. At the callee, byval fills |
| 45 | // the CIRGen param slot with a copy of the incoming object, while non-byval |
| 46 | // rewires that slot to the incoming pointer so the body mutates the caller's |
| 47 | // storage in place. |
| 48 | // |
| 49 | // A byref argument is lowered like byval, apart from its attribute and the |
| 50 | // address space of its pointer. The callee must not write through that |
| 51 | // pointer, so it fills the CIRGen param slot with a copy just as byval does. |
| 52 | // |
| 53 | // For Expand, the single struct argument is replaced by N scalar arguments |
| 54 | // (one per field). At the callee, the N field block arguments are stored |
| 55 | // directly into the parameter's own alloca (the CIRGen spill slot). At the |
| 56 | // call site, the struct operand is decomposed into its fields by reading |
| 57 | // each member from the source alloca (get_member + load) when the operand is |
| 58 | // a load of an alloca, or via cir.extract_member otherwise. |
| 59 | // |
| 60 | // For Direct + canFlatten (where the coerced type is a multi-field struct), the |
| 61 | // coerced struct is similarly passed as N separate arguments, one per field. |
| 62 | // The callee stores the N scalar block args into one slot sized and aligned for |
| 63 | // both the coerced struct and the original argument type, which the body reads. |
| 64 | // The call site reads each field of the coerced struct out of memory holding |
| 65 | // the operand as a separate call argument. |
| 66 | |
| 67 | namespace { |
| 68 | |
| 69 | /// A pointer to \p pointee in target address space \p addrSpace. Zero is the |
| 70 | /// default address space, which carries no attribute. |
| 71 | cir::PointerType getPtrTyInAddrSpace(mlir::Type pointee, unsigned addrSpace) { |
| 72 | mlir::ptr::MemorySpaceAttrInterface asAttr; |
| 73 | if (addrSpace) |
| 74 | asAttr = cir::TargetAddressSpaceAttr::get(context: pointee.getContext(), value: addrSpace); |
| 75 | return cir::PointerType::get(pointee, addrSpace: asAttr); |
| 76 | } |
| 77 | |
| 78 | /// The pointer passed in place of an Indirect or byref value, or an sret |
| 79 | /// return, of type \p pointee. |
| 80 | cir::PointerType getIndirectPtrTy(mlir::Type pointee, |
| 81 | const ArgClassification &ac) { |
| 82 | assert( |
| 83 | (ac.kind == ArgKind::Indirect || ac.kind == ArgKind::IndirectAliased) && |
| 84 | "expected an indirect classification" ); |
| 85 | return getPtrTyInAddrSpace(pointee, addrSpace: ac.indirectAddrSpace); |
| 86 | } |
| 87 | |
| 88 | /// The struct a union is flattened as: its storage member, then its tail |
| 89 | /// padding if it has any. This matches the union's LLVM type. |
| 90 | cir::StructType getUnionFlattenType(cir::UnionType unionTy, |
| 91 | const mlir::DataLayout &dl) { |
| 92 | SmallVector<mlir::Type, 2> fields; |
| 93 | if (mlir::Type storage = unionTy.getUnionStorageType(dataLayout: dl)) |
| 94 | fields.push_back(Elt: storage); |
| 95 | if (mlir::Type padding = unionTy.getPadding()) |
| 96 | fields.push_back(Elt: padding); |
| 97 | return cir::StructType::get(context: unionTy.getContext(), members: fields, packed: unionTy.getPacked(), |
| 98 | /*is_class=*/false, |
| 99 | member_kinds: cir::RecordType::getAllDataKinds(members: fields)); |
| 100 | } |
| 101 | |
| 102 | /// Return the coerced RecordType for a Direct classification that should be |
| 103 | /// flattened into individual scalar arguments, or a null type if the |
| 104 | /// classification does not call for flattening. |
| 105 | /// |
| 106 | /// Flattening applies when all four conditions hold: |
| 107 | /// 1. The classification is Direct with a non-null coercedType. |
| 108 | /// 2. canFlatten is set. |
| 109 | /// 3. The coercedType is a struct, or a union, which is flattened as the |
| 110 | /// struct getUnionFlattenType returns. |
| 111 | /// 4. The struct has at least one field. A struct of one field is passed as |
| 112 | /// that field. |
| 113 | cir::RecordType getFlattenedCoercedType(const ArgClassification &ac, |
| 114 | const mlir::DataLayout &dl) { |
| 115 | if (ac.kind != ArgKind::Direct || !ac.coercedType || !ac.canFlatten) |
| 116 | return {}; |
| 117 | auto recTy = dyn_cast<cir::RecordType>(Val: ac.coercedType); |
| 118 | if (auto unionTy = dyn_cast_if_present<cir::UnionType>(Val&: recTy)) |
| 119 | recTy = getUnionFlattenType(unionTy, dl); |
| 120 | if (!recTy || !recTy.isStruct() || recTy.getMembers().empty()) |
| 121 | return {}; |
| 122 | return recTy; |
| 123 | } |
| 124 | |
| 125 | /// Whether \p ac is an Extend of a record argument of type \p origTy. Such an |
| 126 | /// argument is a transparent union, passed as its first member, which |
| 127 | /// \p ac.coercedType holds. |
| 128 | bool isRecordExtend(const ArgClassification &ac, mlir::Type origTy) { |
| 129 | if (ac.kind != ArgKind::Extend || !mlir::isa<cir::RecordType>(Val: origTy)) |
| 130 | return false; |
| 131 | assert(ac.coercedType && "an Extend of a record names its first member type" ); |
| 132 | return true; |
| 133 | } |
| 134 | |
| 135 | /// Build the new argument-type list for a function whose ABI classification |
| 136 | /// is \p fc. Handles Direct (with or without coercion), Extend, Ignore, |
| 137 | /// Indirect (byval and non-byval), IndirectAliased (byref), and Expand |
| 138 | /// (struct flattening) arguments. |
| 139 | /// The sret return pointer, when present, is prepended by |
| 140 | /// rewriteFunctionDefinition rather than here. |
| 141 | mlir::LogicalResult buildNewArgTypes( |
| 142 | ArrayRef<mlir::Type> oldArgTypes, const FunctionClassification &fc, |
| 143 | SmallVectorImpl<mlir::Type> &newArgTypes, const mlir::DataLayout &dl, |
| 144 | function_ref<mlir::InFlightDiagnostic()> emitError) { |
| 145 | assert(newArgTypes.empty() && "expected an empty output vector" ); |
| 146 | newArgTypes.reserve(N: oldArgTypes.size()); |
| 147 | for (auto [idx, ac] : llvm::enumerate(First: fc.argInfos)) { |
| 148 | mlir::Type origTy = oldArgTypes[idx]; |
| 149 | switch (ac.kind) { |
| 150 | case ArgKind::Direct: |
| 151 | // Direct with canFlatten and a struct coerced type: push one wire type |
| 152 | // per field of the coerced struct rather than the struct itself. |
| 153 | if (cir::RecordType flatTy = getFlattenedCoercedType(ac, dl)) { |
| 154 | llvm::append_range(C&: newArgTypes, R: flatTy.getMembers()); |
| 155 | } else { |
| 156 | // Direct with a coerced type: the lowered signature uses the coerced |
| 157 | // type; the body still expects origTy and insertArgCoercion recovers it |
| 158 | // from a coercion slot. Direct without coercion is a pass-through. |
| 159 | newArgTypes.push_back(Elt: ac.coercedType ? ac.coercedType : origTy); |
| 160 | } |
| 161 | break; |
| 162 | case ArgKind::Ignore: |
| 163 | break; |
| 164 | case ArgKind::Expand: { |
| 165 | // Flatten the struct into one wire argument per field. The |
| 166 | // reassembly in the callee body and the decomposition at the call |
| 167 | // site are handled by insertArgCoercion and rewriteCallSite. |
| 168 | auto recTy = cast<cir::RecordType>(Val&: origTy); |
| 169 | assert(recTy.isStruct() && |
| 170 | "Expand classification requires a struct type, not a union" ); |
| 171 | assert(!recTy.getMembers().empty() && |
| 172 | "Expand classification requires at least one struct field" ); |
| 173 | llvm::append_range(C&: newArgTypes, R: recTy.getMembers()); |
| 174 | break; |
| 175 | } |
| 176 | case ArgKind::Extend: |
| 177 | // Extend keeps the original (narrow) type in the signature; the |
| 178 | // sign/zero extension is communicated to LLVM via the llvm.signext / |
| 179 | // llvm.zeroext arg attribute, attached separately below. For an integer |
| 180 | // the coercedType the classifier set is informational (typically the |
| 181 | // register-width type the value gets extended to in registers) and does |
| 182 | // not change the CIR signature. A transparent union is passed as the |
| 183 | // type of its first member, which coercedType holds. |
| 184 | newArgTypes.push_back(Elt: isRecordExtend(ac, origTy) ? ac.coercedType |
| 185 | : origTy); |
| 186 | break; |
| 187 | case ArgKind::Indirect: |
| 188 | case ArgKind::IndirectAliased: |
| 189 | newArgTypes.push_back(Elt: getIndirectPtrTy(pointee: origTy, ac)); |
| 190 | break; |
| 191 | } |
| 192 | } |
| 193 | return mlir::success(); |
| 194 | } |
| 195 | |
| 196 | /// Compute the new return type for a function whose return classification |
| 197 | /// is \p retInfo. Direct returns keep (or coerce to) their type, Ignore and |
| 198 | /// Indirect (sret) returns become void, Extend keeps its type; Expand and |
| 199 | /// IndirectAliased emit an error. |
| 200 | mlir::Type |
| 201 | computeNewReturnType(mlir::Type origRetTy, const ArgClassification &retInfo, |
| 202 | mlir::MLIRContext *ctx, |
| 203 | function_ref<mlir::InFlightDiagnostic()> emitError) { |
| 204 | switch (retInfo.kind) { |
| 205 | case ArgKind::Direct: |
| 206 | // Direct return with a coerced type uses the coerced type on the wire; |
| 207 | // the rewriter inserts a coercion before each cir.return. |
| 208 | return retInfo.coercedType ? retInfo.coercedType : origRetTy; |
| 209 | case ArgKind::Ignore: |
| 210 | return cir::VoidType::get(ctx); |
| 211 | case ArgKind::Expand: |
| 212 | emitError() << "Expand return is not allowed (classic codegen rejects " |
| 213 | << "it in EmitFunctionEpilog)" ; |
| 214 | return nullptr; |
| 215 | case ArgKind::IndirectAliased: |
| 216 | emitError() << "IndirectAliased return is not supported" ; |
| 217 | return nullptr; |
| 218 | case ArgKind::Extend: |
| 219 | // Same convention as Extend args: keep the original return type in the |
| 220 | // signature; the sign/zero extension is communicated via the |
| 221 | // llvm.signext / llvm.zeroext res attribute attached separately below. |
| 222 | return origRetTy; |
| 223 | case ArgKind::Indirect: |
| 224 | // sret: the value is returned through a pointer argument that the ABI |
| 225 | // synthesizes (rewriteFunctionDefinition prepends it to the argument |
| 226 | // list); it is not part of the source-level signature, so the wire |
| 227 | // return type becomes void. |
| 228 | return cir::VoidType::get(ctx); |
| 229 | } |
| 230 | llvm_unreachable("all ArgKind cases handled" ); |
| 231 | } |
| 232 | |
| 233 | /// Create a typed poison constant to stand in for a value the body of a |
| 234 | /// function (or the result of a call) still references but whose ABI |
| 235 | /// classification is Ignore. Using poison is honest -- the value is |
| 236 | /// genuinely unused at the ABI boundary -- and avoids a fake alloca+load |
| 237 | /// pattern that would suggest we have a value when we don't. |
| 238 | mlir::Value createIgnoredValue(mlir::OpBuilder &builder, mlir::Location loc, |
| 239 | mlir::Type ty) { |
| 240 | return cir::ConstantOp::create(builder, location: loc, res: ty, value: cir::PoisonAttr::get(type: ty)); |
| 241 | } |
| 242 | |
| 243 | /// Whether \p coercedTy is wider than the type a value of type \p origTy has |
| 244 | /// in memory, so the coerced value passes bits the value does not define and |
| 245 | /// cannot be noundef. This is the comparison classic CodeGen makes. A bool |
| 246 | /// vector's memory type is its storage integer, one bit per element and at |
| 247 | /// least a byte wide, not its power-of-two size. |
| 248 | static bool isCoercionWiderThanMemory(mlir::Type origTy, mlir::Type coercedTy, |
| 249 | const mlir::DataLayout &dl) { |
| 250 | uint64_t memoryBits = dl.getTypeSizeInBits(t: origTy).getFixedValue(); |
| 251 | if (auto vecTy = mlir::dyn_cast<cir::VectorType>(Val&: origTy); |
| 252 | vecTy && mlir::isa<cir::BoolType>(Val: vecTy.getElementType())) |
| 253 | memoryBits = vecTy.getBoolStorageWidth(); |
| 254 | return dl.getTypeSizeInBits(t: coercedTy).getFixedValue() > memoryBits; |
| 255 | } |
| 256 | |
| 257 | /// Whether updateArgAttrs has anything to change: an arg that is Ignore (its |
| 258 | /// slot is dropped), Extend (needs llvm.signext / llvm.zeroext), Indirect or |
| 259 | /// IndirectAliased (needs the pointer attributes), Expand (changes the argument |
| 260 | /// count), or a coerced Direct, which may be flattened or may lose |
| 261 | /// llvm.noundef, or that has a stack-slot alignment, which needs |
| 262 | /// llvm.alignstack. |
| 263 | static bool needsArgAttrUpdate(const FunctionClassification &fc) { |
| 264 | return llvm::any_of(Range: fc.argInfos, P: [](const ArgClassification &ac) { |
| 265 | return ac.kind == ArgKind::Ignore || ac.kind == ArgKind::Extend || |
| 266 | ac.kind == ArgKind::Indirect || |
| 267 | ac.kind == ArgKind::IndirectAliased || ac.kind == ArgKind::Expand || |
| 268 | (ac.kind == ArgKind::Direct && (ac.coercedType || ac.directAlign)); |
| 269 | }); |
| 270 | } |
| 271 | |
| 272 | /// Build an updated arg_attrs ArrayAttr that drops Ignore'd args, adds |
| 273 | /// llvm.signext / llvm.zeroext on Extend args, adds the pointer attributes |
| 274 | /// for Indirect and IndirectAliased args in place of llvm.nofpclass, drops |
| 275 | /// llvm.noundef from a coerced Direct arg that passes undefined bits, |
| 276 | /// adds llvm.alignstack on a Direct arg with a stack-slot alignment, gives each |
| 277 | /// field of a Direct+canFlatten argument the argument's attributes, and gives |
| 278 | /// each field of an Expand argument an empty dict. When \p enableNoundefAttrs |
| 279 | /// is set, Extend, Indirect and IndirectAliased args are also marked |
| 280 | /// llvm.noundef. Direct and Extend slots keep their other existing attributes. |
| 281 | /// \p origArgTypes provides the pre-rewrite type for each arg slot. |
| 282 | mlir::ArrayAttr updateArgAttrs(mlir::MLIRContext *ctx, mlir::ModuleOp module, |
| 283 | ArrayRef<mlir::Type> origArgTypes, |
| 284 | mlir::ArrayAttr existingArgAttrs, |
| 285 | const FunctionClassification &fc, |
| 286 | const mlir::DataLayout &dl, |
| 287 | bool enableNoundefAttrs, bool isCallSite) { |
| 288 | mlir::Builder builder(ctx); |
| 289 | SmallVector<mlir::Attribute> newArgAttrs; |
| 290 | newArgAttrs.reserve(N: fc.argInfos.size()); |
| 291 | for (auto [oldIdx, ac] : llvm::enumerate(First: fc.argInfos)) { |
| 292 | if (ac.kind == ArgKind::Ignore) |
| 293 | continue; |
| 294 | mlir::DictionaryAttr existing = builder.getDictionaryAttr(value: {}); |
| 295 | if (existingArgAttrs && oldIdx < existingArgAttrs.size()) |
| 296 | existing = mlir::cast<mlir::DictionaryAttr>(Val: existingArgAttrs[oldIdx]); |
| 297 | if (ac.kind == ArgKind::Direct && ac.coercedType && |
| 298 | isCoercionWiderThanMemory(origTy: origArgTypes[oldIdx], coercedTy: ac.coercedType, dl)) { |
| 299 | mlir::NamedAttrList attrs(existing); |
| 300 | attrs.erase(name: mlir::LLVM::LLVMDialect::getNoUndefAttrName()); |
| 301 | existing = attrs.getDictionary(context: ctx); |
| 302 | } |
| 303 | if (ac.kind == ArgKind::Direct && ac.directAlign) { |
| 304 | mlir::NamedAttrList attrs(existing); |
| 305 | attrs.set(name: mlir::LLVM::LLVMDialect::getStackAlignmentAttrName(), |
| 306 | value: builder.getI64IntegerAttr(value: ac.directAlign->value())); |
| 307 | existing = attrs.getDictionary(context: ctx); |
| 308 | } |
| 309 | if (cir::RecordType flatTy = getFlattenedCoercedType(ac, dl)) { |
| 310 | // Direct + canFlatten: as in classic CodeGen, each flattened field |
| 311 | // carries the attributes of the argument it came from. |
| 312 | newArgAttrs.append(NumInputs: flatTy.getNumElements(), Elt: existing); |
| 313 | } else if (ac.kind == ArgKind::Expand) { |
| 314 | // An expanded record has no attributes classic CodeGen would copy to its |
| 315 | // fields, so each field gets an empty dict. |
| 316 | auto recTy = cast<cir::RecordType>(Val: origArgTypes[oldIdx]); |
| 317 | newArgAttrs.append(NumInputs: recTy.getNumElements(), Elt: builder.getDictionaryAttr(value: {})); |
| 318 | } else if (ac.kind == ArgKind::Extend) { |
| 319 | // An extended value has no undefined bits. This covers a transparent |
| 320 | // union passed as its extended first member, which CIRGen does not |
| 321 | // mark noundef because it is a record. |
| 322 | StringRef attrName = ac.signExtend |
| 323 | ? mlir::LLVM::LLVMDialect::getSExtAttrName() |
| 324 | : mlir::LLVM::LLVMDialect::getZExtAttrName(); |
| 325 | mlir::NamedAttrList attrs(existing); |
| 326 | // Conservative argument extends apply only at the call-site. |
| 327 | if (isCallSite || !ac.conservative) |
| 328 | attrs.set(name: attrName, value: builder.getUnitAttr()); |
| 329 | if (enableNoundefAttrs) |
| 330 | attrs.set(name: mlir::LLVM::LLVMDialect::getNoUndefAttrName(), |
| 331 | value: builder.getUnitAttr()); |
| 332 | newArgAttrs.push_back(Elt: attrs.getDictionary(context: ctx)); |
| 333 | } else if (ac.kind == ArgKind::Indirect) { |
| 334 | // Indirect lowering hands the callee a pointer. llvm.align and |
| 335 | // llvm.noundef describe that pointer, not the bytes it points to, and |
| 336 | // llvm.nofpclass, which describes the value, does not apply to a |
| 337 | // pointer. Without byval the pointer points to the caller's own object. |
| 338 | // With byval the backend copies the pointee on the caller's side of the |
| 339 | // call, so the callee gets an object of its own even when the pointer |
| 340 | // points to the caller's own object. |
| 341 | mlir::Type pointeeTy = origArgTypes[oldIdx]; |
| 342 | mlir::NamedAttrList attrs(existing); |
| 343 | attrs.erase(name: mlir::LLVM::LLVMDialect::getNoFPClassAttrName()); |
| 344 | attrs.set(name: mlir::LLVM::LLVMDialect::getAlignAttrName(), |
| 345 | value: builder.getI64IntegerAttr(value: ac.indirectAlign.value())); |
| 346 | if (enableNoundefAttrs) |
| 347 | attrs.set(name: mlir::LLVM::LLVMDialect::getNoUndefAttrName(), |
| 348 | value: builder.getUnitAttr()); |
| 349 | if (ac.byVal) { |
| 350 | // llvm.byval(T) records the pre-rewrite arg type because the opaque |
| 351 | // LLVM pointer cannot carry it. Classic adds llvm.noalias under |
| 352 | // -fpass-by-value-is-noalias, which CIR does not plumb through. |
| 353 | assert(!cir::MissingFeatures::noaliasOnByvalAttr()); |
| 354 | attrs.set(name: mlir::LLVM::LLVMDialect::getByValAttrName(), |
| 355 | value: mlir::TypeAttr::get(type: pointeeTy)); |
| 356 | } else { |
| 357 | // llvm.nofreeobj says the object cannot be freed while the callee |
| 358 | // runs, which holds because the caller owns it across the call. |
| 359 | attrs.set(name: mlir::LLVM::LLVMDialect::getNoFreeObjAttrName(), |
| 360 | value: builder.getUnitAttr()); |
| 361 | // llvm.dead_on_return says the object's lifetime ends in the callee, |
| 362 | // which holds unless the caller still has a destructor to run on it. |
| 363 | // A record with no layout entry has no destructor. |
| 364 | auto recTy = mlir::dyn_cast<cir::RecordType>(Val&: pointeeTy); |
| 365 | cir::RecordLayoutAttr layout = |
| 366 | recTy ? cir::tryGetRecordLayout(mod: module, name: recTy.getName()) |
| 367 | : cir::RecordLayoutAttr(); |
| 368 | if (!layout || layout.getHasTrivialDtor()) |
| 369 | attrs.set(name: mlir::LLVM::LLVMDialect::getDeadOnReturnAttrName(), |
| 370 | value: builder.getI64IntegerAttr(value: static_cast<int64_t>( |
| 371 | llvm::DeadOnReturnInfo().toIntValue()))); |
| 372 | attrs.set(name: mlir::LLVM::LLVMDialect::getDereferenceableAttrName(), |
| 373 | value: builder.getI64IntegerAttr( |
| 374 | value: dl.getTypeAllocSize(t: pointeeTy).getFixedValue())); |
| 375 | } |
| 376 | newArgAttrs.push_back(Elt: attrs.getDictionary(context: ctx)); |
| 377 | } else if (ac.kind == ArgKind::IndirectAliased) { |
| 378 | // llvm.byref(T) records the pre-rewrite arg type, as llvm.byval does. |
| 379 | // llvm.nofpclass describes the value and does not apply to a pointer. |
| 380 | mlir::NamedAttrList attrs(existing); |
| 381 | attrs.erase(name: mlir::LLVM::LLVMDialect::getNoFPClassAttrName()); |
| 382 | attrs.set(name: mlir::LLVM::LLVMDialect::getAlignAttrName(), |
| 383 | value: builder.getI64IntegerAttr(value: ac.indirectAlign.value())); |
| 384 | if (enableNoundefAttrs) |
| 385 | attrs.set(name: mlir::LLVM::LLVMDialect::getNoUndefAttrName(), |
| 386 | value: builder.getUnitAttr()); |
| 387 | attrs.set(name: mlir::LLVM::LLVMDialect::getByRefAttrName(), |
| 388 | value: mlir::TypeAttr::get(type: origArgTypes[oldIdx])); |
| 389 | newArgAttrs.push_back(Elt: attrs.getDictionary(context: ctx)); |
| 390 | } else { |
| 391 | newArgAttrs.push_back(Elt: existing); |
| 392 | } |
| 393 | } |
| 394 | return builder.getArrayAttr(value: newArgAttrs); |
| 395 | } |
| 396 | |
| 397 | /// Build an updated res_attrs ArrayAttr (single entry, since CIR funcs have |
| 398 | /// at most one result) that adds llvm.signext / llvm.zeroext on an Extend |
| 399 | /// return and drops llvm.noundef from a coerced return that passes undefined |
| 400 | /// bits. Preserves any other existing res attributes. Returns |
| 401 | /// \p existingResAttrs unchanged, which may be null, when there is nothing to |
| 402 | /// add or drop. |
| 403 | mlir::ArrayAttr updateResAttrs(mlir::MLIRContext *ctx, |
| 404 | mlir::ArrayAttr existingResAttrs, |
| 405 | const ArgClassification &retInfo, |
| 406 | mlir::Type origRetTy, |
| 407 | const mlir::DataLayout &dl) { |
| 408 | bool needsNoUndefDrop = |
| 409 | retInfo.kind == ArgKind::Direct && retInfo.coercedType && |
| 410 | isCoercionWiderThanMemory(origTy: origRetTy, coercedTy: retInfo.coercedType, dl); |
| 411 | if (retInfo.kind != ArgKind::Extend && !needsNoUndefDrop) |
| 412 | return existingResAttrs; |
| 413 | |
| 414 | mlir::NamedAttrList attrs; |
| 415 | if (existingResAttrs && !existingResAttrs.empty()) |
| 416 | attrs.append( |
| 417 | newAttributes: mlir::cast<mlir::DictionaryAttr>(Val: existingResAttrs[0]).getValue()); |
| 418 | if (retInfo.kind == ArgKind::Extend) |
| 419 | attrs.set(name: retInfo.signExtend ? "llvm.signext" : "llvm.zeroext" , |
| 420 | value: mlir::UnitAttr::get(context: ctx)); |
| 421 | if (needsNoUndefDrop) |
| 422 | attrs.erase(name: mlir::LLVM::LLVMDialect::getNoUndefAttrName()); |
| 423 | return mlir::ArrayAttr::get(context: ctx, value: {attrs.getDictionary(context: ctx)}); |
| 424 | } |
| 425 | |
| 426 | /// The number of bytes a coercion memory slot needs to hold a value of type |
| 427 | /// \p ty without truncating it. For most types this is the ordinary storage |
| 428 | /// size. For a _BitInt it is deliberately the value's own literal byte |
| 429 | /// footprint (ceil(width/8)) rather than the wider, ABI-alignment-padded |
| 430 | /// footprint a _BitInt gets as a record member (see |
| 431 | /// cir::IntType::getStorageTypeWidth): this coercion is about how many bytes |
| 432 | /// the *value* needs to round-trip, not how a record would lay it out, and |
| 433 | /// those are genuinely different questions for a _BitInt (e.g. _BitInt(33) |
| 434 | /// only needs 5 bytes here, even though it occupies 8 padded bytes as a |
| 435 | /// record member). |
| 436 | static uint64_t coercionByteSize(mlir::Type ty, const mlir::DataLayout &dl) { |
| 437 | if (auto intTy = mlir::dyn_cast<cir::IntType>(Val&: ty)) |
| 438 | return llvm::divideCeil(Numerator: intTy.getWidth(), Denominator: 8); |
| 439 | return dl.getTypeSize(t: ty); |
| 440 | } |
| 441 | |
| 442 | /// A \p wantTy-typed view of the storage \p base points at. When \p wantTy is |
| 443 | /// \p base's own pointee type, \p base is returned unchanged and \p offset must |
| 444 | /// be 0. Any other type is reached by a bitcast, preceded by a byte stride |
| 445 | /// of \p offset bytes when \p offset is nonzero. The view keeps \p base's |
| 446 | /// address space. Any operations created are added to \p createdOps. |
| 447 | static mlir::Value |
| 448 | emitViewAtOffset(mlir::OpBuilder &builder, mlir::Location loc, |
| 449 | mlir::Type wantTy, mlir::Value base, |
| 450 | SmallPtrSetImpl<mlir::Operation *> &createdOps, |
| 451 | unsigned offset) { |
| 452 | auto basePtrTy = mlir::cast<cir::PointerType>(Val: base.getType()); |
| 453 | if (wantTy == basePtrTy.getPointee()) { |
| 454 | assert(offset == 0 && "a view of the storage's own type has no offset" ); |
| 455 | return base; |
| 456 | } |
| 457 | mlir::ptr::MemorySpaceAttrInterface addrSpace = basePtrTy.getAddrSpace(); |
| 458 | mlir::Value view = base; |
| 459 | if (offset != 0) { |
| 460 | auto u8Ty = cir::IntType::get(context: builder.getContext(), width: 8, /*isSigned=*/is_signed: false); |
| 461 | auto u8PtrTy = cir::PointerType::get(pointee: u8Ty, addrSpace); |
| 462 | auto u8Base = cir::CastOp::create(builder, location: loc, result: u8PtrTy, |
| 463 | kind: cir::CastKind::bitcast, src: base); |
| 464 | createdOps.insert(Ptr: u8Base); |
| 465 | auto strideTy = |
| 466 | cir::IntType::get(context: builder.getContext(), width: 64, /*isSigned=*/is_signed: true); |
| 467 | auto strideVal = cir::ConstantOp::create( |
| 468 | builder, location: loc, value: cir::IntAttr::get(type: strideTy, value: offset)); |
| 469 | createdOps.insert(Ptr: strideVal); |
| 470 | view = cir::PtrStrideOp::create(builder, location: loc, result: u8PtrTy, base: u8Base, stride: strideVal); |
| 471 | createdOps.insert(Ptr: view.getDefiningOp()); |
| 472 | } |
| 473 | auto typedView = cir::CastOp::create(builder, location: loc, |
| 474 | result: cir::PointerType::get(pointee: wantTy, addrSpace), |
| 475 | kind: cir::CastKind::bitcast, src: view); |
| 476 | createdOps.insert(Ptr: typedView); |
| 477 | return typedView; |
| 478 | } |
| 479 | |
| 480 | /// A pointer to \p pointee in the target's alloca address space, which every |
| 481 | /// temporary this pass allocates has to live in. |
| 482 | static cir::PointerType getAllocaPtrTy(mlir::Type pointee, |
| 483 | const mlir::DataLayout &dl) { |
| 484 | mlir::ptr::MemorySpaceAttrInterface addrSpace; |
| 485 | if (auto asAttr = mlir::dyn_cast_if_present<mlir::IntegerAttr>( |
| 486 | Val: dl.getAllocaMemorySpace())) |
| 487 | addrSpace = cir::TargetAddressSpaceAttr::get(context: pointee.getContext(), |
| 488 | value: asAttr.getUInt()); |
| 489 | return cir::PointerType::get(pointee, addrSpace); |
| 490 | } |
| 491 | |
| 492 | /// A coercion slot for \p srcTy and \p dstTy, which must differ. It is typed |
| 493 | /// as \p srcTy unless coercionByteSize says \p dstTy is larger, aligned for |
| 494 | /// both, and placed at the start of \p slotBlock. \p offset, where the |
| 495 | /// coerced side sits within the slot, is only checked by assertions. The |
| 496 | /// alloca is added to \p createdOps. |
| 497 | static cir::AllocaOp createCoercionSlot( |
| 498 | mlir::OpBuilder &builder, mlir::Location loc, mlir::Type srcTy, |
| 499 | mlir::Type dstTy, mlir::Block *slotBlock, const mlir::DataLayout &dl, |
| 500 | SmallPtrSetImpl<mlir::Operation *> &createdOps, unsigned offset) { |
| 501 | assert(srcTy != dstTy && "callers must pre-check that the types differ" ); |
| 502 | |
| 503 | cir::CIRDataLayout cirDataLayout( |
| 504 | slotBlock->getParentOp()->getParentOfType<mlir::ModuleOp>()); |
| 505 | uint64_t srcAlign = cirDataLayout.getTypeLanguageAlignment(ty: srcTy); |
| 506 | uint64_t dstAlign = cirDataLayout.getTypeLanguageAlignment(ty: dstTy); |
| 507 | uint64_t allocaAlign = std::max(a: srcAlign, b: dstAlign); |
| 508 | mlir::Type slotTy = coercionByteSize(ty: srcTy, dl) >= coercionByteSize(ty: dstTy, dl) |
| 509 | ? srcTy |
| 510 | : dstTy; |
| 511 | |
| 512 | // Sizes are compared two ways on purpose: relative size in |
| 513 | // coercionByteSize terms, which is how slotTy was picked, and capacity in |
| 514 | // getTypeSize terms, which is what the alloca is given. |
| 515 | [[maybe_unused]] mlir::Type coercedTy = ((slotTy == srcTy) ? dstTy : srcTy); |
| 516 | assert((offset == 0 || |
| 517 | coercionByteSize(coercedTy, dl) < coercionByteSize(slotTy, dl)) && |
| 518 | "a direct offset must land on the coerced side, the smaller one" ); |
| 519 | assert((offset == 0 || |
| 520 | offset + dl.getTypeSize(coercedTy) <= dl.getTypeSize(slotTy)) && |
| 521 | "coerce slot too small for offset access" ); |
| 522 | assert((offset == 0 || |
| 523 | offset % cirDataLayout.getTypeLanguageAlignment(coercedTy) == 0) && |
| 524 | "a direct offset must be aligned for the coerced access" ); |
| 525 | |
| 526 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 527 | builder.setInsertionPointToStart(slotBlock); |
| 528 | auto alloca = cir::AllocaOp::create(builder, location: loc, addr: getAllocaPtrTy(pointee: slotTy, dl), |
| 529 | name: builder.getStringAttr(bytes: "coerce" ), |
| 530 | alignment: builder.getI64IntegerAttr(value: allocaAlign)); |
| 531 | createdOps.insert(Ptr: alloca); |
| 532 | return alloca; |
| 533 | } |
| 534 | |
| 535 | /// Coerce \p src into a temporary memory slot typed for \p dstTy at the |
| 536 | /// current builder insertion point, and return the destination-typed pointer |
| 537 | /// to that slot without loading the value back out. This is the shared |
| 538 | /// memory half of emitCoercion: callers that want the whole coerced value use |
| 539 | /// emitCoercion (below); callers that want to read individual members of a |
| 540 | /// coerced struct (the call-site struct flattening) take the returned pointer |
| 541 | /// and emit their own cir.get_member + cir.load per field. Lowers uniformly |
| 542 | /// for scalar, vector, and record types. |
| 543 | /// |
| 544 | /// The slot is sized to the larger of the two types so that neither the store |
| 545 | /// nor a later load ever runs past it: the coerced ABI type can be larger |
| 546 | /// than the original (e.g. a 12-byte aggregate passed as `{i64, i64}`), so |
| 547 | /// accessing the destination through a source-sized slot would over-read. |
| 548 | /// Alignment is max(srcAlign, dstAlign) to satisfy both accesses. The slot |
| 549 | /// is written through a source-typed view and returned as a destination-typed |
| 550 | /// view. |
| 551 | /// |
| 552 | /// The temporary alloca is placed at the start of \p slotBlock, which must |
| 553 | /// dominate every use of the coerced value and must be a block that ends up |
| 554 | /// inside the enclosing function's entry block after any later outlining. |
| 555 | /// |
| 556 | /// \p offset is where the coerced value sits within the larger of the two |
| 557 | /// types, non-zero when the ABI passes a value in a register read from |
| 558 | /// partway into its storage. The coerced type must be the smaller of the |
| 559 | /// two, so one that can exceed the value it coerces, such as a multi-field |
| 560 | /// register tuple, must not be given an offset. |
| 561 | /// |
| 562 | /// Any operations the helper creates are appended to \p createdOps so the |
| 563 | /// caller can leave them out when it later rewires the source value's uses, |
| 564 | /// which would otherwise clobber the store's value operand. |
| 565 | mlir::Value emitCoercionToMemory(mlir::OpBuilder &builder, mlir::Location loc, |
| 566 | mlir::Type dstTy, mlir::Value src, |
| 567 | mlir::Block *slotBlock, |
| 568 | const mlir::DataLayout &dl, |
| 569 | SmallPtrSetImpl<mlir::Operation *> &createdOps, |
| 570 | unsigned offset) { |
| 571 | mlir::Type srcTy = src.getType(); |
| 572 | cir::AllocaOp alloca = createCoercionSlot(builder, loc, srcTy, dstTy, |
| 573 | slotBlock, dl, createdOps, offset); |
| 574 | mlir::Type slotTy = alloca.getAllocaType(); |
| 575 | |
| 576 | // Store through a source-typed view of the slot. |
| 577 | mlir::Value srcSlot = emitViewAtOffset( |
| 578 | builder, loc, wantTy: srcTy, base: alloca, createdOps, offset: srcTy == slotTy ? 0 : offset); |
| 579 | auto store = cir::StoreOp::create(builder, location: loc, value: src, addr: srcSlot); |
| 580 | createdOps.insert(Ptr: store); |
| 581 | |
| 582 | // Return a destination-typed view of the slot. |
| 583 | return emitViewAtOffset(builder, loc, wantTy: dstTy, base: alloca, createdOps, |
| 584 | offset: dstTy == slotTy ? 0 : offset); |
| 585 | } |
| 586 | |
| 587 | /// Cast \p src to \p dstTy when both are pointers to the same pointee, so only |
| 588 | /// the address space changes. Returns null when a real coercion is needed. |
| 589 | static mlir::Value |
| 590 | emitAddrSpaceCoercion(mlir::OpBuilder &builder, mlir::Location loc, |
| 591 | mlir::Type dstTy, mlir::Value src, |
| 592 | SmallPtrSetImpl<mlir::Operation *> &createdOps, |
| 593 | unsigned offset) { |
| 594 | auto srcPtrTy = dyn_cast<cir::PointerType>(Val: src.getType()); |
| 595 | auto dstPtrTy = dyn_cast<cir::PointerType>(Val&: dstTy); |
| 596 | if (offset || !srcPtrTy || !dstPtrTy || |
| 597 | srcPtrTy.getPointee() != dstPtrTy.getPointee()) |
| 598 | return {}; |
| 599 | assert(srcPtrTy.getAddrSpace() != dstPtrTy.getAddrSpace() && |
| 600 | "pointers to the same pointee must differ in address space" ); |
| 601 | auto cast = cir::CastOp::create(builder, location: loc, result: dstTy, |
| 602 | kind: cir::CastKind::address_space, src); |
| 603 | createdOps.insert(Ptr: cast); |
| 604 | return cast; |
| 605 | } |
| 606 | |
| 607 | /// Coerce \p src to type \p dstTy by going through memory and load the whole |
| 608 | /// coerced value back out. Builds on emitCoercionToMemory, adding the final |
| 609 | /// load of the destination-typed view. A pointer that only changes address |
| 610 | /// space is cast instead. |
| 611 | mlir::Value emitCoercion(mlir::OpBuilder &builder, mlir::Location loc, |
| 612 | mlir::Type dstTy, mlir::Value src, |
| 613 | mlir::Block *slotBlock, const mlir::DataLayout &dl, |
| 614 | SmallPtrSetImpl<mlir::Operation *> &createdOps, |
| 615 | unsigned offset) { |
| 616 | if (mlir::Value cast = |
| 617 | emitAddrSpaceCoercion(builder, loc, dstTy, src, createdOps, offset)) |
| 618 | return cast; |
| 619 | |
| 620 | mlir::Value dstSlot = emitCoercionToMemory(builder, loc, dstTy, src, |
| 621 | slotBlock, dl, createdOps, offset); |
| 622 | auto load = cir::LoadOp::create(builder, location: loc, addr: dstSlot); |
| 623 | createdOps.insert(Ptr: load); |
| 624 | return load; |
| 625 | } |
| 626 | |
| 627 | /// Convenience overload for callers that don't need the createdOps set |
| 628 | /// (e.g. call-site coercion where we don't replaceAllUsesExcept). |
| 629 | mlir::Value emitCoercion(mlir::OpBuilder &builder, mlir::Location loc, |
| 630 | mlir::Type dstTy, mlir::Value src, |
| 631 | mlir::Block *slotBlock, const mlir::DataLayout &dl, |
| 632 | unsigned offset) { |
| 633 | SmallPtrSet<mlir::Operation *, 4> ignored; |
| 634 | return emitCoercion(builder, loc, dstTy, src, slotBlock, dl, createdOps&: ignored, offset); |
| 635 | } |
| 636 | |
| 637 | /// The block a coercion slot's alloca belongs at the start of. |
| 638 | /// |
| 639 | /// Normally the enclosing function's entry block, where HoistAllocas expects |
| 640 | /// allocas to be. A body carrying a call is not always inside a function |
| 641 | /// when this pass runs, though, because LoweringPrepare runs after it: a |
| 642 | /// namespace-scope `T g = makeT();` is still in its cir.global ctor region, |
| 643 | /// and an OpenACC recipe's init and destroy bodies are in regions the module |
| 644 | /// owns. Those take the outermost region below the module, which dominates |
| 645 | /// the whole body and travels with it when the body is outlined. |
| 646 | mlir::Block *coercionSlotBlock(mlir::Operation *op) { |
| 647 | if (auto funcOp = op->getParentOfType<mlir::FunctionOpInterface>()) |
| 648 | return &funcOp->getRegion(index: 0).front(); |
| 649 | mlir::Region *region = op->getParentRegion(); |
| 650 | while (mlir::Region *outer = region->getParentRegion()) { |
| 651 | if (mlir::isa<mlir::ModuleOp>(Val: outer->getParentOp())) |
| 652 | break; |
| 653 | region = outer; |
| 654 | } |
| 655 | assert(!region->empty() && "coercion slot needs a block to hold the alloca" ); |
| 656 | return ®ion->front(); |
| 657 | } |
| 658 | |
| 659 | /// \p val's defining load, if it is simple, meaning neither volatile nor |
| 660 | /// atomic. Null otherwise: a non-simple load's access has to survive as |
| 661 | /// written, and a call result or any other first-class value has no defining |
| 662 | /// load at all. |
| 663 | static cir::LoadOp maybeGetSimpleLoad(mlir::Value val) { |
| 664 | cir::LoadOp load = val.getDefiningOp<cir::LoadOp>(); |
| 665 | if (!load || load.isVolatileOrAtomic()) |
| 666 | return {}; |
| 667 | return load; |
| 668 | } |
| 669 | |
| 670 | /// \p val's defining load, if it loads a record and is not atomic. Null |
| 671 | /// otherwise. The load may be volatile, and then its bytes may only be read |
| 672 | /// through a volatile copy, never in place. |
| 673 | static cir::LoadOp maybeGetCopyableRecordLoad(mlir::Value val) { |
| 674 | cir::LoadOp load = val.getDefiningOp<cir::LoadOp>(); |
| 675 | if (!load || load.getMemOrder() || |
| 676 | !mlir::isa<cir::RecordType>(Val: load.getType())) |
| 677 | return {}; |
| 678 | return load; |
| 679 | } |
| 680 | |
| 681 | /// Emits an error and returns failure if \p val, an operand of \p user, is an |
| 682 | /// atomic load of a record. The record's bytes are read through a copy or in |
| 683 | /// place, neither of which keeps the load atomic. |
| 684 | static mlir::LogicalResult rejectAtomicRecordLoad(mlir::Operation *user, |
| 685 | mlir::Value val) { |
| 686 | auto load = val.getDefiningOp<cir::LoadOp>(); |
| 687 | if (!load || !load.getMemOrder() || |
| 688 | !mlir::isa<cir::RecordType>(Val: load.getType())) |
| 689 | return mlir::success(); |
| 690 | return user->emitOpError() |
| 691 | << "atomic load of a record operand not yet implemented in " |
| 692 | "CallConvLowering" ; |
| 693 | } |
| 694 | |
| 695 | /// Whether the storage the record load \p load reads is an alloca that holds |
| 696 | /// \p coercedTy at \p offset. With no offset, the coerced type's alloc size |
| 697 | /// has to fit in the record's. At an offset, the coerced value has to end |
| 698 | /// within the record's alloc size. |
| 699 | static bool canReadInPlace(cir::LoadOp load, mlir::Type coercedTy, |
| 700 | const cir::CIRDataLayout &dl, unsigned offset) { |
| 701 | if (!cir::getUnderlyingAlloca(addr: load.getAddr())) |
| 702 | return false; |
| 703 | uint64_t readEnd = |
| 704 | offset == 0 ? dl.getTypeAllocSize(ty: coercedTy).getFixedValue() |
| 705 | : offset + dl.getTypeStoreSize(ty: coercedTy).getFixedValue(); |
| 706 | return readEnd <= dl.getTypeAllocSize(ty: load.getType()).getFixedValue(); |
| 707 | } |
| 708 | |
| 709 | /// Read \p coercedTy at \p offset straight out of the storage the record load |
| 710 | /// \p load reads, which must be an alloca. The read goes just before \p load, |
| 711 | /// so it reads the bytes \p load reads, and takes the alloca's alignment at |
| 712 | /// \p offset. |
| 713 | static mlir::Value emitInPlaceCoercedLoad(mlir::OpBuilder &builder, |
| 714 | mlir::Location loc, cir::LoadOp load, |
| 715 | mlir::Type coercedTy, |
| 716 | unsigned offset) { |
| 717 | cir::AllocaOp src = cir::getUnderlyingAlloca(addr: load.getAddr()); |
| 718 | assert(src && "the record load must read an alloca" ); |
| 719 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 720 | builder.setInsertionPoint(load); |
| 721 | SmallPtrSet<mlir::Operation *, 4> ignored; |
| 722 | mlir::Value view = emitViewAtOffset(builder, loc, wantTy: coercedTy, base: load.getAddr(), |
| 723 | createdOps&: ignored, offset); |
| 724 | auto coerced = cir::LoadOp::create(builder, location: loc, addr: view); |
| 725 | coerced.setAlignment(llvm::MinAlign(A: src.getAlignment(), B: offset)); |
| 726 | return coerced; |
| 727 | } |
| 728 | |
| 729 | /// Copy the bytes of the record \p load reads into a new coercion slot for |
| 730 | /// \p coercedTy at \p offset, placed at the start of \p slotBlock, and return |
| 731 | /// the slot. The slot is at least as aligned as the storage copied from. The |
| 732 | /// copy goes just before \p load, so it reads the bytes \p load reads, and is |
| 733 | /// volatile if \p load is. |
| 734 | static cir::AllocaOp |
| 735 | emitCopiedCoercionSlot(mlir::OpBuilder &builder, mlir::Location loc, |
| 736 | cir::LoadOp load, mlir::Type coercedTy, |
| 737 | mlir::Block *slotBlock, const mlir::DataLayout &dl, |
| 738 | unsigned offset) { |
| 739 | auto recTy = cast<cir::RecordType>(Val: load.getType()); |
| 740 | uint64_t srcAlign; |
| 741 | if (cir::AllocaOp src = cir::getUnderlyingAlloca(addr: load.getAddr())) |
| 742 | srcAlign = src.getAlignment(); |
| 743 | else |
| 744 | srcAlign = load.getAlignment().value_or( |
| 745 | u: cir::CIRDataLayout(load->getParentOfType<mlir::ModuleOp>()) |
| 746 | .getTypeLanguageAlignment(ty: recTy)); |
| 747 | SmallPtrSet<mlir::Operation *, 4> ignored; |
| 748 | cir::AllocaOp slot = createCoercionSlot(builder, loc, srcTy: recTy, dstTy: coercedTy, |
| 749 | slotBlock, dl, createdOps&: ignored, offset); |
| 750 | slot.setAlignment(std::max(a: slot.getAlignment(), b: srcAlign)); |
| 751 | assert((offset == 0 || slot.getAllocaType() == recTy) && |
| 752 | "a direct offset must leave the record as the slot's type" ); |
| 753 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 754 | builder.setInsertionPoint(load); |
| 755 | mlir::Value recView = |
| 756 | emitViewAtOffset(builder, loc, wantTy: recTy, base: slot, createdOps&: ignored, /*offset=*/0); |
| 757 | auto copy = |
| 758 | cir::CopyOp::create(builder, location: loc, dst: recView, src: load.getAddr(), |
| 759 | dst_alignment: builder.getI64IntegerAttr(value: slot.getAlignment()), |
| 760 | src_alignment: builder.getI64IntegerAttr(value: srcAlign)); |
| 761 | copy.setIsVolatile(load.getIsVolatile()); |
| 762 | return slot; |
| 763 | } |
| 764 | |
| 765 | /// Copy the bytes of the record \p load reads into a coercion slot, as |
| 766 | /// emitCopiedCoercionSlot does, then read \p coercedTy out of the slot at |
| 767 | /// \p offset. The read takes the slot's alignment at \p offset. |
| 768 | static mlir::Value emitCopiedCoercedLoad(mlir::OpBuilder &builder, |
| 769 | mlir::Location loc, cir::LoadOp load, |
| 770 | mlir::Type coercedTy, |
| 771 | mlir::Block *slotBlock, |
| 772 | const mlir::DataLayout &dl, |
| 773 | unsigned offset) { |
| 774 | cir::AllocaOp slot = emitCopiedCoercionSlot(builder, loc, load, coercedTy, |
| 775 | slotBlock, dl, offset); |
| 776 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 777 | builder.setInsertionPoint(load); |
| 778 | SmallPtrSet<mlir::Operation *, 4> ignored; |
| 779 | mlir::Value coercedView = |
| 780 | emitViewAtOffset(builder, loc, wantTy: coercedTy, base: slot, createdOps&: ignored, offset); |
| 781 | auto coerced = cir::LoadOp::create(builder, location: loc, addr: coercedView); |
| 782 | coerced.setAlignment(llvm::MinAlign(A: slot.getAlignment(), B: offset)); |
| 783 | return coerced; |
| 784 | } |
| 785 | |
| 786 | /// Read \p coercedTy at \p offset out of the bytes the record load \p load |
| 787 | /// reads: straight out of its storage if \p load is not volatile and |
| 788 | /// canReadInPlace accepts it, and otherwise out of a coercion slot at the start |
| 789 | /// of \p slotBlock that those bytes are copied into. emitInPlaceCoercedLoad |
| 790 | /// does not mark its read volatile, so a volatile load is read through the |
| 791 | /// volatile copy instead. Either way no new load of the record type is |
| 792 | /// emitted. |
| 793 | static mlir::Value emitCoercedRecordLoad( |
| 794 | mlir::OpBuilder &builder, mlir::Location loc, cir::LoadOp load, |
| 795 | mlir::Type coercedTy, mlir::Block *slotBlock, const mlir::DataLayout &dl, |
| 796 | const cir::CIRDataLayout &cirDataLayout, unsigned offset) { |
| 797 | if (!load.getIsVolatile() && |
| 798 | canReadInPlace(load, coercedTy, dl: cirDataLayout, offset)) |
| 799 | return emitInPlaceCoercedLoad(builder, loc, load, coercedTy, offset); |
| 800 | return emitCopiedCoercedLoad(builder, loc, load, coercedTy, slotBlock, dl, |
| 801 | offset); |
| 802 | } |
| 803 | |
| 804 | /// Whether a cir.copy of \p ty into \p dst that stands in for a store of it |
| 805 | /// skips tail padding. Unless \p dst is an alloca, which holds a complete |
| 806 | /// object, it can be a potentially-overlapping subobject, such as a |
| 807 | /// [[no_unique_address]] member or a base, whose tail padding holds another |
| 808 | /// object. Classic CodeGen writes only the data size into a destination it is |
| 809 | /// handed (EmitCall) and into one that may overlap (EmitAggregateCopy). |
| 810 | /// Copying the whole object into an alloca is equivalent, since nothing else |
| 811 | /// lives in its tail padding. A struct has tail padding when its last members |
| 812 | /// are pad members, which CIRGen emits for padding another object may reuse and |
| 813 | /// for some other structs, such as an over-aligned one. |
| 814 | static bool shouldSkipTailPadding(mlir::Value dst, mlir::Type ty) { |
| 815 | auto structTy = mlir::dyn_cast<cir::StructType>(Val&: ty); |
| 816 | if (!structTy || cir::getUnderlyingAlloca(addr: dst)) |
| 817 | return false; |
| 818 | llvm::ArrayRef<mlir::Type> members = structTy.getMembers(); |
| 819 | llvm::ArrayRef<cir::RecordMemberKind> kinds = structTy.getMemberKinds(); |
| 820 | for (auto [member, kind] : llvm::reverse(C: llvm::zip_equal(t&: members, u&: kinds))) |
| 821 | if (cir::memberOwnsBytes(memberTy: member)) |
| 822 | return kind == cir::RecordMemberKind::Pad; |
| 823 | return false; |
| 824 | } |
| 825 | |
| 826 | /// Rewire the uses of the record value \p val, whose bytes sit at \p addr, |
| 827 | /// which is aligned to \p addrAlign, so that each one finds the record in |
| 828 | /// memory. \p val may already have been retyped to the type it is passed as, a |
| 829 | /// coerced type or a pointer, but its uses still expect the record. |
| 830 | /// |
| 831 | /// A store of \p val becomes a cir.copy from \p addr at the store, volatile if |
| 832 | /// the store is and skipping tail padding as shouldSkipTailPadding says. A |
| 833 | /// store of the record value writes only what its lowered type stores, so |
| 834 | /// another union member's data past those bytes would be lost. A consumer may |
| 835 | /// be rewritten before \p val's defining op, and has then already stored \p val |
| 836 | /// into a slot of its own. That store becomes a copy here too. |
| 837 | /// |
| 838 | /// Every other use is pointed at one cir.load of \p addr created at the |
| 839 | /// builder's insertion point. Only a call argument, a return and a read of a |
| 840 | /// member that is not itself a record may use it, since those either read |
| 841 | /// \p addr when they are rewritten or read a field. Uses by an operation in |
| 842 | /// \p exceptions are left alone. Fails, after reporting it, on an atomic store |
| 843 | /// of \p val or a use by anything else, before anything is rewired. |
| 844 | static mlir::LogicalResult |
| 845 | rewireRecordUses(mlir::OpBuilder &builder, mlir::Location loc, mlir::Value val, |
| 846 | mlir::Value addr, uint64_t addrAlign, |
| 847 | const mlir::DataLayout &dl, |
| 848 | const SmallPtrSetImpl<mlir::Operation *> &exceptions) { |
| 849 | mlir::Type recTy = mlir::cast<cir::PointerType>(Val: addr.getType()).getPointee(); |
| 850 | assert(mlir::isa<cir::RecordType>(recTy) && "only a record is rewired" ); |
| 851 | SmallVector<cir::StoreOp> stores; |
| 852 | SmallVector<mlir::OpOperand *> otherUses; |
| 853 | for (mlir::OpOperand &use : val.getUses()) { |
| 854 | mlir::Operation *user = use.getOwner(); |
| 855 | if (exceptions.contains(Ptr: user)) |
| 856 | continue; |
| 857 | if (auto store = mlir::dyn_cast<cir::StoreOp>(Val: user)) { |
| 858 | if (store.getMemOrder()) |
| 859 | return store.emitOpError() |
| 860 | << "atomic store of a record argument or result not yet " |
| 861 | "implemented in CallConvLowering" ; |
| 862 | stores.push_back(Elt: store); |
| 863 | continue; |
| 864 | } |
| 865 | auto member = mlir::dyn_cast<cir::ExtractMemberOp>(Val: user); |
| 866 | bool isFieldRead = member && !mlir::isa<cir::RecordType>(Val: member.getType()); |
| 867 | if (!isFieldRead && |
| 868 | !mlir::isa<cir::CIRCallOpInterface, cir::ReturnOp>(Val: user)) |
| 869 | return user->emitOpError() |
| 870 | << "use of a record argument or result other than by a store, " |
| 871 | "call, return or member read not yet implemented in " |
| 872 | "CallConvLowering" ; |
| 873 | otherUses.push_back(Elt: &use); |
| 874 | } |
| 875 | |
| 876 | mlir::IntegerAttr srcAlign = builder.getI64IntegerAttr(value: addrAlign); |
| 877 | cir::CIRDataLayout cirDataLayout( |
| 878 | val.getParentRegion()->getParentOfType<mlir::ModuleOp>()); |
| 879 | for (cir::StoreOp store : stores) { |
| 880 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 881 | builder.setInsertionPoint(store); |
| 882 | uint64_t dstAlign = store.getAlignment().value_or( |
| 883 | u: cirDataLayout.getTypeLanguageAlignment(ty: recTy)); |
| 884 | auto copy = |
| 885 | cir::CopyOp::create(builder, location: store.getLoc(), dst: store.getAddr(), src: addr, |
| 886 | dst_alignment: builder.getI64IntegerAttr(value: dstAlign), src_alignment: srcAlign); |
| 887 | copy.setIsVolatile(store.getIsVolatile()); |
| 888 | copy.setSkipTailPadding(shouldSkipTailPadding(dst: store.getAddr(), ty: recTy)); |
| 889 | store->erase(); |
| 890 | } |
| 891 | |
| 892 | if (otherUses.empty()) |
| 893 | return mlir::success(); |
| 894 | auto load = cir::LoadOp::create(builder, location: loc, addr); |
| 895 | load.setAlignment(addrAlign); |
| 896 | for (mlir::OpOperand *use : otherUses) |
| 897 | use->set(load); |
| 898 | return mlir::success(); |
| 899 | } |
| 900 | |
| 901 | /// Point the uses of \p val outside \p exceptions at the value of \p val's |
| 902 | /// original type that \p view, a view of the start of a coercion slot, holds. |
| 903 | /// For a record, rewireRecordUses points the uses at the slot. Any other type |
| 904 | /// is loaded out of it at the builder's insertion point. |
| 905 | static mlir::LogicalResult |
| 906 | replaceUsesFromSlot(mlir::OpBuilder &builder, mlir::Location loc, |
| 907 | mlir::Value val, mlir::Value view, |
| 908 | const mlir::DataLayout &dl, |
| 909 | const SmallPtrSetImpl<mlir::Operation *> &exceptions) { |
| 910 | if (!mlir::isa<cir::RecordType>( |
| 911 | Val: mlir::cast<cir::PointerType>(Val: view.getType()).getPointee())) { |
| 912 | auto load = cir::LoadOp::create(builder, location: loc, addr: view); |
| 913 | val.replaceAllUsesExcept(newValue: load, exceptions); |
| 914 | return mlir::success(); |
| 915 | } |
| 916 | cir::AllocaOp slot = cir::getUnderlyingAlloca(addr: view); |
| 917 | assert(slot && "the view must start at the coercion slot" ); |
| 918 | return rewireRecordUses(builder, loc, val, addr: view, addrAlign: slot.getAlignment(), dl, |
| 919 | exceptions); |
| 920 | } |
| 921 | |
| 922 | /// Coerce \p src, which is \p val retyped or the value replacing it, back to |
| 923 | /// \p origTy through a coercion slot at the start of \p slotBlock, and point |
| 924 | /// the uses of \p val outside \p createdOps at the result with |
| 925 | /// replaceUsesFromSlot. A direct \p offset lands on the coerced side, so the |
| 926 | /// view of \p origTy starts at the slot. A pointer that only changes address |
| 927 | /// space is cast instead. The coercion's operations are added to |
| 928 | /// \p createdOps. |
| 929 | static mlir::LogicalResult |
| 930 | emitCoercionAndReplaceUses(mlir::OpBuilder &builder, mlir::Location loc, |
| 931 | mlir::Type origTy, mlir::Value src, mlir::Value val, |
| 932 | mlir::Block *slotBlock, const mlir::DataLayout &dl, |
| 933 | SmallPtrSetImpl<mlir::Operation *> &createdOps, |
| 934 | unsigned offset) { |
| 935 | if (mlir::Value cast = emitAddrSpaceCoercion(builder, loc, dstTy: origTy, src, |
| 936 | createdOps, offset)) { |
| 937 | val.replaceAllUsesExcept(newValue: cast, exceptions: createdOps); |
| 938 | return mlir::success(); |
| 939 | } |
| 940 | mlir::Value view = emitCoercionToMemory(builder, loc, dstTy: origTy, src, slotBlock, |
| 941 | dl, createdOps, offset); |
| 942 | return replaceUsesFromSlot(builder, loc, val, view, dl, exceptions: createdOps); |
| 943 | } |
| 944 | |
| 945 | /// Coerce the value each cir.return returns to the new (coerced) return type. |
| 946 | /// |
| 947 | /// When the returned value is a non-atomic load of a record, the coerced value |
| 948 | /// is read by emitCoercedRecordLoad, and the record load is erased once nothing |
| 949 | /// else uses it. Any other value is stored into a coercion slot and read back. |
| 950 | /// A record with no storage behind it, such as a constant, defines no bytes |
| 951 | /// beyond its fields, so the store writes every byte it defines. A call |
| 952 | /// result's store is turned into a copy when its call is rewritten later. |
| 953 | /// Fails, after reporting it, on an atomic load of a record. |
| 954 | mlir::LogicalResult |
| 955 | insertReturnCoercion(mlir::FunctionOpInterface funcOp, mlir::Type origRetTy, |
| 956 | mlir::Type coercedRetTy, mlir::OpBuilder &builder, |
| 957 | const mlir::DataLayout &dl, |
| 958 | const cir::CIRDataLayout &cirDataLayout, unsigned offset) { |
| 959 | SmallVector<cir::ReturnOp> returns; |
| 960 | funcOp.walk(callback: [&](cir::ReturnOp r) { returns.push_back(Elt: r); }); |
| 961 | for (cir::ReturnOp r : returns) { |
| 962 | if (r.getInput().empty()) |
| 963 | continue; |
| 964 | mlir::Value origVal = r.getInput()[0]; |
| 965 | if (origVal.getType() == coercedRetTy) |
| 966 | continue; |
| 967 | |
| 968 | if (failed(Result: rejectAtomicRecordLoad(user: r, val: origVal))) |
| 969 | return mlir::failure(); |
| 970 | if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(val: origVal)) { |
| 971 | mlir::Value coerced = emitCoercedRecordLoad( |
| 972 | builder, loc: r.getLoc(), load: recLoad, coercedTy: coercedRetTy, |
| 973 | slotBlock: &funcOp->getRegion(index: 0).front(), dl, cirDataLayout, offset); |
| 974 | r->setOperand(idx: 0, value: coerced); |
| 975 | if (recLoad->use_empty()) |
| 976 | recLoad->erase(); |
| 977 | continue; |
| 978 | } |
| 979 | |
| 980 | builder.setInsertionPoint(r); |
| 981 | mlir::Value coerced = |
| 982 | emitCoercion(builder, loc: r.getLoc(), dstTy: coercedRetTy, src: origVal, |
| 983 | slotBlock: &funcOp->getRegion(index: 0).front(), dl, offset); |
| 984 | r->setOperand(idx: 0, value: coerced); |
| 985 | } |
| 986 | return mlir::success(); |
| 987 | } |
| 988 | |
| 989 | /// \p recordVal's defining load, if it is simple and its address resolves to |
| 990 | /// an alloca. Null otherwise. |
| 991 | static cir::LoadOp getWholeRecordLoad(mlir::Value recordVal) { |
| 992 | cir::LoadOp load = maybeGetSimpleLoad(val: recordVal); |
| 993 | if (!load || !cir::getUnderlyingAlloca(addr: load.getAddr())) |
| 994 | return {}; |
| 995 | return load; |
| 996 | } |
| 997 | |
| 998 | /// Returns the address \p load read from, as a pointer of type \p paramTy, if |
| 999 | /// it can be passed in place of the loaded value for an indirect argument that |
| 1000 | /// needs \p minAlign alignment, and null otherwise, including when \p load is |
| 1001 | /// null. A byval or byref argument also needs storageUnwrittenBetween. On |
| 1002 | /// null the caller reports a non-byval argument or fills a slot of its own. |
| 1003 | /// |
| 1004 | /// The address has to have \p paramTy, the rewritten parameter's type, since a |
| 1005 | /// cir.load pins the pointee type without pinning the address space. An |
| 1006 | /// address_space cast of a \p paramTy pointer qualifies through its source. |
| 1007 | /// The storage it names has to be a slot allocated here, reached through |
| 1008 | /// storage-preserving casts, already stating at least \p minAlign. The |
| 1009 | /// enclosing function's own non-byval parameter qualifies as well, since its |
| 1010 | /// slot stands until finalizeParameterSlots and prepareNonByvalParameters has |
| 1011 | /// restated it with the alignment that parameter promises rather than the one |
| 1012 | /// CIRGen chose for a local copy. A byval or byref parameter's slot keeps |
| 1013 | /// CIRGen's alignment, so it qualifies only where that already covers |
| 1014 | /// \p minAlign. |
| 1015 | /// |
| 1016 | /// Raising a slot's alignment here would not hold for one that stands in |
| 1017 | /// for a parameter, since finalizeParameterSlots replaces it with the |
| 1018 | /// incoming pointer, which promises only what the caller gave it. |
| 1019 | static mlir::Value forwardableIndirectAddr(cir::LoadOp load, |
| 1020 | cir::PointerType paramTy, |
| 1021 | uint64_t minAlign) { |
| 1022 | if (!load) |
| 1023 | return {}; |
| 1024 | mlir::Value addr = load.getAddr(); |
| 1025 | // CIRGen reads an alloca outside the default address space through a cast. |
| 1026 | if (auto view = addr.getDefiningOp<cir::CastOp>(); |
| 1027 | view && view.getKind() == cir::CastKind::address_space && |
| 1028 | view.getSrc().getType() == paramTy) |
| 1029 | addr = view.getSrc(); |
| 1030 | if (addr.getType() != paramTy) |
| 1031 | return {}; |
| 1032 | cir::AllocaOp slot = cir::getUnderlyingAlloca(addr); |
| 1033 | return slot && slot.getAlignment() >= minAlign ? addr : mlir::Value(); |
| 1034 | } |
| 1035 | |
| 1036 | /// True when \p aliasAnalysis shows that nothing between \p load and \p call |
| 1037 | /// can write the storage \p load read, so a byval argument may name that |
| 1038 | /// storage rather than a copy of it. byval takes the callee's copy at the |
| 1039 | /// call, while the operand carries the value as of the load, so a write in |
| 1040 | /// between would reach the callee that the operand does not hold. |
| 1041 | /// prepareNonByvalParameters reads a parameter at its spill, which leaves the |
| 1042 | /// body's own stores in between. |
| 1043 | /// |
| 1044 | /// A load in another block is not forwardable here, since the paths from it |
| 1045 | /// to the call are not walked. |
| 1046 | static bool storageUnwrittenBetween(cir::LoadOp load, mlir::Operation *call, |
| 1047 | mlir::AliasAnalysis &aliasAnalysis) { |
| 1048 | if (load->getBlock() != call->getBlock()) |
| 1049 | return false; |
| 1050 | for (mlir::Operation *op = load->getNextNode(); op != call; |
| 1051 | op = op->getNextNode()) { |
| 1052 | assert(op && "the load must precede the call in the block they share" ); |
| 1053 | if (op->getNumRegions() != 0 || |
| 1054 | aliasAnalysis.getModRef(op, location: load.getAddr()).isMod()) |
| 1055 | return false; |
| 1056 | } |
| 1057 | return true; |
| 1058 | } |
| 1059 | |
| 1060 | /// The alignment a copy may claim for the storage \p load read, which is the |
| 1061 | /// one \p load states, or the underlying slot's when it states none. A load |
| 1062 | /// with neither names storage this pass cannot reason about, so the copy is |
| 1063 | /// left to assume nothing. |
| 1064 | static mlir::IntegerAttr loadSourceAlignment(cir::LoadOp load, |
| 1065 | mlir::OpBuilder &builder) { |
| 1066 | if (mlir::IntegerAttr stated = load.getAlignmentAttr()) |
| 1067 | return stated; |
| 1068 | if (cir::AllocaOp slot = cir::getUnderlyingAlloca(addr: load.getAddr())) |
| 1069 | return builder.getI64IntegerAttr(value: slot.getAlignment()); |
| 1070 | return builder.getI64IntegerAttr(value: 1); |
| 1071 | } |
| 1072 | |
| 1073 | /// Decompose a struct value into one scalar call argument per field of \p |
| 1074 | /// recTy, appending the field values to \p newArgs. When \p structVal is a |
| 1075 | /// simple load from an alloca, read each field with cir.get_member + |
| 1076 | /// cir.load from the address the load used, emitted at the original load's |
| 1077 | /// position so they observe the same memory state, and record the now-dead |
| 1078 | /// whole-struct load in \p deadRecordLoads for later erasure. Otherwise (a |
| 1079 | /// call result, compound literal, or a volatile or atomic load) extract each |
| 1080 | /// field from the value with cir.extract_member. Loading the members from |
| 1081 | /// memory rather than extracting from a whole-struct value keeps the result in |
| 1082 | /// a form SROA can promote (it does not reason about extractvalue). Shared by |
| 1083 | /// the Expand and Direct+canFlatten argument paths. |
| 1084 | static void emitStructFieldArgs(mlir::OpBuilder &builder, mlir::Location loc, |
| 1085 | mlir::Value structVal, cir::RecordType recTy, |
| 1086 | SmallVectorImpl<mlir::Value> &newArgs, |
| 1087 | SmallVectorImpl<cir::LoadOp> &deadRecordLoads) { |
| 1088 | cir::LoadOp srcLoad = getWholeRecordLoad(recordVal: structVal); |
| 1089 | |
| 1090 | if (srcLoad) { |
| 1091 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 1092 | builder.setInsertionPoint(srcLoad); |
| 1093 | cir::PointerType baseTy = srcLoad.getAddr().getType(); |
| 1094 | for (auto [f, fieldTy] : llvm::enumerate(First: recTy.getMembers())) { |
| 1095 | mlir::Type fieldPtrTy = |
| 1096 | cir::PointerType::get(pointee: fieldTy, addrSpace: baseTy.getAddrSpace()); |
| 1097 | mlir::Value fieldPtr = cir::GetMemberOp::create( |
| 1098 | builder, location: loc, type: fieldPtrTy, value: srcLoad.getAddr(), /*name=*/"" , |
| 1099 | /*index=*/f); |
| 1100 | newArgs.push_back(Elt: cir::LoadOp::create(builder, location: loc, addr: fieldPtr)); |
| 1101 | } |
| 1102 | deadRecordLoads.push_back(Elt: srcLoad); |
| 1103 | } else { |
| 1104 | for (unsigned f = 0; f < recTy.getNumElements(); ++f) |
| 1105 | newArgs.push_back( |
| 1106 | Elt: cir::ExtractMemberOp::create(builder, location: loc, record: structVal, index: f)); |
| 1107 | } |
| 1108 | } |
| 1109 | |
| 1110 | /// Erase the loads that a rewritten call left unused. The old call must |
| 1111 | /// already be erased, since until then it still counts as a user. One load can |
| 1112 | /// feed two operands of the same call, as in f(s, s), so \p loads can hold the |
| 1113 | /// same load twice. A load that another op still reads is left alone. |
| 1114 | static void eraseDeadRecordLoads(ArrayRef<cir::LoadOp> loads) { |
| 1115 | llvm::SmallSetVector<mlir::Operation *, 4> uniqueLoads(llvm::from_range, |
| 1116 | loads); |
| 1117 | for (mlir::Operation *load : uniqueLoads) |
| 1118 | if (load->use_empty()) |
| 1119 | load->erase(); |
| 1120 | } |
| 1121 | |
| 1122 | /// The old call, their only user, must already be erased. |
| 1123 | static void eraseDeadFreezes(ArrayRef<cir::FreezeOp> freezes) { |
| 1124 | for (cir::FreezeOp freeze : freezes) |
| 1125 | freeze->erase(); |
| 1126 | } |
| 1127 | |
| 1128 | /// The store that spills indirect parameter \p blockArg, if the parameter's |
| 1129 | /// only use is that store. Null otherwise. |
| 1130 | static cir::StoreOp maybeFindParamSpill(mlir::BlockArgument blockArg) { |
| 1131 | if (!blockArg.hasOneUse()) |
| 1132 | return {}; |
| 1133 | auto store = dyn_cast<cir::StoreOp>(Val: *blockArg.user_begin()); |
| 1134 | if (!store || store.getValue() != blockArg) |
| 1135 | return {}; |
| 1136 | return store; |
| 1137 | } |
| 1138 | |
| 1139 | /// The alloca a parameter spill writes to through \p addr, either directly or |
| 1140 | /// through an address_space cast of it. Null otherwise. |
| 1141 | static cir::AllocaOp getSpillSlot(mlir::Value addr) { |
| 1142 | if (auto view = addr.getDefiningOp<cir::CastOp>(); |
| 1143 | view && view.getKind() == cir::CastKind::address_space) |
| 1144 | addr = view.getSrc(); |
| 1145 | return addr.getDefiningOp<cir::AllocaOp>(); |
| 1146 | } |
| 1147 | |
| 1148 | /// The store that spills non-byval indirect parameter \p blockArg, and the |
| 1149 | /// slot it spills into. prepareNonByvalParameters has already established |
| 1150 | /// that the spill is the block argument's only use and that it stores into |
| 1151 | /// an alloca, which is what the assertions below rest on. Both results are |
| 1152 | /// null when the parameter has no spill, which is so only when nothing uses |
| 1153 | /// it at all. |
| 1154 | static std::pair<cir::StoreOp, cir::AllocaOp> |
| 1155 | findParamSpill(mlir::BlockArgument blockArg) { |
| 1156 | if (blockArg.use_empty()) |
| 1157 | return {}; |
| 1158 | cir::StoreOp store = maybeFindParamSpill(blockArg); |
| 1159 | assert(store && "non-byval arg's only use must be its CIRGen param spill" ); |
| 1160 | cir::AllocaOp slot = getSpillSlot(addr: store.getAddr()); |
| 1161 | assert(slot && "non-byval arg's spill must store to an alloca" ); |
| 1162 | return {store, slot}; |
| 1163 | } |
| 1164 | |
| 1165 | /// For each Direct arg with a coerced type, change the block argument's type to |
| 1166 | /// the coerced type and insert a coercion at function entry that maps it back |
| 1167 | /// to the original type for body uses, which find a record in the coercion slot |
| 1168 | /// through rewireRecordUses. For each Indirect byval arg and each |
| 1169 | /// IndirectAliased arg, change the block argument's type to a pointer, then |
| 1170 | /// replace a record's single param spill with a copy from that pointer. |
| 1171 | /// Without such a spill, a record's uses find it at the pointer through |
| 1172 | /// rewireRecordUses, and a value of any other type is loaded from the pointer |
| 1173 | /// at entry. For each Indirect non-byval arg, change the block argument to a |
| 1174 | /// pointer and queue the param-slot alloca to be replaced by it (no entry load |
| 1175 | /// or byte-copy) so the body operates on the caller's storage in place. For |
| 1176 | /// each Expand arg, replace the single struct block argument with N scalar |
| 1177 | /// block arguments (one per field) and store each field directly into the |
| 1178 | /// parameter's own alloca (the CIRGen spill slot), erasing the original |
| 1179 | /// whole-struct store. |
| 1180 | /// |
| 1181 | /// \p hasSRetArg is true when the function has an sret return (a hidden return |
| 1182 | /// pointer is prepended as block argument 0). Expand arguments expand the |
| 1183 | /// block argument count, so a running index tracks the current block argument |
| 1184 | /// position rather than computing the classification index + \p hasSRetArg |
| 1185 | /// directly. Fails, after reporting it, when rewireRecordUses does. |
| 1186 | mlir::LogicalResult |
| 1187 | insertArgCoercion(mlir::FunctionOpInterface funcOp, |
| 1188 | const FunctionClassification &fc, mlir::OpBuilder &builder, |
| 1189 | const mlir::DataLayout &dl, bool hasSRetArg, |
| 1190 | SmallVectorImpl<std::pair<cir::AllocaOp, mlir::BlockArgument>> |
| 1191 | &pendingParamSlots) { |
| 1192 | mlir::Region &body = funcOp->getRegion(index: 0); |
| 1193 | if (body.empty()) |
| 1194 | return mlir::success(); |
| 1195 | mlir::Block &entry = body.front(); |
| 1196 | |
| 1197 | // Running block argument index. Each non-Expand classification occupies |
| 1198 | // one block argument slot; each Expand classification occupies N slots |
| 1199 | // (one per struct field), so the running index must be incremented by N |
| 1200 | // rather than 1 after processing an Expand arg. |
| 1201 | unsigned blockArgIdx = hasSRetArg ? 1 : 0; |
| 1202 | |
| 1203 | for (const ArgClassification &ac : fc.argInfos) { |
| 1204 | assert(blockArgIdx < entry.getNumArguments() && |
| 1205 | "classification count must not exceed entry block arguments" ); |
| 1206 | |
| 1207 | if (ac.kind == ArgKind::Expand) { |
| 1208 | // The block arg at blockArgIdx currently has the original struct type. |
| 1209 | // Replace it with N scalar args (one per field) and store each field |
| 1210 | // directly into the parameter's own alloca. |
| 1211 | mlir::BlockArgument origArg = entry.getArgument(i: blockArgIdx); |
| 1212 | auto recTy = cast<cir::RecordType>(Val: origArg.getType()); |
| 1213 | assert(recTy.isStruct() && |
| 1214 | "Expand classification requires a struct type, not a union" ); |
| 1215 | unsigned numFields = recTy.getNumElements(); |
| 1216 | assert(numFields > 0 && |
| 1217 | "Expand classification requires at least one struct field" ); |
| 1218 | mlir::Location loc = funcOp.getLoc(); |
| 1219 | |
| 1220 | // CIRGen spills every by-value struct parameter into its local alloca |
| 1221 | // with a single store before any other use, so the struct block arg's |
| 1222 | // only use is that spill. Capture it and the destination alloca so the |
| 1223 | // expanded fields can be stored straight into that alloca, preserving |
| 1224 | // the alloca's variable name and `init` flag and avoiding a |
| 1225 | // reassemble-then-reload roundtrip. DCE may have run earlier and |
| 1226 | // removed the spill (leaving the block arg unused); tolerate that by |
| 1227 | // only flattening the signature and emitting no field stores. |
| 1228 | cir::StoreOp paramStore; |
| 1229 | cir::AllocaOp destAlloca; |
| 1230 | if (!origArg.use_empty()) { |
| 1231 | assert(origArg.hasOneUse() && |
| 1232 | "Expand arg must have exactly one use (the CIRGen param spill)" ); |
| 1233 | paramStore = cast<cir::StoreOp>(Val: *origArg.user_begin()); |
| 1234 | assert(paramStore.getValue() == origArg && |
| 1235 | "Expand arg's use must be the value operand of its store" ); |
| 1236 | destAlloca = cast<cir::AllocaOp>(Val: paramStore.getAddr().getDefiningOp()); |
| 1237 | } |
| 1238 | |
| 1239 | // Erase the original whole-struct spill before retyping the block |
| 1240 | // argument, so the store is never left feeding a type-mismatched value. |
| 1241 | // The field stores take its place, just before the following operation |
| 1242 | // (the spill always precedes the entry block's terminator). |
| 1243 | mlir::Operation *fieldStoreInsertPt = nullptr; |
| 1244 | if (paramStore) { |
| 1245 | fieldStoreInsertPt = paramStore->getNextNode(); |
| 1246 | assert(fieldStoreInsertPt && |
| 1247 | "param spill must be followed by a block terminator" ); |
| 1248 | paramStore->erase(); |
| 1249 | } |
| 1250 | |
| 1251 | // Split the single struct block arg into N scalar field block args (slot |
| 1252 | // 0 reuses the original; slots 1..N-1 are inserted after it). The |
| 1253 | // reshape needs no insertion point. The field stores are gated on the |
| 1254 | // same destAlloca condition: when the spill survived we set the insert |
| 1255 | // point to its old slot (which sits after the CIRGen allocas) and store |
| 1256 | // each field there; when DCE removed the spill the parameter is dead, so |
| 1257 | // we only reshape the signature and emit no stores. |
| 1258 | if (destAlloca) |
| 1259 | builder.setInsertionPoint(fieldStoreInsertPt); |
| 1260 | for (auto [f, fieldTy] : llvm::enumerate(First: recTy.getMembers())) { |
| 1261 | if (f == 0) |
| 1262 | origArg.setType(fieldTy); |
| 1263 | else |
| 1264 | entry.insertArgument(index: blockArgIdx + f, type: fieldTy, loc); |
| 1265 | if (!destAlloca) |
| 1266 | continue; |
| 1267 | mlir::Type fieldPtrTy = cir::PointerType::get( |
| 1268 | pointee: fieldTy, |
| 1269 | addrSpace: mlir::cast<cir::PointerType>(Val: destAlloca.getType()).getAddrSpace()); |
| 1270 | auto fieldPtr = cir::GetMemberOp::create(builder, location: loc, type: fieldPtrTy, |
| 1271 | value: destAlloca, /*name=*/"" , |
| 1272 | /*index=*/f); |
| 1273 | cir::StoreOp::create(builder, location: loc, value: entry.getArgument(i: blockArgIdx + f), |
| 1274 | addr: fieldPtr); |
| 1275 | } |
| 1276 | |
| 1277 | blockArgIdx += numFields; |
| 1278 | continue; |
| 1279 | } |
| 1280 | |
| 1281 | mlir::BlockArgument blockArg = entry.getArgument(i: blockArgIdx); |
| 1282 | |
| 1283 | if (cir::RecordType flatTy = getFlattenedCoercedType(ac, dl)) { |
| 1284 | // Direct + canFlatten: the coerced type is a struct whose fields are |
| 1285 | // passed as separate arguments. As in the Expand path, the single block |
| 1286 | // arg becomes N scalar block args, and each is stored straight into a |
| 1287 | // slot sized and aligned for both the coerced struct and the original |
| 1288 | // argument type, which the body then reads. |
| 1289 | unsigned numFields = flatTy.getNumElements(); |
| 1290 | Type origTy = blockArg.getType(); |
| 1291 | Location loc = funcOp.getLoc(); |
| 1292 | |
| 1293 | // Change slot 0 to field 0's type; insert slots 1..N-1 after it. |
| 1294 | blockArg.setType(flatTy.getElementType(idx: 0)); |
| 1295 | for (unsigned f = 1; f < numFields; ++f) |
| 1296 | entry.insertArgument(index: blockArgIdx + f, type: flatTy.getElementType(idx: f), loc); |
| 1297 | |
| 1298 | // setInsertionPointToStart: see comment in the Expand arm above. |
| 1299 | builder.setInsertionPointToStart(&entry); |
| 1300 | assert(!ac.directOffset && |
| 1301 | "the coerced struct starts at slot offset 0 here, so a flattened " |
| 1302 | "coercion cannot honor a direct offset" ); |
| 1303 | SmallPtrSet<Operation *, 8> flattenOps; |
| 1304 | cir::AllocaOp slot; |
| 1305 | if (origTy == flatTy) { |
| 1306 | slot = cir::AllocaOp::create( |
| 1307 | builder, location: loc, addr: getAllocaPtrTy(pointee: flatTy, dl), |
| 1308 | name: builder.getStringAttr(bytes: "coerce" ), |
| 1309 | alignment: builder.getI64IntegerAttr( |
| 1310 | value: cir::CIRDataLayout(funcOp->getParentOfType<mlir::ModuleOp>()) |
| 1311 | .getTypeLanguageAlignment(ty: flatTy))); |
| 1312 | flattenOps.insert(Ptr: slot); |
| 1313 | } else { |
| 1314 | slot = createCoercionSlot(builder, loc, srcTy: flatTy, dstTy: origTy, slotBlock: &entry, dl, |
| 1315 | createdOps&: flattenOps, /*offset=*/0); |
| 1316 | } |
| 1317 | Value flatView = emitViewAtOffset(builder, loc, wantTy: flatTy, base: slot, createdOps&: flattenOps, |
| 1318 | /*offset=*/0); |
| 1319 | auto flatPtrTy = cast<cir::PointerType>(Val: flatView.getType()); |
| 1320 | for (auto [f, fieldTy] : llvm::enumerate(First: flatTy.getMembers())) { |
| 1321 | Type fieldPtrTy = |
| 1322 | cir::PointerType::get(pointee: fieldTy, addrSpace: flatPtrTy.getAddrSpace()); |
| 1323 | auto fieldPtr = cir::GetMemberOp::create(builder, location: loc, type: fieldPtrTy, |
| 1324 | value: flatView, /*name=*/"" , |
| 1325 | /*index=*/f); |
| 1326 | flattenOps.insert(Ptr: fieldPtr); |
| 1327 | auto storeOp = cir::StoreOp::create( |
| 1328 | builder, location: loc, value: entry.getArgument(i: blockArgIdx + f), addr: fieldPtr); |
| 1329 | // A packed slot can be less aligned than the field's own type. |
| 1330 | uint64_t fieldAlign = |
| 1331 | llvm::MinAlign(A: slot.getAlignment(), B: flatTy.getElementOffset(dataLayout: dl, idx: f)); |
| 1332 | if (fieldAlign < dl.getTypeABIAlignment(t: fieldTy)) |
| 1333 | storeOp.setAlignment(fieldAlign); |
| 1334 | flattenOps.insert(Ptr: storeOp); |
| 1335 | } |
| 1336 | |
| 1337 | // Point the original body uses of the struct block arg, now field 0, at |
| 1338 | // the original-type view of the slot. |
| 1339 | Value origView = emitViewAtOffset(builder, loc, wantTy: origTy, base: slot, createdOps&: flattenOps, |
| 1340 | /*offset=*/0); |
| 1341 | if (failed(Result: replaceUsesFromSlot(builder, loc, val: blockArg, view: origView, dl, |
| 1342 | exceptions: flattenOps))) |
| 1343 | return mlir::failure(); |
| 1344 | |
| 1345 | blockArgIdx += numFields; |
| 1346 | continue; |
| 1347 | } |
| 1348 | |
| 1349 | if ((ac.kind == ArgKind::Direct && ac.coercedType) || |
| 1350 | isRecordExtend(ac, origTy: blockArg.getType())) { |
| 1351 | mlir::Type oldArgTy = blockArg.getType(); |
| 1352 | mlir::Type newArgTy = ac.coercedType; |
| 1353 | if (oldArgTy == newArgTy) { |
| 1354 | ++blockArgIdx; |
| 1355 | continue; |
| 1356 | } |
| 1357 | blockArg.setType(newArgTy); |
| 1358 | |
| 1359 | // The coercion's own operations are left out of the rewiring, since they |
| 1360 | // read blockArg as the coerced type. |
| 1361 | builder.setInsertionPointToStart(&entry); |
| 1362 | SmallPtrSet<mlir::Operation *, 4> coercionOps; |
| 1363 | if (failed(Result: emitCoercionAndReplaceUses(builder, loc: funcOp.getLoc(), origTy: oldArgTy, |
| 1364 | src: blockArg, val: blockArg, slotBlock: &entry, dl, |
| 1365 | createdOps&: coercionOps, offset: ac.directOffset))) |
| 1366 | return mlir::failure(); |
| 1367 | } else if (ac.kind == ArgKind::Indirect || |
| 1368 | ac.kind == ArgKind::IndirectAliased) { |
| 1369 | // byval and non-byval both lower to a pointer to T in the classified |
| 1370 | // address space, and which it is shows up only in the attrs |
| 1371 | // updateArgAttrs applies. Body lowering differs: the body addresses a |
| 1372 | // byval parameter through the CIRGen slot it was spilled into, so that |
| 1373 | // slot has to be filled from the incoming pointer, while a non-byval |
| 1374 | // parameter's slot is replaced by the pointer so the body works on the |
| 1375 | // caller's storage in place. A byref parameter takes the byval path. |
| 1376 | bool byRef = ac.kind == ArgKind::IndirectAliased; |
| 1377 | cir::PointerType ptrTy = getIndirectPtrTy(pointee: blockArg.getType(), ac); |
| 1378 | |
| 1379 | if (!byRef && !ac.byVal) { |
| 1380 | // Without byval, drop the spill store and let the slot's uses read the |
| 1381 | // incoming pointer, so the body operates on the caller's storage in |
| 1382 | // place. A byte-copy would be wrong for non-trivially-copyable |
| 1383 | // aggregates (e.g. libstdc++ SSO std::string, where it would leave |
| 1384 | // `_M_p` aliasing the source's `_M_local_buf`). |
| 1385 | auto [paramStore, destAlloca] = findParamSpill(blockArg); |
| 1386 | |
| 1387 | if (paramStore) |
| 1388 | paramStore->erase(); |
| 1389 | |
| 1390 | // Update the block argument to point to its original type. |
| 1391 | blockArg.setType(ptrTy); |
| 1392 | |
| 1393 | // Pointing the slot's uses at the incoming pointer waits until every |
| 1394 | // call site has been rewritten. A call that hands this parameter |
| 1395 | // straight on recognizes it by the slot its operand was loaded from, |
| 1396 | // and collapsing the slot here would leave that call reading a block |
| 1397 | // argument with no defining operation to inspect. A dead spill DCE |
| 1398 | // already removed leaves nothing to collapse. |
| 1399 | if (destAlloca) |
| 1400 | pendingParamSlots.emplace_back(Args&: destAlloca, Args&: blockArg); |
| 1401 | } else { |
| 1402 | // A record's spill slot is filled by a byte copy of the incoming |
| 1403 | // object rather than by a store of the loaded value: its padding |
| 1404 | // bytes can hold another union member's live data, which a value |
| 1405 | // store would leave unwritten. A _BitInt keeps its load and spill |
| 1406 | // store, which are its conversion between value and in-memory form. |
| 1407 | cir::StoreOp paramStore; |
| 1408 | if (mlir::isa<cir::RecordType>(Val: blockArg.getType())) |
| 1409 | paramStore = maybeFindParamSpill(blockArg); |
| 1410 | |
| 1411 | // Erasing the spill before the block argument is retyped keeps the |
| 1412 | // store from being left with a value operand of the wrong type. |
| 1413 | mlir::Value destAddr; |
| 1414 | mlir::IntegerAttr destAlign; |
| 1415 | mlir::Operation *spillPos = nullptr; |
| 1416 | bool spillIsVolatile = false; |
| 1417 | if (paramStore) { |
| 1418 | if (paramStore.getMemOrder()) |
| 1419 | return paramStore.emitOpError() |
| 1420 | << "atomic store of a record argument or result not yet " |
| 1421 | "implemented in CallConvLowering" ; |
| 1422 | spillIsVolatile = paramStore.getIsVolatile(); |
| 1423 | destAddr = paramStore.getAddr(); |
| 1424 | cir::AllocaOp destAlloca = cir::getUnderlyingAlloca(addr: destAddr); |
| 1425 | destAlign = paramStore.getAlignmentAttr(); |
| 1426 | if (!destAlign && destAlloca) |
| 1427 | destAlign = destAlloca.getAlignmentAttr(); |
| 1428 | spillPos = paramStore->getNextNode(); |
| 1429 | assert(spillPos && "param spill must be followed by a block " |
| 1430 | "terminator" ); |
| 1431 | paramStore->erase(); |
| 1432 | } |
| 1433 | |
| 1434 | blockArg.setType(ptrTy); |
| 1435 | |
| 1436 | if (destAddr) { |
| 1437 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 1438 | builder.setInsertionPoint(spillPos); |
| 1439 | auto copy = cir::CopyOp::create( |
| 1440 | builder, location: funcOp.getLoc(), dst: destAddr, src: blockArg, dst_alignment: destAlign, |
| 1441 | src_alignment: builder.getI64IntegerAttr(value: ac.indirectAlign.value())); |
| 1442 | copy.setIsVolatile(spillIsVolatile); |
| 1443 | copy.setSkipTailPadding( |
| 1444 | shouldSkipTailPadding(dst: destAddr, ty: ptrTy.getPointee())); |
| 1445 | } else if (!blockArg.use_empty()) { |
| 1446 | builder.setInsertionPointToStart(&entry); |
| 1447 | if (mlir::isa<cir::RecordType>(Val: ptrTy.getPointee())) { |
| 1448 | SmallPtrSet<mlir::Operation *, 1> noExceptions; |
| 1449 | if (failed(Result: rewireRecordUses(builder, loc: funcOp.getLoc(), val: blockArg, |
| 1450 | addr: blockArg, addrAlign: ac.indirectAlign.value(), dl, |
| 1451 | exceptions: noExceptions))) |
| 1452 | return mlir::failure(); |
| 1453 | } else { |
| 1454 | auto loadOp = |
| 1455 | cir::LoadOp::create(builder, location: funcOp.getLoc(), addr: blockArg); |
| 1456 | SmallPtrSet<mlir::Operation *, 1> loadOps = {loadOp}; |
| 1457 | blockArg.replaceAllUsesExcept(newValue: loadOp.getResult(), exceptions: loadOps); |
| 1458 | } |
| 1459 | } |
| 1460 | } |
| 1461 | } |
| 1462 | // Ignore, Extend, and Direct-without-coerce need no block-level changes. |
| 1463 | |
| 1464 | ++blockArgIdx; |
| 1465 | } |
| 1466 | return mlir::success(); |
| 1467 | } |
| 1468 | |
| 1469 | /// Replace each address_space cast of \p slot to the type of \p ptr with |
| 1470 | /// \p ptr. |
| 1471 | void replaceAddrSpaceViews(cir::AllocaOp slot, mlir::Value ptr) { |
| 1472 | for (mlir::Operation *user : llvm::make_early_inc_range(Range: slot->getUsers())) { |
| 1473 | auto view = mlir::dyn_cast<cir::CastOp>(Val: user); |
| 1474 | if (view && view.getKind() == cir::CastKind::address_space && |
| 1475 | view.getType() == ptr.getType()) { |
| 1476 | view.getResult().replaceAllUsesWith(newValue: ptr); |
| 1477 | view->erase(); |
| 1478 | } |
| 1479 | } |
| 1480 | } |
| 1481 | |
| 1482 | /// Rewrite each cir.return so the return value flows through the sret |
| 1483 | /// pointer (the prepended first block argument) and the function returns |
| 1484 | /// void. |
| 1485 | /// |
| 1486 | /// CIRGen emits a local `__retval` alloca and emits `cir.return %loaded` |
| 1487 | /// where `%loaded = cir.load __retval`. The naive lowering -- store the |
| 1488 | /// loaded SSA value through the sret pointer -- byte-copies the record, |
| 1489 | /// which is wrong for non-trivially-copyable types: e.g. libstdc++'s SSO |
| 1490 | /// `std::string` has a `_M_p` pointer that aliases the source's internal |
| 1491 | /// `_M_local_buf`, so a byte-copy leaves the destination pointing at the |
| 1492 | /// source's (now-dying) stack storage and the destination's destructor |
| 1493 | /// later `free()`s a stack pointer. |
| 1494 | /// |
| 1495 | /// Instead, route construction directly into the sret slot: find the |
| 1496 | /// `__retval` alloca, replace its uses with the sret pointer, and drop the |
| 1497 | /// trailing `cir.load __retval` so the rewritten return has no operand. |
| 1498 | /// The CIRGen-emitted constructor / store-into-`__retval` then targets the |
| 1499 | /// sret slot uniformly, matching classic CodeGen's "construct directly into |
| 1500 | /// `%agg.result`" pattern. |
| 1501 | /// |
| 1502 | /// CIRGen emits one `%v = cir.load %__retval` / `cir.return %v` pair per |
| 1503 | /// return statement, and every such load reads the single `__retval` |
| 1504 | /// alloca (CIR does not merge returns into a shared epilogue block). The |
| 1505 | /// alloca is therefore rewired to the sret pointer once; each cir.return is |
| 1506 | /// then collapsed to a bare return and its now-dead load erased. This |
| 1507 | /// `cir.return (cir.load <alloca>)` shape is an invariant guaranteed by |
| 1508 | /// CIRGen, so it is asserted via `cast<>` rather than guarded with a |
| 1509 | /// fallback. |
| 1510 | /// |
| 1511 | /// A `__retval` outside the sret pointer's address space is not replaced |
| 1512 | /// itself. Its address_space casts to the sret pointer's type are. |
| 1513 | void insertSRetStores(mlir::FunctionOpInterface funcOp, mlir::Type origRetTy, |
| 1514 | mlir::OpBuilder &builder) { |
| 1515 | mlir::Value sretPtr = funcOp.getArguments()[0]; |
| 1516 | |
| 1517 | SmallVector<cir::ReturnOp> returnOps; |
| 1518 | funcOp->walk(callback: [&](cir::ReturnOp retOp) { returnOps.push_back(Elt: retOp); }); |
| 1519 | |
| 1520 | cir::AllocaOp retAlloca = nullptr; |
| 1521 | for (cir::ReturnOp retOp : returnOps) { |
| 1522 | // Every cir.return in an sret function must carry the loaded return |
| 1523 | // value -- a bare return would mean the sret slot was never written. |
| 1524 | assert(!retOp.getInput().empty() && |
| 1525 | "cir.return in sret function must have an operand" ); |
| 1526 | |
| 1527 | cir::LoadOp retLoad = |
| 1528 | mlir::cast<cir::LoadOp>(Val: retOp.getInput()[0].getDefiningOp()); |
| 1529 | |
| 1530 | // Rewire the shared `__retval` alloca to the sret pointer once. |
| 1531 | // replaceAllUsesWith updates every load of the alloca (including those |
| 1532 | // feeding the other cir.return ops) to read from sretPtr instead, so |
| 1533 | // all returns are covered by this single rewiring. |
| 1534 | if (!retAlloca) { |
| 1535 | retAlloca = mlir::cast<cir::AllocaOp>(Val: retLoad.getAddr().getDefiningOp()); |
| 1536 | if (retAlloca.getType() == sretPtr.getType()) |
| 1537 | retAlloca.getResult().replaceAllUsesWith(newValue: sretPtr); |
| 1538 | else |
| 1539 | replaceAddrSpaceViews(slot: retAlloca, ptr: sretPtr); |
| 1540 | } |
| 1541 | |
| 1542 | // The sret slot now holds the return value directly; replace the |
| 1543 | // value-carrying return with a void return (no operand). |
| 1544 | builder.setInsertionPoint(retOp); |
| 1545 | cir::ReturnOp::create(builder, location: retOp.getLoc()); |
| 1546 | retOp->erase(); |
| 1547 | if (retLoad.use_empty()) |
| 1548 | retLoad->erase(); |
| 1549 | } |
| 1550 | |
| 1551 | if (retAlloca) { |
| 1552 | assert(retAlloca->use_empty() && |
| 1553 | "__retval used other than by its returns and address_space views" ); |
| 1554 | retAlloca->erase(); |
| 1555 | } |
| 1556 | } |
| 1557 | |
| 1558 | /// Build the attribute dictionary for the sret slot (slot 0 of an |
| 1559 | /// sret-returning function or call). Matches classic CodeGen's |
| 1560 | /// `sret(T) align A [noalias] writable dead_on_unwind`. noalias is only |
| 1561 | /// valid on the callee's parameter, not at the call site, so it is gated by |
| 1562 | /// \p withNoalias. Key order is irrelevant: DictionaryAttr sorts by name. |
| 1563 | SmallVector<mlir::NamedAttribute> buildSretSlotAttrs(mlir::OpBuilder &builder, |
| 1564 | mlir::Type retTy, |
| 1565 | uint64_t align, |
| 1566 | bool withNoalias) { |
| 1567 | SmallVector<mlir::NamedAttribute> attrs; |
| 1568 | // The sret type must be carried explicitly: LLVM's sret attribute requires |
| 1569 | // it, and once the CIR `!cir.ptr<retTy>` lowers to an opaque LLVM `ptr` the |
| 1570 | // pointee type can no longer be recovered from the pointer. |
| 1571 | attrs.push_back( |
| 1572 | Elt: builder.getNamedAttr(name: "llvm.sret" , val: mlir::TypeAttr::get(type: retTy))); |
| 1573 | attrs.push_back( |
| 1574 | Elt: builder.getNamedAttr(name: "llvm.align" , val: builder.getI64IntegerAttr(value: align))); |
| 1575 | if (withNoalias) |
| 1576 | attrs.push_back( |
| 1577 | Elt: builder.getNamedAttr(name: "llvm.noalias" , val: builder.getUnitAttr())); |
| 1578 | attrs.push_back(Elt: builder.getNamedAttr(name: "llvm.writable" , val: builder.getUnitAttr())); |
| 1579 | attrs.push_back( |
| 1580 | Elt: builder.getNamedAttr(name: "llvm.dead_on_unwind" , val: builder.getUnitAttr())); |
| 1581 | return attrs; |
| 1582 | } |
| 1583 | |
| 1584 | /// Prepend the sret slot's attrs at position 0 of newCall's arg_attrs. |
| 1585 | /// Called after the call has been rewritten with the sret pointer at |
| 1586 | /// operand 0, so the operand count now includes the sret slot. \p argAttrs |
| 1587 | /// must already be shaped for the rewritten argument list (Extend slots |
| 1588 | /// carry signext/zeroext, Ignore slots dropped); it is shifted to slots |
| 1589 | /// 1..N behind the sret slot. |
| 1590 | void applySretSlotAttrs(cir::CallOp newCall, mlir::ArrayAttr argAttrs, |
| 1591 | mlir::Type retTy, uint64_t align, |
| 1592 | mlir::OpBuilder &builder) { |
| 1593 | mlir::MLIRContext *ctx = newCall->getContext(); |
| 1594 | SmallVector<mlir::NamedAttribute> sretAttrs = |
| 1595 | buildSretSlotAttrs(builder, retTy, align, /*withNoalias=*/false); |
| 1596 | |
| 1597 | SmallVector<mlir::Attribute> newArgAttrs; |
| 1598 | newArgAttrs.reserve(N: newCall.getArgOperands().size()); |
| 1599 | newArgAttrs.push_back(Elt: mlir::DictionaryAttr::get(context: ctx, value: sretAttrs)); |
| 1600 | if (argAttrs) |
| 1601 | llvm::append_range(C&: newArgAttrs, R&: argAttrs); |
| 1602 | assert(newArgAttrs.size() <= newCall.getArgOperands().size() && |
| 1603 | "arg_attrs wider than the rewritten call's operand list" ); |
| 1604 | newArgAttrs.resize(N: newCall.getArgOperands().size(), |
| 1605 | NV: mlir::DictionaryAttr::get(context: ctx)); |
| 1606 | newCall->setAttr(name: "arg_attrs" , value: mlir::ArrayAttr::get(context: ctx, value: newArgAttrs)); |
| 1607 | } |
| 1608 | |
| 1609 | /// Copy the call attributes from \p source to the rebuilt \p target. The |
| 1610 | /// callee is already established by CallOp::create and may have been rewritten, |
| 1611 | /// so it is not copied. Existing discardable attributes on \p target take |
| 1612 | /// precedence over attributes from \p source. |
| 1613 | static void copyCallAttributes(cir::CallOp source, cir::CallOp target) { |
| 1614 | source->getName().walkInherentAttrs(op: source, visitor: [&](llvm::StringRef name, |
| 1615 | mlir::Attribute &attr) { |
| 1616 | if (name != cir::CIRDialect::getCalleeAttrName()) |
| 1617 | target->setInherentAttr(name: mlir::StringAttr::get(context: source->getContext(), bytes: name), |
| 1618 | value: attr); |
| 1619 | }); |
| 1620 | for (mlir::NamedAttribute attr : source->getDiscardableAttrs()) |
| 1621 | if (!target->hasDiscardableAttr(name: attr.getName())) |
| 1622 | target->setDiscardableAttr(name: attr.getName(), value: attr.getValue()); |
| 1623 | } |
| 1624 | |
| 1625 | /// For an indirect call, prepend the callee function pointer as operand 0 so |
| 1626 | /// CallOp::create rebuilds it as an indirect call, bitcasting it to a function |
| 1627 | /// pointer that takes \p args, less the last \p numEllipsisArgs, and returns |
| 1628 | /// \p retTy. Those last arguments were passed through the callee's ellipsis |
| 1629 | /// and stay out of the function type, as in classic CodeGen's |
| 1630 | /// GetFunctionType: LLVM treats operands past a call's function type as |
| 1631 | /// variadic, and some backends pass variadic arguments differently. No-op |
| 1632 | /// for direct calls. |
| 1633 | static void prependIndirectCallee(cir::CallOp call, |
| 1634 | SmallVectorImpl<mlir::Value> &args, |
| 1635 | mlir::Type retTy, unsigned numEllipsisArgs, |
| 1636 | mlir::OpBuilder &builder) { |
| 1637 | if (!call.isIndirect()) |
| 1638 | return; |
| 1639 | assert(numEllipsisArgs <= args.size() && |
| 1640 | "more ellipsis arguments than arguments" ); |
| 1641 | mlir::Value calleePtr = call.getIndirectCall(); |
| 1642 | SmallVector<mlir::Type> paramTypes; |
| 1643 | paramTypes.reserve(N: args.size() - numEllipsisArgs); |
| 1644 | llvm::transform(Range: ArrayRef(args).drop_back(N: numEllipsisArgs), |
| 1645 | d_first: std::back_inserter(x&: paramTypes), |
| 1646 | F: [](mlir::Value v) { return v.getType(); }); |
| 1647 | // Lowering builds an indirect call's LLVM function type from the callee |
| 1648 | // pointer's pointee and takes the call's result from that type, so the |
| 1649 | // pointee's return type has to track the rewrite: an sret return would |
| 1650 | // leave a result the call no longer produces, and a coerced return one of |
| 1651 | // the wrong type. The ellipsis has to survive for the same reason: the |
| 1652 | // rebuilt pointee is what makes the lowered call variadic, and only a |
| 1653 | // variadic call gets the vector-register count that the x86_64 SysV ABI |
| 1654 | // passes in AL and that the callee's va_arg reads back. |
| 1655 | cir::FuncType calleeFnTy = getIndirectCalleeType(call); |
| 1656 | assert((calleeFnTy.isVarArg() || numEllipsisArgs == 0) && |
| 1657 | "only a variadic callee takes arguments through an ellipsis" ); |
| 1658 | auto newPtrTy = cir::PointerType::get( |
| 1659 | pointee: cir::FuncType::get(inputs: paramTypes, returnType: retTy, isVarArg: calleeFnTy.isVarArg())); |
| 1660 | if (calleePtr.getType() != newPtrTy) |
| 1661 | calleePtr = cir::CastOp::create(builder, location: call.getLoc(), result: newPtrTy, |
| 1662 | kind: cir::CastKind::bitcast, src: calleePtr); |
| 1663 | args.insert(I: args.begin(), Elt: calleePtr); |
| 1664 | } |
| 1665 | |
| 1666 | /// Rewrite an indirect-return (sret) call site: prepend a return-slot pointer |
| 1667 | /// as operand 0, make the call return void, and either reuse a dominating |
| 1668 | /// single-use store destination as the slot (so construction flows directly |
| 1669 | /// into it) or allocate a fresh slot and point the result's uses at it, a |
| 1670 | /// record's through rewireRecordUses and any other type's through a load of the |
| 1671 | /// slot after the call. \p newArgs is the already-shaped (Ignore-dropped, |
| 1672 | /// coercion-applied) non-sret argument list, the last \p numEllipsisArgs of |
| 1673 | /// which were passed through the callee's ellipsis. The caller guarantees the |
| 1674 | /// call has a result and an indirect-return classification. Fails, after |
| 1675 | /// reporting it, when rewireRecordUses does. |
| 1676 | mlir::LogicalResult |
| 1677 | rewriteIndirectReturnCall(cir::CallOp call, const FunctionClassification &fc, |
| 1678 | ArrayRef<mlir::Value> newArgs, |
| 1679 | unsigned numEllipsisArgs, mlir::Type origRetTy, |
| 1680 | ArrayRef<mlir::Type> origCallArgTypes, |
| 1681 | mlir::OpBuilder &builder, const mlir::DataLayout &dl, |
| 1682 | bool enableNoundefAttrs) { |
| 1683 | mlir::MLIRContext *ctx = call->getContext(); |
| 1684 | cir::PointerType ptrTy = getIndirectPtrTy(pointee: origRetTy, ac: fc.returnInfo); |
| 1685 | builder.setInsertionPoint(call); |
| 1686 | uint64_t sretAlign = fc.returnInfo.indirectAlign.value(); |
| 1687 | |
| 1688 | // CIRGen emits `cir.store %callResult, %dest` when the call's result is |
| 1689 | // bound to a local (e.g. `T s = make();`). Allocating a fresh sret slot |
| 1690 | // and copying into %dest would byte-copy the record, which is wrong for |
| 1691 | // non-trivially-copyable types (the libstdc++ SSO `_M_p` pointer |
| 1692 | // survives a byte-copy but ends up pointing at the dying temp's local |
| 1693 | // buffer, so the destination's destructor later `free()`s a stack |
| 1694 | // pointer). When the result has a single store-into-%dest use, use |
| 1695 | // %dest as the sret slot directly so construction flows into it, |
| 1696 | // matching classic CodeGen's "pass %s as sret" pattern. %dest must |
| 1697 | // dominate the call so the rewritten call (which takes it as operand 0) |
| 1698 | // does not use a value before its definition. An address_space cast of a |
| 1699 | // slot that already has the sret pointer's type stands for that slot. |
| 1700 | mlir::Value sretSlot = nullptr; |
| 1701 | cir::StoreOp reuseStore = nullptr; |
| 1702 | if (call.getResult().hasOneUse()) { |
| 1703 | mlir::Operation *user = *call.getResult().getUsers().begin(); |
| 1704 | auto store = mlir::dyn_cast<cir::StoreOp>(Val: user); |
| 1705 | if (store && store.getValue() == call.getResult()) { |
| 1706 | mlir::Value dest = store.getAddr(); |
| 1707 | if (auto view = dest.getDefiningOp<cir::CastOp>(); |
| 1708 | view && view.getKind() == cir::CastKind::address_space && |
| 1709 | view.getSrc().getType() == ptrTy) |
| 1710 | dest = view.getSrc(); |
| 1711 | if (dest.getType() == ptrTy && |
| 1712 | mlir::DominanceInfo().properlyDominates(a: dest, b: call)) { |
| 1713 | sretSlot = dest; |
| 1714 | reuseStore = store; |
| 1715 | } |
| 1716 | } |
| 1717 | } |
| 1718 | if (!sretSlot) { |
| 1719 | cir::PointerType allocaPtrTy = getAllocaPtrTy(pointee: origRetTy, dl); |
| 1720 | sretSlot = cir::AllocaOp::create( |
| 1721 | builder, location: call.getLoc(), addr: allocaPtrTy, |
| 1722 | /*name=*/builder.getStringAttr(bytes: "sret" ), |
| 1723 | /*alignment=*/builder.getI64IntegerAttr(value: sretAlign)); |
| 1724 | // The sret pointer can be outside the alloca address space. |
| 1725 | if (allocaPtrTy != ptrTy) |
| 1726 | sretSlot = cir::CastOp::create(builder, location: call.getLoc(), result: ptrTy, |
| 1727 | kind: cir::CastKind::address_space, src: sretSlot); |
| 1728 | } |
| 1729 | |
| 1730 | SmallVector<mlir::Value> sretArgs; |
| 1731 | sretArgs.push_back(Elt: sretSlot); |
| 1732 | sretArgs.append(in_start: newArgs.begin(), in_end: newArgs.end()); |
| 1733 | |
| 1734 | // numEllipsisArgs counts from the end, so the sret slot prepended here stays |
| 1735 | // in the retyped signature, as it does in classic CodeGen. |
| 1736 | mlir::Type sretVoidTy = cir::VoidType::get(ctx); |
| 1737 | prependIndirectCallee(call, args&: sretArgs, retTy: sretVoidTy, numEllipsisArgs, builder); |
| 1738 | auto newCall = cir::CallOp::create( |
| 1739 | builder, location: call.getLoc(), callee: call.getCalleeAttr(), resType: sretVoidTy, operands: sretArgs); |
| 1740 | copyCallAttributes(source: call, target: newCall); |
| 1741 | newCall->removeAttr(name: "res_attrs" ); |
| 1742 | |
| 1743 | // Shape the per-argument attrs exactly as the non-sret path does, through |
| 1744 | // updateArgAttrs, before prepending the sret slot, so sret composes with |
| 1745 | // every argument classification. |
| 1746 | mlir::ArrayAttr argAttrs = call->getAttrOfType<mlir::ArrayAttr>(name: "arg_attrs" ); |
| 1747 | if (needsArgAttrUpdate(fc)) |
| 1748 | argAttrs = |
| 1749 | updateArgAttrs(ctx, module: call->getParentOfType<mlir::ModuleOp>(), |
| 1750 | origArgTypes: origCallArgTypes, existingArgAttrs: argAttrs, fc, dl, enableNoundefAttrs, |
| 1751 | /*isCallSite=*/true); |
| 1752 | applySretSlotAttrs(newCall, argAttrs, retTy: origRetTy, align: sretAlign, builder); |
| 1753 | |
| 1754 | if (reuseStore) { |
| 1755 | // The callee now constructs directly into the destination slot, so the |
| 1756 | // original store-from-result is redundant; dropping it avoids a |
| 1757 | // byte-copy of the record. |
| 1758 | reuseStore->erase(); |
| 1759 | } else if (mlir::isa<cir::RecordType>(Val: origRetTy)) { |
| 1760 | builder.setInsertionPointAfter(newCall); |
| 1761 | SmallPtrSet<mlir::Operation *, 1> noExceptions; |
| 1762 | if (failed(Result: rewireRecordUses(builder, loc: call.getLoc(), val: call.getResult(), |
| 1763 | addr: sretSlot, addrAlign: sretAlign, dl, exceptions: noExceptions))) |
| 1764 | return mlir::failure(); |
| 1765 | } else { |
| 1766 | builder.setInsertionPointAfter(newCall); |
| 1767 | auto load = cir::LoadOp::create(builder, location: call.getLoc(), result: origRetTy, addr: sretSlot, |
| 1768 | /*isDeref=*/mlir::UnitAttr(), |
| 1769 | /*isVolatile=*/is_volatile: mlir::UnitAttr(), |
| 1770 | /*is_nontemporal=*/mlir::UnitAttr(), |
| 1771 | /*alignment=*/mlir::IntegerAttr(), |
| 1772 | /*sync_scope=*/cir::SyncScopeKindAttr(), |
| 1773 | /*mem_order=*/cir::MemOrderAttr(), |
| 1774 | /*invariant=*/mlir::UnitAttr()); |
| 1775 | call.getResult().replaceAllUsesWith(newValue: load); |
| 1776 | } |
| 1777 | call->erase(); |
| 1778 | return mlir::success(); |
| 1779 | } |
| 1780 | |
| 1781 | /// Whether \p ty, a type the classifier named for one register of a |
| 1782 | /// coercion, is carried in a vector register. |
| 1783 | bool isSSERegisterClass(mlir::Type ty) { |
| 1784 | return mlir::isa<cir::VectorType, cir::FPTypeInterface>(Val: ty); |
| 1785 | } |
| 1786 | |
| 1787 | } // namespace |
| 1788 | |
| 1789 | cir::FuncType cir::getIndirectCalleeType(cir::CIRCallOpInterface call) { |
| 1790 | assert(call.isIndirect() && "expected an indirect call" ); |
| 1791 | return cast<cir::FuncType>( |
| 1792 | Val: cast<cir::PointerType>(Val: call.getIndirectCall().getType()).getPointee()); |
| 1793 | } |
| 1794 | |
| 1795 | /// Bring \p funcOp's non-byval indirect parameter \p argNo into the shape the |
| 1796 | /// rest of the rewrite assumes. \p claimedSlots carries the slots \p funcOp's |
| 1797 | /// earlier non-byval indirect parameters took. The shape itself, and the |
| 1798 | /// cases reported rather than repaired, are documented on |
| 1799 | /// CIRABIRewriteContext::prepareNonByvalParameters in the header. |
| 1800 | static mlir::LogicalResult prepareOneNonByvalParameter( |
| 1801 | cir::FuncOp funcOp, unsigned argNo, const ArgClassification &ac, |
| 1802 | mlir::BlockArgument blockArg, mlir::DominanceInfo &dom, |
| 1803 | SmallPtrSetImpl<mlir::Operation *> &claimedSlots) { |
| 1804 | cir::PointerType paramPtrTy = getIndirectPtrTy(pointee: blockArg.getType(), ac); |
| 1805 | |
| 1806 | // The spill is the store that writes the parameter itself. Any other use |
| 1807 | // consumes the record value. At -O1 and above such a use comes from |
| 1808 | // cir-simplify: CIRGen marks a const-qualified parameter's slot const, so |
| 1809 | // the load of it folds to the stored parameter. |
| 1810 | cir::StoreOp spill; |
| 1811 | cir::StoreOp ; |
| 1812 | mlir::Operation *otherUse = nullptr; |
| 1813 | SmallVector<mlir::OpOperand *> callArgs; |
| 1814 | for (mlir::OpOperand &use : blockArg.getUses()) { |
| 1815 | auto store = dyn_cast<cir::StoreOp>(Val: use.getOwner()); |
| 1816 | if (store && store.getValue() == blockArg) { |
| 1817 | // Which of two stores is the spill decides which slot stands in for the |
| 1818 | // parameter, and the use list is in no particular order, so there is |
| 1819 | // nothing to prefer between them. |
| 1820 | if (spill) |
| 1821 | extraSpill = store; |
| 1822 | else |
| 1823 | spill = store; |
| 1824 | continue; |
| 1825 | } |
| 1826 | // Only an argument is served by a load of the slot. Any other consumer |
| 1827 | // belongs to a rewrite that reads the parameter its own way: a returned |
| 1828 | // record, for one, is rewritten through the sret slot, which assumes the |
| 1829 | // returned load names the return slot and not this one. |
| 1830 | auto call = dyn_cast<cir::CIRCallOpInterface>(Val: use.getOwner()); |
| 1831 | if (call && llvm::is_contained(Range: call.getArgOperands(), Element: blockArg)) |
| 1832 | callArgs.push_back(Elt: &use); |
| 1833 | else if (!otherUse) |
| 1834 | otherUse = use.getOwner(); |
| 1835 | } |
| 1836 | |
| 1837 | if (extraSpill) |
| 1838 | return extraSpill->emitOpError() |
| 1839 | << "non-byval parameter " << argNo |
| 1840 | << " spilled more than once is not yet implemented in " |
| 1841 | "CallConvLowering" ; |
| 1842 | |
| 1843 | if (otherUse) |
| 1844 | return otherUse->emitOpError() |
| 1845 | << "non-byval parameter " << argNo |
| 1846 | << " consumed other than as a call argument is not yet implemented " |
| 1847 | "in CallConvLowering" ; |
| 1848 | |
| 1849 | if (!spill) { |
| 1850 | // Every other kind of use was reported above, so the parameter has no |
| 1851 | // uses at all and needs neither a slot nor a read. |
| 1852 | if (callArgs.empty()) |
| 1853 | return mlir::success(); |
| 1854 | |
| 1855 | // Without a spill the parameter becomes the incoming pointer directly, |
| 1856 | // which is the storage an argument taken from it has to name. Giving it |
| 1857 | // the spill it lacks lets the checks and the read below apply unchanged, |
| 1858 | // and neither survives the pass: insertArgCoercion erases the store and |
| 1859 | // finalizeParameterSlots replaces the slot. |
| 1860 | mlir::OpBuilder builder(funcOp.getContext()); |
| 1861 | builder.setInsertionPointToStart(blockArg.getOwner()); |
| 1862 | auto synthesized = cir::AllocaOp::create( |
| 1863 | builder, location: funcOp.getLoc(), addr: paramPtrTy, |
| 1864 | name: builder.getStringAttr(bytes: "nonbyval.param" ), |
| 1865 | alignment: builder.getI64IntegerAttr(value: ac.indirectAlign.value())); |
| 1866 | spill = |
| 1867 | cir::StoreOp::create(builder, location: funcOp.getLoc(), value: blockArg, addr: synthesized); |
| 1868 | } |
| 1869 | |
| 1870 | // The incoming pointer replaces the slot itself, which also covers a spill |
| 1871 | // through an address_space cast of it. Retargeting only the cast would |
| 1872 | // leave the allocation's other views reading storage nothing writes. |
| 1873 | // Storage in another address space than the incoming pointer, or storage |
| 1874 | // that is not a local alloca at all, is not something the incoming pointer |
| 1875 | // can be substituted for. |
| 1876 | cir::AllocaOp slot = getSpillSlot(addr: spill.getAddr()); |
| 1877 | if (!slot || slot.getType() != paramPtrTy) |
| 1878 | return spill->emitOpError() |
| 1879 | << "non-byval parameter " << argNo |
| 1880 | << " spilled to storage that cannot take the incoming pointer is " |
| 1881 | "not yet implemented in CallConvLowering" ; |
| 1882 | |
| 1883 | // One slot stands in for one non-byval parameter, since the incoming |
| 1884 | // pointer replaces it. Two of them spilled to the same slot are each |
| 1885 | // spilled once, so the check above cannot see the collision and only |
| 1886 | // comparing the slots can. |
| 1887 | if (!claimedSlots.insert(Ptr: slot).second) |
| 1888 | return spill->emitOpError() |
| 1889 | << "non-byval parameter " << argNo |
| 1890 | << " sharing its spill slot with another parameter is not yet " |
| 1891 | "implemented in CallConvLowering" ; |
| 1892 | |
| 1893 | // CIRGen picked the slot's alignment for a local copy of the record, but the |
| 1894 | // slot is about to stand in for the parameter, and a call forwarding it may |
| 1895 | // only promise what the incoming pointer does. |
| 1896 | slot.setAlignment(ac.indirectAlign.value()); |
| 1897 | |
| 1898 | if (callArgs.empty()) |
| 1899 | return mlir::success(); |
| 1900 | |
| 1901 | // A reader the spill does not dominate could read the slot before anything |
| 1902 | // wrote it, and a load placed at the spill would not dominate it either. |
| 1903 | for (mlir::OpOperand *callArg : callArgs) |
| 1904 | if (!dom.properlyDominates(a: spill, b: callArg->getOwner())) |
| 1905 | return callArg->getOwner()->emitOpError() |
| 1906 | << "non-byval parameter " << argNo |
| 1907 | << " read before its spill is not yet implemented in " |
| 1908 | "CallConvLowering" ; |
| 1909 | |
| 1910 | // Read the record back out of the slot instead, so every reader sees the |
| 1911 | // shape CIRGen emits without cir-simplify: a load naming the storage the |
| 1912 | // argument would have to name anyway. Reading at the spill, not at the |
| 1913 | // reader, keeps the value the parameter's own whatever the body later |
| 1914 | // stores into the slot, and the load promises only the alignment the |
| 1915 | // classification gives the incoming pointer. |
| 1916 | // |
| 1917 | // A call that forwards the argument consumes the load once it recognizes |
| 1918 | // the slot. One that wants a copy keeps it, reading the incoming pointer |
| 1919 | // once finalizeParameterSlots replaces the slot. |
| 1920 | mlir::OpBuilder builder(spill); |
| 1921 | builder.setInsertionPointAfter(spill); |
| 1922 | auto reload = cir::LoadOp::create(builder, location: spill.getLoc(), addr: slot); |
| 1923 | reload.setAlignment(ac.indirectAlign.value()); |
| 1924 | for (mlir::OpOperand *callArg : callArgs) |
| 1925 | callArg->set(reload.getResult()); |
| 1926 | return mlir::success(); |
| 1927 | } |
| 1928 | |
| 1929 | mlir::LogicalResult CIRABIRewriteContext::prepareNonByvalParameters( |
| 1930 | cir::FuncOp funcOp, const FunctionClassification &fc) { |
| 1931 | if (!funcOp.isDefinition()) |
| 1932 | return mlir::success(); |
| 1933 | mlir::Region &body = funcOp->getRegion(index: 0); |
| 1934 | if (body.empty()) |
| 1935 | return mlir::success(); |
| 1936 | mlir::Block &entry = body.front(); |
| 1937 | mlir::DominanceInfo dom; |
| 1938 | SmallPtrSet<mlir::Operation *, 4> claimedSlots; |
| 1939 | |
| 1940 | // No signature has been rewritten yet, so no sret pointer has been prepended |
| 1941 | // and no Expand argument has been split into its fields. Every |
| 1942 | // classification therefore still maps to the entry block argument at its own |
| 1943 | // index. |
| 1944 | for (auto [idx, ac] : llvm::enumerate(First: fc.argInfos)) { |
| 1945 | if (ac.kind != ArgKind::Indirect || ac.byVal) |
| 1946 | continue; |
| 1947 | assert(idx < entry.getNumArguments() && |
| 1948 | "classification count must not exceed entry block arguments" ); |
| 1949 | if (failed(Result: prepareOneNonByvalParameter( |
| 1950 | funcOp, argNo: idx, ac, blockArg: entry.getArgument(i: idx), dom, claimedSlots))) |
| 1951 | return mlir::failure(); |
| 1952 | } |
| 1953 | return mlir::success(); |
| 1954 | } |
| 1955 | |
| 1956 | void CIRABIRewriteContext::finalizeParameterSlots() { |
| 1957 | for (auto [slot, incoming] : pendingParamSlots) { |
| 1958 | slot.getResult().replaceAllUsesWith(newValue: incoming); |
| 1959 | slot->erase(); |
| 1960 | } |
| 1961 | pendingParamSlots.clear(); |
| 1962 | } |
| 1963 | |
| 1964 | mlir::LogicalResult CIRABIRewriteContext::rewriteFunctionDefinition( |
| 1965 | mlir::FunctionOpInterface funcOpInterface, const FunctionClassification &fc, |
| 1966 | mlir::OpBuilder &builder) { |
| 1967 | // The pass driver (CallConvLoweringPass) only ever hands us cir.func ops. |
| 1968 | // Cast once at the top so the rest of the function reads in CIR's own |
| 1969 | // vocabulary, and so we can dispatch to the CIRGlobalValueInterface for |
| 1970 | // isDefinition() (FunctionOpInterface alone does not inherit from |
| 1971 | // CIRGlobalValueInterface). |
| 1972 | cir::FuncOp funcOp = mlir::cast<cir::FuncOp>(Val&: funcOpInterface); |
| 1973 | |
| 1974 | if (!fc.needsRewrite()) |
| 1975 | return mlir::success(); |
| 1976 | |
| 1977 | ArrayRef<mlir::Type> oldArgTypes = funcOp.getArgumentTypes(); |
| 1978 | ArrayRef<mlir::Type> oldResultTypes = funcOp.getResultTypes(); |
| 1979 | mlir::MLIRContext *ctx = funcOp->getContext(); |
| 1980 | |
| 1981 | // CIR follows LLVM IR's single-result rule: a function returns either |
| 1982 | // zero or one value. Document the invariant so a future multi-result |
| 1983 | // change forces us to revisit the return-handling below. |
| 1984 | assert(oldResultTypes.size() <= 1 && |
| 1985 | "CIR functions return zero or one value" ); |
| 1986 | |
| 1987 | SmallVector<mlir::Type> newArgTypes; |
| 1988 | if (mlir::failed(Result: buildNewArgTypes(oldArgTypes, fc, newArgTypes, dl, |
| 1989 | emitError: [&]() { return funcOp.emitOpError(); }))) |
| 1990 | return mlir::failure(); |
| 1991 | |
| 1992 | mlir::Type voidTy = cir::VoidType::get(ctx); |
| 1993 | mlir::Type origRetTy = oldResultTypes.empty() ? voidTy : oldResultTypes[0]; |
| 1994 | mlir::Type newRetTy = computeNewReturnType( |
| 1995 | origRetTy, retInfo: fc.returnInfo, ctx, emitError: [&]() { return funcOp.emitOpError(); }); |
| 1996 | if (!newRetTy) |
| 1997 | return mlir::failure(); |
| 1998 | SmallVector<mlir::Type> newResultTypes = {newRetTy}; |
| 1999 | |
| 2000 | // sret return: the value is returned through a pointer the ABI inserts as |
| 2001 | // argument 0. This pointer is not part of the function's source-level |
| 2002 | // signature -- it is synthesized here -- and the wire return type was |
| 2003 | // already set to void by computeNewReturnType. Every classification index |
| 2004 | // therefore maps to a block argument shifted by one in the body handling |
| 2005 | // below. |
| 2006 | bool hasSRet = |
| 2007 | fc.returnInfo.kind == ArgKind::Indirect && !oldResultTypes.empty(); |
| 2008 | cir::PointerType sretPtrTy; |
| 2009 | if (hasSRet) { |
| 2010 | sretPtrTy = getIndirectPtrTy(pointee: origRetTy, ac: fc.returnInfo); |
| 2011 | newArgTypes.insert(I: newArgTypes.begin(), Elt: sretPtrTy); |
| 2012 | } |
| 2013 | |
| 2014 | if (funcOp.isDefinition()) { |
| 2015 | mlir::Region &body = funcOp->getRegion(index: 0); |
| 2016 | if (!body.empty()) { |
| 2017 | // Prepend the sret pointer block argument and route every cir.return |
| 2018 | // through it before any index-based argument handling below (which |
| 2019 | // then accounts for the +1 offset). |
| 2020 | if (hasSRet) { |
| 2021 | body.front().insertArgument(index: 0u, type: sretPtrTy, loc: funcOp.getLoc()); |
| 2022 | insertSRetStores(funcOp, origRetTy, builder); |
| 2023 | } |
| 2024 | |
| 2025 | // Reshape the entry block for the classifications that change what a |
| 2026 | // parameter looks like to the body: a Direct arg with a coerced type |
| 2027 | // gets a memory roundtrip back to its original type, an Indirect arg |
| 2028 | // becomes a pointer, and an Expand arg becomes one block argument per |
| 2029 | // field. Done before the Ignore-drop below so the entry block |
| 2030 | // argument indices used here still refer to the original positions. |
| 2031 | if (failed(Result: insertArgCoercion(funcOp, fc, builder, dl, hasSRetArg: hasSRet, |
| 2032 | pendingParamSlots))) |
| 2033 | return mlir::failure(); |
| 2034 | |
| 2035 | // Direct return with coerced type: insert a coercion at every |
| 2036 | // cir.return so the returned value matches the (coerced) return |
| 2037 | // type in the new function signature set below. |
| 2038 | if (fc.returnInfo.kind == ArgKind::Direct && fc.returnInfo.coercedType && |
| 2039 | !oldResultTypes.empty() && fc.returnInfo.coercedType != origRetTy && |
| 2040 | failed(Result: insertReturnCoercion( |
| 2041 | funcOp, origRetTy, coercedRetTy: fc.returnInfo.coercedType, builder, dl, |
| 2042 | cirDataLayout, offset: fc.returnInfo.directOffset))) |
| 2043 | return mlir::failure(); |
| 2044 | |
| 2045 | mlir::Block &entry = body.front(); |
| 2046 | |
| 2047 | // Drop each Ignored argument's block argument, replacing any remaining |
| 2048 | // body uses with a poison constant (an Ignore arg is not passed at the |
| 2049 | // ABI level, so any use is vacuous; poison says exactly that). Walk |
| 2050 | // forward with a running block-argument index that mirrors |
| 2051 | // insertArgCoercion: an Expand arg or a Direct+canFlatten arg occupies N |
| 2052 | // slots, every other kept kind one. On erase, do not advance the index |
| 2053 | // -- the next block argument shifts into the vacated slot. |
| 2054 | unsigned blockArgIdx = hasSRet ? 1 : 0; |
| 2055 | for (auto [i, ac] : llvm::enumerate(First: fc.argInfos)) { |
| 2056 | if (blockArgIdx >= entry.getNumArguments()) |
| 2057 | break; |
| 2058 | if (ac.kind == ArgKind::Ignore) { |
| 2059 | mlir::BlockArgument arg = entry.getArgument(i: blockArgIdx); |
| 2060 | if (!arg.use_empty()) { |
| 2061 | builder.setInsertionPointToStart(&entry); |
| 2062 | mlir::Value poison = |
| 2063 | createIgnoredValue(builder, loc: funcOp.getLoc(), ty: arg.getType()); |
| 2064 | arg.replaceAllUsesWith(newValue: poison); |
| 2065 | } |
| 2066 | entry.eraseArgument(index: blockArgIdx); |
| 2067 | continue; |
| 2068 | } |
| 2069 | if (cir::RecordType flatTy = getFlattenedCoercedType(ac, dl)) |
| 2070 | blockArgIdx += flatTy.getNumElements(); |
| 2071 | else if (ac.kind == ArgKind::Expand) |
| 2072 | blockArgIdx += cast<cir::RecordType>(Val: oldArgTypes[i]).getNumElements(); |
| 2073 | else |
| 2074 | ++blockArgIdx; |
| 2075 | } |
| 2076 | } |
| 2077 | |
| 2078 | // When the return is classified Ignore but the original function had |
| 2079 | // a non-void return type, every cir.return becomes a naked return. |
| 2080 | // This relies on the invariant that computeNewReturnType has set |
| 2081 | // newRetTy = void for Ignore above, and that the function type is |
| 2082 | // updated below to match. Asserting this keeps the dependency |
| 2083 | // explicit. |
| 2084 | if (fc.returnInfo.kind == ArgKind::Ignore && !oldResultTypes.empty()) { |
| 2085 | assert(mlir::isa<cir::VoidType>(newRetTy) && |
| 2086 | "Ignore-return path requires the new return type to be void" ); |
| 2087 | SmallVector<cir::ReturnOp> returns; |
| 2088 | funcOp.walk(callback: [&](cir::ReturnOp r) { returns.push_back(Elt: r); }); |
| 2089 | for (cir::ReturnOp r : returns) { |
| 2090 | if (r.getNumOperands() == 0) |
| 2091 | continue; |
| 2092 | builder.setInsertionPoint(r); |
| 2093 | cir::ReturnOp::create(builder, location: r.getLoc()); |
| 2094 | r.erase(); |
| 2095 | } |
| 2096 | } |
| 2097 | } |
| 2098 | |
| 2099 | mlir::Type newFnTy = funcOp.cloneTypeWith(inputs: newArgTypes, results: newResultTypes); |
| 2100 | funcOp.setFunctionTypeAttr(mlir::TypeAttr::get(type: newFnTy)); |
| 2101 | |
| 2102 | // Rebuild arg_attrs when the function has an sret slot (slot 0 needs the |
| 2103 | // sret attribute set) or any arg needs updateArgAttrs. |
| 2104 | if (hasSRet || needsArgAttrUpdate(fc)) { |
| 2105 | auto existing = funcOp->getAttrOfType<mlir::ArrayAttr>(name: "arg_attrs" ); |
| 2106 | mlir::ArrayAttr updated = updateArgAttrs(ctx, module, origArgTypes: oldArgTypes, existingArgAttrs: existing, |
| 2107 | fc, dl, enableNoundefAttrs, |
| 2108 | /*isCallSite=*/false); |
| 2109 | if (hasSRet) { |
| 2110 | // Prepend the sret slot's attribute dict (slot 0); the per-argument |
| 2111 | // dicts shift to slots 1..N. noalias is valid only on the callee's |
| 2112 | // parameter, so it is added only for definitions. |
| 2113 | SmallVector<mlir::NamedAttribute> sretAttrs = buildSretSlotAttrs( |
| 2114 | builder, retTy: origRetTy, align: fc.returnInfo.indirectAlign.value(), |
| 2115 | /*withNoalias=*/funcOp.isDefinition()); |
| 2116 | SmallVector<mlir::Attribute> withSret; |
| 2117 | withSret.push_back(Elt: mlir::DictionaryAttr::get(context: ctx, value: sretAttrs)); |
| 2118 | llvm::append_range(C&: withSret, R&: updated); |
| 2119 | funcOp->setAttr(name: "arg_attrs" , value: mlir::ArrayAttr::get(context: ctx, value: withSret)); |
| 2120 | } else { |
| 2121 | funcOp->setAttr(name: "arg_attrs" , value: updated); |
| 2122 | } |
| 2123 | } |
| 2124 | |
| 2125 | if (mlir::isa<cir::VoidType>(Val: newRetTy)) { |
| 2126 | funcOp->removeAttr(name: "res_attrs" ); |
| 2127 | } else if (fc.returnInfo.kind == ArgKind::Extend || |
| 2128 | fc.returnInfo.kind == ArgKind::Direct) { |
| 2129 | auto existing = funcOp->getAttrOfType<mlir::ArrayAttr>(name: "res_attrs" ); |
| 2130 | if (mlir::ArrayAttr updated = |
| 2131 | updateResAttrs(ctx, existingResAttrs: existing, retInfo: fc.returnInfo, origRetTy, dl)) |
| 2132 | funcOp->setAttr(name: "res_attrs" , value: updated); |
| 2133 | } |
| 2134 | |
| 2135 | return mlir::success(); |
| 2136 | } |
| 2137 | |
| 2138 | mlir::LogicalResult |
| 2139 | CIRABIRewriteContext::rewriteCallSite(mlir::Operation *callOp, |
| 2140 | const FunctionClassification &fc, |
| 2141 | mlir::OpBuilder &builder) { |
| 2142 | // The rewrite below pairs the classification with the call's operands one |
| 2143 | // for one, so a call passing arguments through an ellipsis needs a |
| 2144 | // classification built from its own operands rather than from the callee's |
| 2145 | // declared parameters. Both directions of a mismatch have to be reported |
| 2146 | // before the pass-through early return, or a call whose declared parameters |
| 2147 | // happen to be pass-through is left as written with its surplus operands |
| 2148 | // never classified. |
| 2149 | // |
| 2150 | // A surplus operand means fc was built from the callee's declared |
| 2151 | // parameters alone. A shortfall means the callee was declared no_proto, |
| 2152 | // which turns off the verifier's argument-count check altogether. |
| 2153 | unsigned numOperands = |
| 2154 | mlir::cast<cir::CIRCallOpInterface>(Val: callOp).getNumArgOperands(); |
| 2155 | if (numOperands > fc.argInfos.size()) |
| 2156 | return callOp->emitOpError() |
| 2157 | << "variadic arguments not yet implemented in CallConvLowering" ; |
| 2158 | if (numOperands < fc.argInfos.size()) |
| 2159 | return callOp->emitOpError() |
| 2160 | << "call passes fewer arguments than the callee declares, which is " |
| 2161 | "not yet implemented in CallConvLowering" ; |
| 2162 | |
| 2163 | if (!fc.needsRewrite()) |
| 2164 | return mlir::success(); |
| 2165 | |
| 2166 | if (mlir::isa<cir::TryCallOp>(Val: callOp)) |
| 2167 | return callOp->emitOpError() |
| 2168 | << "TryCallOp not yet implemented in CallConvLowering" ; |
| 2169 | |
| 2170 | auto call = mlir::cast<cir::CallOp>(Val: callOp); |
| 2171 | mlir::MLIRContext *ctx = callOp->getContext(); |
| 2172 | mlir::Block *slotBlock = coercionSlotBlock(op: call); |
| 2173 | |
| 2174 | builder.setInsertionPoint(call); |
| 2175 | |
| 2176 | SmallVector<mlir::Value> newArgs; |
| 2177 | mlir::ValueRange argOperands = call.getArgOperands(); |
| 2178 | newArgs.reserve(N: argOperands.size()); |
| 2179 | |
| 2180 | // Loads that the new call leaves unused: a Direct (coerced or flattened) or |
| 2181 | // Expand record is read out of its load's storage or a copy of it, and an |
| 2182 | // Indirect argument passes or copies from the address the load read from. |
| 2183 | // The old call still uses them, so erase them only after it is gone. |
| 2184 | SmallVector<cir::LoadOp> deadRecordLoads; |
| 2185 | // Erased before deadRecordLoads, since a freeze keeps its load alive. |
| 2186 | SmallVector<cir::FreezeOp> deadFreezes; |
| 2187 | |
| 2188 | // Capture original arg types before building newArgs (an Indirect argument |
| 2189 | // changes the wire argument from T to !cir.ptr<T>, so we save the |
| 2190 | // pre-rewrite types here for use in updateArgAttrs). |
| 2191 | SmallVector<mlir::Type> origCallArgTypes; |
| 2192 | llvm::append_range(C&: origCallArgTypes, R: argOperands.getTypes()); |
| 2193 | |
| 2194 | // Operands past an indirect callee's declared parameters were passed |
| 2195 | // through its ellipsis. Note where they begin in newArgs, so |
| 2196 | // prependIndirectCallee can leave them out of the retyped callee type. A |
| 2197 | // direct call has no callee pointer to retype, so it records no boundary. |
| 2198 | unsigned numDeclared = fc.argInfos.size(); |
| 2199 | if (call.isIndirect()) { |
| 2200 | numDeclared = getIndirectCalleeType(call).getNumInputs(); |
| 2201 | assert(numDeclared <= fc.argInfos.size() && |
| 2202 | "the classification covers at least the declared parameters" ); |
| 2203 | } |
| 2204 | // Recorded before the Ignore skip below, so an ignored first ellipsis |
| 2205 | // argument still marks where the ellipsis arguments begin in newArgs. |
| 2206 | std::optional<unsigned> ellipsisStart; |
| 2207 | // Where each argument starts in newArgs, and the freezes to reapply there |
| 2208 | // once every argument is rewritten. |
| 2209 | SmallVector<unsigned> argBegin; |
| 2210 | SmallVector<std::pair<unsigned, cir::FreezeOp>> refreezes; |
| 2211 | for (auto [idx, ac] : llvm::enumerate(First: fc.argInfos)) { |
| 2212 | if (idx == numDeclared) |
| 2213 | ellipsisStart = newArgs.size(); |
| 2214 | argBegin.push_back(Elt: newArgs.size()); |
| 2215 | mlir::Value arg = argOperands[idx]; |
| 2216 | auto freeze = arg.getDefiningOp<cir::FreezeOp>(); |
| 2217 | if (freeze && !freeze->hasOneUse()) |
| 2218 | freeze = {}; |
| 2219 | if (ac.kind == ArgKind::Ignore) { |
| 2220 | if (freeze) |
| 2221 | deadFreezes.push_back(Elt: freeze); |
| 2222 | continue; |
| 2223 | } |
| 2224 | |
| 2225 | // Refreeze after the ABI rewrite, as classic CodeGen does, so padding is |
| 2226 | // frozen too. Indirect freezes only the pointer, so keep non-load freezes. |
| 2227 | if (freeze && ((ac.kind != ArgKind::Indirect && |
| 2228 | ac.kind != ArgKind::IndirectAliased) || |
| 2229 | freeze.getInput().getDefiningOp<cir::LoadOp>())) { |
| 2230 | arg = freeze.getInput(); |
| 2231 | deadFreezes.push_back(Elt: freeze); |
| 2232 | refreezes.push_back(Elt: {idx, freeze}); |
| 2233 | } |
| 2234 | if (cir::RecordType flatTy = getFlattenedCoercedType(ac, dl)) { |
| 2235 | // Direct + canFlatten: pass one scalar call argument per field of the |
| 2236 | // ABI-coerced struct. A non-atomic record load is read in place or |
| 2237 | // through a copy of its storage, as emitCoercedRecordLoad decides, and |
| 2238 | // any other operand is stored into a coercion slot first. When the types |
| 2239 | // already match, the fields come from the struct value directly (reading |
| 2240 | // from its source alloca when possible). |
| 2241 | if (arg.getType() != flatTy) { |
| 2242 | SmallPtrSet<mlir::Operation *, 4> coercionOps; |
| 2243 | assert(!ac.directOffset && |
| 2244 | "the coerced struct starts at slot offset 0 here, so a " |
| 2245 | "flattened coercion cannot honor a direct offset" ); |
| 2246 | if (failed(Result: rejectAtomicRecordLoad(user: call, val: arg))) |
| 2247 | return mlir::failure(); |
| 2248 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 2249 | mlir::Value coercedPtr; |
| 2250 | uint64_t baseAlign = 0; |
| 2251 | if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(val: arg)) { |
| 2252 | mlir::Value base = recLoad.getAddr(); |
| 2253 | if (!recLoad.getIsVolatile() && |
| 2254 | canReadInPlace(load: recLoad, coercedTy: flatTy, dl: cirDataLayout, /*offset=*/0)) { |
| 2255 | cir::AllocaOp baseAlloca = cir::getUnderlyingAlloca(addr: base); |
| 2256 | assert(baseAlloca && "canReadInPlace requires an alloca" ); |
| 2257 | baseAlign = baseAlloca.getAlignment(); |
| 2258 | } else { |
| 2259 | base = emitCopiedCoercionSlot(builder, loc: call.getLoc(), load: recLoad, |
| 2260 | coercedTy: flatTy, slotBlock, dl, /*offset=*/0); |
| 2261 | } |
| 2262 | builder.setInsertionPoint(recLoad); |
| 2263 | coercedPtr = emitViewAtOffset(builder, loc: call.getLoc(), wantTy: flatTy, base, |
| 2264 | createdOps&: coercionOps, /*offset=*/0); |
| 2265 | deadRecordLoads.push_back(Elt: recLoad); |
| 2266 | } else { |
| 2267 | coercedPtr = |
| 2268 | emitCoercionToMemory(builder, loc: call.getLoc(), dstTy: flatTy, src: arg, |
| 2269 | slotBlock, dl, createdOps&: coercionOps, /*offset=*/0); |
| 2270 | } |
| 2271 | auto coercedPtrTy = mlir::cast<cir::PointerType>(Val: coercedPtr.getType()); |
| 2272 | for (auto [f, fieldTy] : llvm::enumerate(First: flatTy.getMembers())) { |
| 2273 | mlir::Type fieldPtrTy = |
| 2274 | cir::PointerType::get(pointee: fieldTy, addrSpace: coercedPtrTy.getAddrSpace()); |
| 2275 | auto fieldPtr = |
| 2276 | cir::GetMemberOp::create(builder, location: call.getLoc(), type: fieldPtrTy, |
| 2277 | value: coercedPtr, /*name=*/"" , /*index=*/f); |
| 2278 | auto field = cir::LoadOp::create(builder, location: call.getLoc(), result: fieldTy, |
| 2279 | addr: fieldPtr.getResult()); |
| 2280 | // Storage read in place may be less aligned than the coerced struct, |
| 2281 | // so each field claims only what its offset in it allows. |
| 2282 | if (baseAlign) |
| 2283 | field.setAlignment( |
| 2284 | llvm::MinAlign(A: baseAlign, B: flatTy.getElementOffset(dataLayout: dl, idx: f))); |
| 2285 | newArgs.push_back(Elt: field); |
| 2286 | } |
| 2287 | } else { |
| 2288 | emitStructFieldArgs(builder, loc: call.getLoc(), structVal: arg, recTy: flatTy, newArgs, |
| 2289 | deadRecordLoads); |
| 2290 | } |
| 2291 | } else if (ac.kind == ArgKind::Expand) { |
| 2292 | // Decompose the struct value into its constituent scalar fields and |
| 2293 | // pass each as a separate argument. |
| 2294 | auto recTy = cast<cir::RecordType>(Val: arg.getType()); |
| 2295 | assert(recTy.isStruct() && |
| 2296 | "Expand classification requires a struct type, not a union" ); |
| 2297 | emitStructFieldArgs(builder, loc: call.getLoc(), structVal: arg, recTy, newArgs, |
| 2298 | deadRecordLoads); |
| 2299 | } else if ((ac.kind == ArgKind::Direct && ac.coercedType && |
| 2300 | arg.getType() != ac.coercedType) || |
| 2301 | isRecordExtend(ac, origTy: arg.getType())) { |
| 2302 | if (failed(Result: rejectAtomicRecordLoad(user: call, val: arg))) |
| 2303 | return mlir::failure(); |
| 2304 | if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(val: arg)) { |
| 2305 | newArgs.push_back(Elt: emitCoercedRecordLoad( |
| 2306 | builder, loc: call.getLoc(), load: recLoad, coercedTy: ac.coercedType, slotBlock, dl, |
| 2307 | cirDataLayout, offset: ac.directOffset)); |
| 2308 | deadRecordLoads.push_back(Elt: recLoad); |
| 2309 | } else { |
| 2310 | newArgs.push_back(Elt: emitCoercion(builder, loc: call.getLoc(), dstTy: ac.coercedType, |
| 2311 | src: arg, slotBlock, dl, offset: ac.directOffset)); |
| 2312 | } |
| 2313 | } else if (ac.kind == ArgKind::Indirect || |
| 2314 | ac.kind == ArgKind::IndirectAliased) { |
| 2315 | uint64_t minAlign = ac.indirectAlign.value(); |
| 2316 | // The callee copies a byref object at entry, so the caller treats it |
| 2317 | // like byval. |
| 2318 | bool byRef = ac.kind == ArgKind::IndirectAliased; |
| 2319 | bool byCopy = byRef || ac.byVal; |
| 2320 | cir::PointerType ptrTy = getIndirectPtrTy(pointee: arg.getType(), ac); |
| 2321 | |
| 2322 | // A record operand that names storage is passed from that storage |
| 2323 | // rather than from the loaded value: a record value carries only the |
| 2324 | // fields of its type, so a store leaves the type's padding bytes |
| 2325 | // unwritten, and for a union those can be another member's live data. |
| 2326 | // A byval _BitInt is passed from its value, since the load and the |
| 2327 | // store below are its conversion between value and in-memory form. |
| 2328 | cir::LoadOp srcLoad; |
| 2329 | if (!byCopy || mlir::isa<cir::RecordType>(Val: arg.getType())) |
| 2330 | srcLoad = maybeGetSimpleLoad(val: arg); |
| 2331 | |
| 2332 | // A non-byval argument must name the caller's storage, so that the |
| 2333 | // object the callee operates on is the one the caller destroys. That |
| 2334 | // means passing the address the operand was loaded from rather than the |
| 2335 | // loaded value, so a store to that storage after the load is visible to |
| 2336 | // the callee. byval fills a slot below, this cannot. |
| 2337 | if (!byCopy) { |
| 2338 | mlir::Value fwdAddr = forwardableIndirectAddr(load: srcLoad, paramTy: ptrTy, minAlign); |
| 2339 | if (!fwdAddr) |
| 2340 | return call->emitOpError() |
| 2341 | << "non-byval indirect argument that does not name the " |
| 2342 | "caller's storage is not yet implemented in " |
| 2343 | "CallConvLowering" ; |
| 2344 | newArgs.push_back(Elt: fwdAddr); |
| 2345 | deadRecordLoads.push_back(Elt: srcLoad); |
| 2346 | continue; |
| 2347 | } |
| 2348 | |
| 2349 | // byval hands the callee a copy taken at the call, so naming the object's |
| 2350 | // own storage leaves it with the same copy and saves this one. That |
| 2351 | // holds only while nothing writes that storage between the load and the |
| 2352 | // call, since the operand carries the value as of the load. A volatile |
| 2353 | // load is not forwarded, since the copy byval takes at the call is not |
| 2354 | // volatile. |
| 2355 | if (mlir::Value fwdAddr = |
| 2356 | forwardableIndirectAddr(load: srcLoad, paramTy: ptrTy, minAlign); |
| 2357 | fwdAddr && storageUnwrittenBetween(load: srcLoad, call: call.getOperation(), |
| 2358 | aliasAnalysis)) { |
| 2359 | newArgs.push_back(Elt: fwdAddr); |
| 2360 | deadRecordLoads.push_back(Elt: srcLoad); |
| 2361 | continue; |
| 2362 | } |
| 2363 | |
| 2364 | // A byref slot has to be an alloca of the parameter's own type, which |
| 2365 | // only the alloca address space can hold. |
| 2366 | if (byRef && ptrTy != getAllocaPtrTy(pointee: arg.getType(), dl)) |
| 2367 | return call->emitOpError() |
| 2368 | << "byref argument outside the alloca address space that does " |
| 2369 | "not name forwardable storage is not yet implemented in " |
| 2370 | "CallConvLowering" ; |
| 2371 | |
| 2372 | // Otherwise fill a slot, at the load's position when there is one so |
| 2373 | // the copy reads the memory state the load read. |
| 2374 | if (failed(Result: rejectAtomicRecordLoad(user: call, val: arg))) |
| 2375 | return mlir::failure(); |
| 2376 | if (!srcLoad) |
| 2377 | srcLoad = maybeGetCopyableRecordLoad(val: arg); |
| 2378 | mlir::IntegerAttr slotAlign = builder.getI64IntegerAttr(value: minAlign); |
| 2379 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 2380 | if (srcLoad) |
| 2381 | builder.setInsertionPoint(srcLoad); |
| 2382 | |
| 2383 | auto slot = cir::AllocaOp::create( |
| 2384 | builder, location: call.getLoc(), addr: ptrTy, |
| 2385 | name: builder.getStringAttr(bytes: byRef ? "byref" : "byval" ), alignment: slotAlign); |
| 2386 | if (srcLoad) { |
| 2387 | auto copy = cir::CopyOp::create(builder, location: call.getLoc(), dst: slot, |
| 2388 | src: srcLoad.getAddr(), dst_alignment: slotAlign, |
| 2389 | src_alignment: loadSourceAlignment(load: srcLoad, builder)); |
| 2390 | copy.setIsVolatile(srcLoad.getIsVolatile()); |
| 2391 | deadRecordLoads.push_back(Elt: srcLoad); |
| 2392 | } else { |
| 2393 | // A record with no storage behind it, such as a constant, defines no |
| 2394 | // bytes beyond its fields. A producer rewritten later turns this store |
| 2395 | // into a copy through rewireRecordUses. |
| 2396 | cir::StoreOp::create(builder, location: call.getLoc(), value: arg, addr: slot); |
| 2397 | } |
| 2398 | newArgs.push_back(Elt: slot); |
| 2399 | } else { |
| 2400 | newArgs.push_back(Elt: arg); |
| 2401 | } |
| 2402 | } |
| 2403 | argBegin.push_back(Elt: newArgs.size()); |
| 2404 | builder.setInsertionPoint(call); |
| 2405 | for (auto [idx, freeze] : refreezes) |
| 2406 | for (unsigned i = argBegin[idx]; i != argBegin[idx + 1]; ++i) |
| 2407 | newArgs[i] = cir::FreezeOp::create(builder, location: freeze.getLoc(), input: newArgs[i]); |
| 2408 | unsigned numEllipsisArgs = |
| 2409 | ellipsisStart ? newArgs.size() - *ellipsisStart : 0; |
| 2410 | |
| 2411 | bool hasResult = call.getNumResults() > 0; |
| 2412 | mlir::Type origRetTy = |
| 2413 | hasResult ? call.getResult().getType() : cir::VoidType::get(ctx); |
| 2414 | |
| 2415 | // An indirect (sret) return has a different call shape than the coerce / |
| 2416 | // extend / ignore return handling further down (the value is returned |
| 2417 | // through a prepended pointer slot, not as a result), so dispatch to a |
| 2418 | // dedicated helper for it; everything below handles the by-value returns. |
| 2419 | if (fc.returnInfo.kind == ArgKind::Indirect && hasResult) { |
| 2420 | if (failed(Result: rewriteIndirectReturnCall(call, fc, newArgs, numEllipsisArgs, |
| 2421 | origRetTy, origCallArgTypes, builder, |
| 2422 | dl, enableNoundefAttrs))) |
| 2423 | return mlir::failure(); |
| 2424 | eraseDeadFreezes(freezes: deadFreezes); |
| 2425 | eraseDeadRecordLoads(loads: deadRecordLoads); |
| 2426 | return mlir::success(); |
| 2427 | } |
| 2428 | |
| 2429 | mlir::Type callRetTy = origRetTy; |
| 2430 | if (fc.returnInfo.kind == ArgKind::Ignore && hasResult) |
| 2431 | callRetTy = cir::VoidType::get(ctx); |
| 2432 | bool returnNeedsCoercion = |
| 2433 | hasResult && fc.returnInfo.kind == ArgKind::Direct && |
| 2434 | fc.returnInfo.coercedType && fc.returnInfo.coercedType != origRetTy; |
| 2435 | if (returnNeedsCoercion) |
| 2436 | callRetTy = fc.returnInfo.coercedType; |
| 2437 | |
| 2438 | builder.setInsertionPoint(call); |
| 2439 | prependIndirectCallee(call, args&: newArgs, retTy: callRetTy, numEllipsisArgs, builder); |
| 2440 | auto newCall = cir::CallOp::create(builder, location: call.getLoc(), |
| 2441 | callee: call.getCalleeAttr(), resType: callRetTy, operands: newArgs); |
| 2442 | copyCallAttributes(source: call, target: newCall); |
| 2443 | |
| 2444 | // Direct return with coercion: the new call returns the coerced type, which |
| 2445 | // is stored into a coercion slot where the old call's uses find the original |
| 2446 | // type. |
| 2447 | if (returnNeedsCoercion) { |
| 2448 | builder.setInsertionPointAfter(newCall); |
| 2449 | SmallPtrSet<mlir::Operation *, 4> coercionOps; |
| 2450 | if (failed(Result: emitCoercionAndReplaceUses(builder, loc: call.getLoc(), origTy: origRetTy, |
| 2451 | src: newCall.getResult(), val: call.getResult(), |
| 2452 | slotBlock, dl, createdOps&: coercionOps, |
| 2453 | offset: fc.returnInfo.directOffset))) |
| 2454 | return mlir::failure(); |
| 2455 | } |
| 2456 | |
| 2457 | if (needsArgAttrUpdate(fc)) { |
| 2458 | auto existing = call->getAttrOfType<mlir::ArrayAttr>(name: "arg_attrs" ); |
| 2459 | newCall->setAttr(name: "arg_attrs" , |
| 2460 | value: updateArgAttrs(ctx, module, origArgTypes: origCallArgTypes, existingArgAttrs: existing, fc, |
| 2461 | dl, enableNoundefAttrs, |
| 2462 | /*isCallSite=*/true)); |
| 2463 | } |
| 2464 | // Add llvm.signext / llvm.zeroext to an Extend return, and drop llvm.noundef |
| 2465 | // from a coerced Direct return that passes undefined bits. |
| 2466 | if (fc.returnInfo.kind == ArgKind::Extend || |
| 2467 | (hasResult && fc.returnInfo.kind == ArgKind::Direct)) { |
| 2468 | auto existing = call->getAttrOfType<mlir::ArrayAttr>(name: "res_attrs" ); |
| 2469 | if (mlir::ArrayAttr updated = |
| 2470 | updateResAttrs(ctx, existingResAttrs: existing, retInfo: fc.returnInfo, origRetTy, dl)) |
| 2471 | newCall->setAttr(name: "res_attrs" , value: updated); |
| 2472 | } else if (hasResult && mlir::isa<cir::VoidType>(Val: callRetTy)) { |
| 2473 | newCall->removeAttr(name: "res_attrs" ); |
| 2474 | } |
| 2475 | |
| 2476 | if (hasResult && fc.returnInfo.kind == ArgKind::Ignore) { |
| 2477 | // The new call returns void, but the original call's result may still |
| 2478 | // have uses. Substitute a poison constant of the original type so |
| 2479 | // those uses remain well-formed without pretending we have a real |
| 2480 | // value at the ABI boundary. |
| 2481 | if (!call.getResult().use_empty()) { |
| 2482 | builder.setInsertionPointAfter(newCall); |
| 2483 | mlir::Value poison = |
| 2484 | createIgnoredValue(builder, loc: call.getLoc(), ty: origRetTy); |
| 2485 | call.getResult().replaceAllUsesWith(newValue: poison); |
| 2486 | } |
| 2487 | } else if (hasResult && !returnNeedsCoercion) { |
| 2488 | // returnNeedsCoercion already wired up the coerced result above. |
| 2489 | call.getResult().replaceAllUsesWith(newValue: newCall.getResult()); |
| 2490 | } |
| 2491 | |
| 2492 | call->erase(); |
| 2493 | eraseDeadFreezes(freezes: deadFreezes); |
| 2494 | eraseDeadRecordLoads(loads: deadRecordLoads); |
| 2495 | |
| 2496 | return mlir::success(); |
| 2497 | } |
| 2498 | |
| 2499 | void CIRABIRewriteContext::rewriteFunctionAddress(cir::GetGlobalOp addrOp, |
| 2500 | cir::FuncOp funcOp, |
| 2501 | mlir::OpBuilder &builder) { |
| 2502 | auto oldPtrTy = mlir::cast<cir::PointerType>(Val: addrOp.getAddr().getType()); |
| 2503 | cir::FuncType newFuncTy = funcOp.getFunctionType(); |
| 2504 | // An extension rides on an argument attribute and leaves the signature |
| 2505 | // alone, so such a callee still matches the written type. |
| 2506 | if (newFuncTy == oldPtrTy.getPointee()) |
| 2507 | return; |
| 2508 | |
| 2509 | // The verifier requires the retype even when nothing reads the address. |
| 2510 | addrOp.getAddr().setType(cir::PointerType::get(pointee: newFuncTy)); |
| 2511 | if (addrOp.getAddr().use_empty()) |
| 2512 | return; |
| 2513 | |
| 2514 | // A later indirect call through the written type is classified and coerced |
| 2515 | // on its own, so casting back leaves it to that rewrite. |
| 2516 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 2517 | builder.setInsertionPointAfter(addrOp); |
| 2518 | auto bitcast = cir::CastOp::create(builder, location: addrOp.getLoc(), result: oldPtrTy, |
| 2519 | kind: cir::CastKind::bitcast, src: addrOp.getAddr()); |
| 2520 | addrOp.getAddr().replaceAllUsesExcept(newValue: bitcast.getResult(), exceptedUser: bitcast); |
| 2521 | } |
| 2522 | |
| 2523 | namespace { |
| 2524 | |
| 2525 | /// What one `va_arg` expansion needs from the op it rewrites. The x86-64 |
| 2526 | /// cursor fields are reached through `vaFields` by index: 0 gp_offset, |
| 2527 | /// 1 fp_offset, 2 overflow_arg_area, 3 reg_save_area. |
| 2528 | struct VAArgFetch { |
| 2529 | VAArgFetch(mlir::Location loc, mlir::Value valist, |
| 2530 | llvm::ArrayRef<mlir::Type> vaFields, mlir::Type resultTy, |
| 2531 | const ArgClassification &ac, const mlir::DataLayout &dl, |
| 2532 | mlir::ModuleOp module) |
| 2533 | : loc(loc), valist(valist), vaFields(vaFields), resultTy(resultTy), |
| 2534 | ac(ac), dl(dl), module(module) { |
| 2535 | assert(vaFields.size() == 4 && |
| 2536 | "the x86-64 va_list is a four-field cursor, checked by the caller" ); |
| 2537 | } |
| 2538 | |
| 2539 | mlir::Location loc; |
| 2540 | mlir::Value valist; |
| 2541 | llvm::ArrayRef<mlir::Type> vaFields; |
| 2542 | mlir::Type resultTy; |
| 2543 | const ArgClassification ∾ |
| 2544 | const mlir::DataLayout &dl; |
| 2545 | mlir::ModuleOp module; |
| 2546 | }; |
| 2547 | |
| 2548 | /// The cursor fields a register fetch reads, and the predicate saying every |
| 2549 | /// class it needs still has room. An offset is null when the fetch needs no |
| 2550 | /// register of that class. |
| 2551 | struct RegisterCursor { |
| 2552 | mlir::Value gpOffsetP; |
| 2553 | mlir::Value fpOffsetP; |
| 2554 | mlir::Value gpOffset; |
| 2555 | mlir::Value fpOffset; |
| 2556 | mlir::Value inRegs; |
| 2557 | }; |
| 2558 | |
| 2559 | mlir::LogicalResult reportVAArgNYI(cir::VAArgOp op, llvm::StringRef what) { |
| 2560 | op->emitOpError() << "va_arg of " << what |
| 2561 | << " not yet implemented in CallConvLowering" ; |
| 2562 | return mlir::failure(); |
| 2563 | } |
| 2564 | |
| 2565 | /// Sets \p isRegPair when a two-register argument is a coerced pair, one |
| 2566 | /// register per member, rather than a single wide scalar, and records in |
| 2567 | /// \p pairIsSse which element is SSE class. Fails on a coercion that is not |
| 2568 | /// two eightbytes. |
| 2569 | mlir::LogicalResult classifyRegisterPair(cir::VAArgOp op, |
| 2570 | const ArgClassification &ac, |
| 2571 | std::array<bool, 2> &pairIsSse, |
| 2572 | bool &isRegPair) { |
| 2573 | auto pairTy = mlir::dyn_cast<cir::RecordType>(Val: ac.coercedType); |
| 2574 | if (!pairTy) |
| 2575 | return mlir::success(); |
| 2576 | |
| 2577 | if (pairTy.getNumElements() != 2) |
| 2578 | return reportVAArgNYI(op, what: "a register coercion that is not two eightbytes" ); |
| 2579 | |
| 2580 | assert(!ac.directOffset && |
| 2581 | "a pair already spans both eightbytes, so it cannot also start " |
| 2582 | "partway into the value" ); |
| 2583 | for (auto [i, memberTy] : llvm::enumerate(First: pairTy.getMembers())) |
| 2584 | pairIsSse[i] = isSSERegisterClass(ty: memberTy); |
| 2585 | isRegPair = true; |
| 2586 | return mlir::success(); |
| 2587 | } |
| 2588 | |
| 2589 | /// The alignment an argument is placed at, which is the language alignment of |
| 2590 | /// its type. |
| 2591 | uint64_t argumentAreaAlign(mlir::Type ty, mlir::ModuleOp modOp) { |
| 2592 | return cir::CIRDataLayout(modOp).getTypeLanguageAlignment(ty); |
| 2593 | } |
| 2594 | |
| 2595 | mlir::Value roundPointerUpToAlignment(CIRBaseBuilderTy &b, mlir::Location loc, |
| 2596 | mlir::Value bytePtr, uint64_t align, |
| 2597 | const mlir::DataLayout &dl) { |
| 2598 | assert(mlir::cast<cir::PointerType>(bytePtr.getType()).getPointee() == |
| 2599 | b.getUIntNTy(8) && |
| 2600 | "the bump strides in bytes, so the pointee must be u8" ); |
| 2601 | assert(llvm::isPowerOf2_64(align) && |
| 2602 | "mask rounding needs a power-of-two alignment" ); |
| 2603 | mlir::Value bumped = |
| 2604 | b.createPtrStride(loc, base: bytePtr, stride: b.getSignedInt(loc, val: align - 1, numBits: 32)); |
| 2605 | std::optional<uint64_t> indexWidth = |
| 2606 | dl.getTypeIndexBitwidth(t: bytePtr.getType()); |
| 2607 | assert(indexWidth && "a pointer in the argument area has an index width" ); |
| 2608 | mlir::Value mask = b.getSignedInt(loc, val: -static_cast<int64_t>(align), |
| 2609 | numBits: static_cast<unsigned>(*indexWidth)); |
| 2610 | return cir::PtrMaskOp::create(builder&: b, location: loc, result: bytePtr.getType(), ptr: bumped, mask); |
| 2611 | } |
| 2612 | |
| 2613 | /// Reads the argument's address out of the overflow area, which also advances |
| 2614 | /// the cursor past the argument. |
| 2615 | mlir::Value buildOverflowAddrAndAdvance(CIRBaseBuilderTy &b, |
| 2616 | const VAArgFetch &f) { |
| 2617 | cir::IntType byteTy = b.getUIntNTy(n: 8); |
| 2618 | mlir::Value overflowP = b.createGetMember( |
| 2619 | loc: f.loc, resultTy: b.getPointerTo(ty: f.vaFields[2]), base: f.valist, name: "overflow_arg_area" , index: 2); |
| 2620 | mlir::Value overflow = b.createLoad(loc: f.loc, ptr: overflowP); |
| 2621 | mlir::Value bytePtr = b.createPtrBitcast(src: overflow, newPointeeTy: byteTy); |
| 2622 | |
| 2623 | uint64_t tyAlign = argumentAreaAlign(ty: f.resultTy, modOp: f.module); |
| 2624 | if (tyAlign > 8) |
| 2625 | bytePtr = roundPointerUpToAlignment(b, loc: f.loc, bytePtr, align: tyAlign, dl: f.dl); |
| 2626 | |
| 2627 | uint64_t stride = |
| 2628 | llvm::alignTo(Value: f.dl.getTypeAllocSize(t: f.resultTy).getFixedValue(), Align: 8); |
| 2629 | mlir::Value strideVal = b.getSignedInt(loc: f.loc, val: stride, numBits: 32); |
| 2630 | mlir::Value next = b.createPtrStride(loc: f.loc, base: bytePtr, stride: strideVal); |
| 2631 | b.createStore(loc: f.loc, val: next, dst: overflowP); |
| 2632 | return bytePtr; |
| 2633 | } |
| 2634 | |
| 2635 | /// Loads the cursor offsets and builds the predicate that sends the fetch to |
| 2636 | /// the register-save area. Both offsets count from the start of that area, so |
| 2637 | /// the integer limit is the six GP registers at 48 and the vector limit is |
| 2638 | /// those plus the eight SSE registers at 176. |
| 2639 | RegisterCursor buildRegisterGate(CIRBaseBuilderTy &b, const VAArgFetch &f, |
| 2640 | unsigned neededInt, unsigned neededSse) { |
| 2641 | RegisterCursor cursor; |
| 2642 | if (neededInt) { |
| 2643 | cursor.gpOffsetP = b.createGetMember(loc: f.loc, resultTy: b.getPointerTo(ty: f.vaFields[0]), |
| 2644 | base: f.valist, name: "gp_offset" , index: 0); |
| 2645 | cursor.gpOffset = b.createLoad(loc: f.loc, ptr: cursor.gpOffsetP); |
| 2646 | mlir::Value limit = |
| 2647 | b.getConstantInt(loc: f.loc, ty: cursor.gpOffset.getType(), value: 48 - neededInt * 8); |
| 2648 | cursor.inRegs = |
| 2649 | b.createCompare(loc: f.loc, kind: cir::CmpOpKind::le, lhs: cursor.gpOffset, rhs: limit); |
| 2650 | } |
| 2651 | if (neededSse) { |
| 2652 | cursor.fpOffsetP = b.createGetMember(loc: f.loc, resultTy: b.getPointerTo(ty: f.vaFields[1]), |
| 2653 | base: f.valist, name: "fp_offset" , index: 1); |
| 2654 | cursor.fpOffset = b.createLoad(loc: f.loc, ptr: cursor.fpOffsetP); |
| 2655 | mlir::Value limit = b.getConstantInt(loc: f.loc, ty: cursor.fpOffset.getType(), |
| 2656 | value: 176 - neededSse * 16); |
| 2657 | mlir::Value fitsInFp = |
| 2658 | b.createCompare(loc: f.loc, kind: cir::CmpOpKind::le, lhs: cursor.fpOffset, rhs: limit); |
| 2659 | cursor.inRegs = cursor.inRegs |
| 2660 | ? b.createLogicalAnd(loc: f.loc, lhs: cursor.inRegs, rhs: fitsInFp) |
| 2661 | : fitsInFp; |
| 2662 | } |
| 2663 | return cursor; |
| 2664 | } |
| 2665 | |
| 2666 | /// Copies each half of a non-contiguous pair out of the register-save area |
| 2667 | /// into \p regPairTemp, laid out as the coerced pair. |
| 2668 | void reassembleRegisterPair(CIRBaseBuilderTy &b, mlir::Location loc, |
| 2669 | const ArgClassification &ac, |
| 2670 | const RegisterCursor &cursor, |
| 2671 | const std::array<bool, 2> &pairIsSse, |
| 2672 | mlir::Value regSaveArea, mlir::Value regPairTemp) { |
| 2673 | auto pairTy = mlir::cast<cir::RecordType>(Val: ac.coercedType); |
| 2674 | // Both halves of an all-SSE pair sit in 16-byte slots, which classic |
| 2675 | // CodeGen tells the load about. It leaves a mixed pair to the element's |
| 2676 | // own alignment, so match that rather than claiming the slot there. |
| 2677 | bool bothSse = pairIsSse[0] && pairIsSse[1]; |
| 2678 | // Track how many of each class came before, since a slot is reached from |
| 2679 | // its class's own cursor. |
| 2680 | unsigned seenOfClass[2] = {0, 0}; |
| 2681 | for (unsigned i = 0; i < 2; ++i) { |
| 2682 | bool isSse = pairIsSse[i]; |
| 2683 | mlir::Value base = isSse ? cursor.fpOffset : cursor.gpOffset; |
| 2684 | unsigned regSize = isSse ? 16 : 8; |
| 2685 | unsigned prior = seenOfClass[isSse]; |
| 2686 | ++seenOfClass[isSse]; |
| 2687 | mlir::Value off = base; |
| 2688 | if (prior) { |
| 2689 | off = b.createAdd(loc, lhs: base, |
| 2690 | rhs: b.getConstantInt(loc, ty: base.getType(), value: prior * regSize)); |
| 2691 | } |
| 2692 | mlir::Value src = b.createPtrStride(loc, base: regSaveArea, stride: off); |
| 2693 | mlir::Type elemTy = pairTy.getElementType(idx: i); |
| 2694 | mlir::Value elemPtr = b.createPtrBitcast(src, newPointeeTy: elemTy); |
| 2695 | mlir::Value val = bothSse ? b.createAlignedLoad(loc, ptr: elemPtr, alignment: 16) |
| 2696 | : b.createLoad(loc, ptr: elemPtr); |
| 2697 | b.createStore( |
| 2698 | loc, val, |
| 2699 | dst: b.createGetMember(loc, resultTy: b.getPointerTo(ty: elemTy), base: regPairTemp, name: "" , index: i)); |
| 2700 | } |
| 2701 | } |
| 2702 | |
| 2703 | /// The bytes carried by the registers of this fetch's one class. |
| 2704 | uint64_t registerSlotSize(unsigned neededInt, unsigned neededSse) { |
| 2705 | assert(!(neededInt && neededSse) && |
| 2706 | "a fetch needing both classes is a pair, reassembled elsewhere" ); |
| 2707 | return neededSse ? neededSse * 16 : neededInt * 8; |
| 2708 | } |
| 2709 | |
| 2710 | /// Whether the register cannot be read in place, either because it carries |
| 2711 | /// less than the whole result or because its slot is under-aligned for it. |
| 2712 | bool needsTempCopy(const VAArgFetch &f, unsigned neededInt, |
| 2713 | unsigned neededSse) { |
| 2714 | uint64_t tySize = f.dl.getTypeSize(t: f.resultTy).getFixedValue(); |
| 2715 | if (f.ac.coercedType && |
| 2716 | (f.ac.directOffset || registerSlotSize(neededInt, neededSse) < tySize)) |
| 2717 | return true; |
| 2718 | // A slot is only as aligned as its class, 8 for a GP register and 16 for an |
| 2719 | // SSE one, so a fetch the ABI places more strictly is copied through a temp. |
| 2720 | uint64_t slotAlign = neededSse ? 16 : 8; |
| 2721 | return argumentAreaAlign(ty: f.resultTy, modOp: f.module) > slotAlign; |
| 2722 | } |
| 2723 | |
| 2724 | /// Copies what the registers carry into \p temp, which is the size of the |
| 2725 | /// whole result, and returns the address to read the result from. |
| 2726 | mlir::Value copyRegisterToTemp(CIRBaseBuilderTy &b, mlir::Location loc, |
| 2727 | const VAArgFetch &f, mlir::Value regAddr, |
| 2728 | mlir::Value temp, unsigned neededInt, |
| 2729 | unsigned neededSse) { |
| 2730 | cir::IntType byteTy = b.getUIntNTy(n: 8); |
| 2731 | uint64_t tySize = f.dl.getTypeSize(t: f.resultTy).getFixedValue(); |
| 2732 | |
| 2733 | if (f.ac.coercedType && |
| 2734 | (f.ac.directOffset || registerSlotSize(neededInt, neededSse) < tySize)) { |
| 2735 | // The registers carry less than the whole result, either because the |
| 2736 | // eightbytes below directOffset hold no field or because the result is |
| 2737 | // wider than the registers carrying it. Copy only what they carry, so |
| 2738 | // that reading the temp cannot run on into the neighboring slot. |
| 2739 | mlir::Value val = b.createAlignedLoad( |
| 2740 | loc, ptr: b.createPtrBitcast(src: regAddr, newPointeeTy: f.ac.coercedType), alignment: 8); |
| 2741 | mlir::Value dst = temp; |
| 2742 | if (f.ac.directOffset) { |
| 2743 | mlir::Value tempByte = b.createPtrBitcast(src: temp, newPointeeTy: byteTy); |
| 2744 | mlir::Value offset = b.getSignedInt(loc, val: f.ac.directOffset, numBits: 32); |
| 2745 | dst = b.createPtrStride(loc, base: tempByte, stride: offset); |
| 2746 | } |
| 2747 | b.createStore(loc, val, dst: b.createPtrBitcast(src: dst, newPointeeTy: f.ac.coercedType)); |
| 2748 | return b.createPtrBitcast(src: temp, newPointeeTy: byteTy); |
| 2749 | } |
| 2750 | |
| 2751 | mlir::Value regView = b.createPtrBitcast(src: regAddr, newPointeeTy: f.resultTy); |
| 2752 | if (mlir::isa<cir::RecordType>(Val: f.resultTy)) |
| 2753 | cir::CopyOp::create( |
| 2754 | builder&: b, location: loc, dst: temp, src: regView, |
| 2755 | dst_alignment: b.getI64IntegerAttr(value: argumentAreaAlign(ty: f.resultTy, modOp: f.module)), |
| 2756 | src_alignment: b.getI64IntegerAttr(value: 8)); |
| 2757 | else |
| 2758 | b.createStore(loc, val: b.createAlignedLoad(loc, ptr: regView, alignment: 8), dst: temp); |
| 2759 | return b.createPtrBitcast(src: temp, newPointeeTy: byteTy); |
| 2760 | } |
| 2761 | |
| 2762 | void advanceRegisterCursors(CIRBaseBuilderTy &b, mlir::Location loc, |
| 2763 | const RegisterCursor &cursor, unsigned neededInt, |
| 2764 | unsigned neededSse) { |
| 2765 | if (neededInt) { |
| 2766 | b.createStore(loc, |
| 2767 | val: b.createAdd(loc, lhs: cursor.gpOffset, |
| 2768 | rhs: b.getConstantInt(loc, ty: cursor.gpOffset.getType(), |
| 2769 | value: neededInt * 8)), |
| 2770 | dst: cursor.gpOffsetP); |
| 2771 | } |
| 2772 | if (neededSse) { |
| 2773 | b.createStore(loc, |
| 2774 | val: b.createAdd(loc, lhs: cursor.fpOffset, |
| 2775 | rhs: b.getConstantInt(loc, ty: cursor.fpOffset.getType(), |
| 2776 | value: neededSse * 16)), |
| 2777 | dst: cursor.fpOffsetP); |
| 2778 | } |
| 2779 | } |
| 2780 | |
| 2781 | } // namespace |
| 2782 | |
| 2783 | mlir::LogicalResult |
| 2784 | CIRABIRewriteContext::rewriteVAArg(mlir::Operation *vaArgOp, |
| 2785 | const ArgClassification &ac, |
| 2786 | mlir::OpBuilder &opBuilder) { |
| 2787 | auto op = mlir::cast<cir::VAArgOp>(Val: vaArgOp); |
| 2788 | CIRBaseBuilderTy builder(opBuilder); |
| 2789 | mlir::Location loc = op.getLoc(); |
| 2790 | mlir::Type resultTy = op.getType(); |
| 2791 | mlir::Value valist = op.getArgList(); |
| 2792 | |
| 2793 | // An ignored type travels in no register and no stack slot, so the fetch |
| 2794 | // reads nothing and must leave the cursor unchanged. The value holds no |
| 2795 | // bytes, so poison stands in for it. |
| 2796 | if (ac.kind == ArgKind::Ignore) { |
| 2797 | builder.setInsertionPoint(op); |
| 2798 | op.getResult().replaceAllUsesWith( |
| 2799 | newValue: createIgnoredValue(builder, loc, ty: resultTy)); |
| 2800 | op->erase(); |
| 2801 | return mlir::success(); |
| 2802 | } |
| 2803 | |
| 2804 | if (ac.kind == ArgKind::Indirect && !ac.byVal) |
| 2805 | return reportVAArgNYI(op, what: "a non-trivially-copyable type" ); |
| 2806 | if (ac.kind == ArgKind::IndirectAliased) |
| 2807 | return reportVAArgNYI(op, what: "a type passed byref" ); |
| 2808 | |
| 2809 | // neededInt counts 8-byte integer slots and neededSse counts 16-byte vector |
| 2810 | // slots. Zero of both means the type travels in memory and is read |
| 2811 | // straight from the overflow area. |
| 2812 | unsigned neededInt = ac.neededIntRegs; |
| 2813 | unsigned neededSse = ac.neededSseRegs; |
| 2814 | |
| 2815 | // Which coerced-pair element (0 = low eightbyte, 1 = high) is SSE rather |
| 2816 | // than INTEGER class. Only meaningful when isRegPair is set. |
| 2817 | std::array<bool, 2> pairIsSse = {false, false}; |
| 2818 | bool isRegPair = false; |
| 2819 | if (ac.kind == ArgKind::Direct && neededInt + neededSse == 2 && |
| 2820 | ac.coercedType) { |
| 2821 | if (classifyRegisterPair(op, ac, pairIsSse, isRegPair).failed()) |
| 2822 | return mlir::failure(); |
| 2823 | } |
| 2824 | |
| 2825 | auto vaListRecTy = mlir::dyn_cast<cir::RecordType>( |
| 2826 | Val: mlir::cast<cir::PointerType>(Val: valist.getType()).getPointee()); |
| 2827 | if (!vaListRecTy || vaListRecTy.getNumElements() != 4) { |
| 2828 | return reportVAArgNYI(op, |
| 2829 | what: "a va_list that is not the four-field gp_offset / " |
| 2830 | "fp_offset / overflow_arg_area / reg_save_area " |
| 2831 | "cursor" ); |
| 2832 | } |
| 2833 | |
| 2834 | const VAArgFetch fetch{loc, valist, vaListRecTy.getMembers(), resultTy, ac, |
| 2835 | dl, module}; |
| 2836 | cir::IntType byteTy = builder.getUIntNTy(n: 8); |
| 2837 | |
| 2838 | builder.setInsertionPoint(op); |
| 2839 | |
| 2840 | mlir::Value addr; |
| 2841 | if (neededInt == 0 && neededSse == 0) { |
| 2842 | addr = buildOverflowAddrAndAdvance(b&: builder, f: fetch); |
| 2843 | } else { |
| 2844 | RegisterCursor cursor = |
| 2845 | buildRegisterGate(b&: builder, f: fetch, neededInt, neededSse); |
| 2846 | |
| 2847 | // A two-eightbyte pair that is purely INTEGER class is contiguous in the |
| 2848 | // register-save area, since GP slots are 8-byte packed, so the address of |
| 2849 | // its low eightbyte is already the address of the whole value. A pure |
| 2850 | // SSE pair or a mixed pair is not contiguous, since SSE slots are 16-byte |
| 2851 | // spaced and a mixed pair's halves live in disjoint areas, so each half is |
| 2852 | // copied into a temp laid out as the coerced pair. |
| 2853 | bool pairNeedsReassembly = isRegPair && neededSse != 0; |
| 2854 | |
| 2855 | // Both temps are allocated here, since a ternary arm yields their address. |
| 2856 | mlir::Value regPairTemp; |
| 2857 | if (pairNeedsReassembly) { |
| 2858 | // The temp is written one member at a time through the coerced pair, so |
| 2859 | // it has to meet that type's alignment, and read back as the result, so |
| 2860 | // it has to meet the result's alignment too. |
| 2861 | regPairTemp = builder.createAlloca( |
| 2862 | loc, addrType: builder.getPointerTo(ty: ac.coercedType), name: "vaarg.reg" , |
| 2863 | alignment: clang::CharUnits::fromQuantity( |
| 2864 | Quantity: std::max(a: cirDataLayout.getTypeLanguageAlignment(ty: ac.coercedType), |
| 2865 | b: argumentAreaAlign(ty: resultTy, modOp: module)))); |
| 2866 | } |
| 2867 | |
| 2868 | mlir::Value regTemp; |
| 2869 | bool copyThroughTemp = |
| 2870 | !pairNeedsReassembly && needsTempCopy(f: fetch, neededInt, neededSse); |
| 2871 | if (copyThroughTemp) { |
| 2872 | regTemp = builder.createAlloca( |
| 2873 | loc, addrType: builder.getPointerTo(ty: resultTy), name: "vaarg.reg" , |
| 2874 | alignment: clang::CharUnits::fromQuantity(Quantity: argumentAreaAlign(ty: resultTy, modOp: module))); |
| 2875 | } |
| 2876 | |
| 2877 | addr = cir::TernaryOp::create( |
| 2878 | builder, location: loc, cond: cursor.inRegs, |
| 2879 | /*trueBuilder=*/ |
| 2880 | [&](mlir::OpBuilder &ob, mlir::Location l) { |
| 2881 | CIRBaseBuilderTy b(ob); |
| 2882 | mlir::Value regSaveArea = b.createLoad( |
| 2883 | loc: l, ptr: b.createGetMember(loc: l, resultTy: b.getPointerTo(ty: fetch.vaFields[3]), |
| 2884 | base: valist, name: "reg_save_area" , index: 3)); |
| 2885 | regSaveArea = b.createPtrBitcast(src: regSaveArea, newPointeeTy: byteTy); |
| 2886 | |
| 2887 | mlir::Value regAddr; |
| 2888 | if (pairNeedsReassembly) { |
| 2889 | reassembleRegisterPair(b, loc: l, ac, cursor, pairIsSse, |
| 2890 | regSaveArea, regPairTemp); |
| 2891 | regAddr = b.createPtrBitcast(src: regPairTemp, newPointeeTy: byteTy); |
| 2892 | } else { |
| 2893 | mlir::Value off = |
| 2894 | neededSse ? cursor.fpOffset : cursor.gpOffset; |
| 2895 | regAddr = b.createPtrStride(loc: l, base: regSaveArea, stride: off); |
| 2896 | if (copyThroughTemp) { |
| 2897 | regAddr = copyRegisterToTemp(b, loc: l, f: fetch, regAddr, temp: regTemp, |
| 2898 | neededInt, neededSse); |
| 2899 | } |
| 2900 | } |
| 2901 | |
| 2902 | advanceRegisterCursors(b, loc: l, cursor, neededInt, neededSse); |
| 2903 | cir::YieldOp::create(builder&: b, location: l, args: regAddr); |
| 2904 | }, |
| 2905 | /*falseBuilder=*/ |
| 2906 | [&](mlir::OpBuilder &ob, mlir::Location l) { |
| 2907 | CIRBaseBuilderTy b(ob); |
| 2908 | mlir::Value memAddr = buildOverflowAddrAndAdvance(b, f: fetch); |
| 2909 | cir::YieldOp::create(builder&: b, location: l, args: memAddr); |
| 2910 | }) |
| 2911 | .getResult(); |
| 2912 | } |
| 2913 | |
| 2914 | // Every path above places the result at least this well aligned. |
| 2915 | mlir::Value resultAddr = builder.createPtrBitcast(src: addr, newPointeeTy: resultTy); |
| 2916 | uint64_t resultAlign = argumentAreaAlign(ty: resultTy, modOp: module); |
| 2917 | if (mlir::isa<cir::RecordType>(Val: resultTy)) { |
| 2918 | SmallPtrSet<mlir::Operation *, 1> noExceptions; |
| 2919 | if (failed(Result: rewireRecordUses(builder, loc, val: op.getResult(), addr: resultAddr, |
| 2920 | addrAlign: resultAlign, dl, exceptions: noExceptions))) |
| 2921 | return mlir::failure(); |
| 2922 | } else { |
| 2923 | op.getResult().replaceAllUsesWith( |
| 2924 | newValue: builder.createAlignedLoad(loc, ptr: resultAddr, alignment: resultAlign)); |
| 2925 | } |
| 2926 | op->erase(); |
| 2927 | return mlir::success(); |
| 2928 | } |
| 2929 | |