1//===- CIRABIRewriteContext.cpp - CIR ABI rewrite context ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "CIRABIRewriteContext.h"
10#include "mlir/Dialect/LLVMIR/LLVMDialectDecl.h"
11#include "mlir/IR/Builders.h"
12#include "mlir/IR/Dominance.h"
13#include "clang/CIR/Dialect/Builder/CIRBaseBuilder.h"
14#include "clang/CIR/Dialect/IR/CIRAttrs.h"
15#include "clang/CIR/Dialect/IR/CIRDataLayout.h"
16#include "clang/CIR/Dialect/IR/CIRDialect.h"
17#include "clang/CIR/Dialect/IR/CIRTypes.h"
18#include "clang/CIR/MissingFeatures.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/IR/Attributes.h"
21#include "llvm/Support/MathExtras.h"
22#include <algorithm>
23#include <array>
24#include <utility>
25
26using namespace cir;
27using namespace mlir;
28using namespace mlir::abi;
29
30// This rewrite context supports the Direct (with or without coercion),
31// Extend, Ignore, Indirect-return (sret), Indirect-argument (byval and
32// non-byval), IndirectAliased-argument (byref), and Expand (struct
33// flattening) classifications.
34//
35// An Indirect argument is byval or not, following its classification's
36// byVal flag. byval is a by-value parameter the ABI passes in memory rather
37// than registers, usually for its size. Non-byval is a by-value parameter
38// whose type cannot be copied freely, because it has a non-trivial copy
39// constructor, move constructor, or destructor, so the callee works on the
40// caller's own object rather than a copy.
41//
42// At the call site both forward the caller's storage, since byval is copied
43// on the caller's side of the call. byval falls back to a slot of its own
44// when the operand's storage cannot be handed on. At the callee, byval fills
45// the CIRGen param slot with a copy of the incoming object, while non-byval
46// rewires that slot to the incoming pointer so the body mutates the caller's
47// storage in place.
48//
49// A byref argument is lowered like byval, apart from its attribute and the
50// address space of its pointer. The callee must not write through that
51// pointer, so it fills the CIRGen param slot with a copy just as byval does.
52//
53// For Expand, the single struct argument is replaced by N scalar arguments
54// (one per field). At the callee, the N field block arguments are stored
55// directly into the parameter's own alloca (the CIRGen spill slot). At the
56// call site, the struct operand is decomposed into its fields by reading
57// each member from the source alloca (get_member + load) when the operand is
58// a load of an alloca, or via cir.extract_member otherwise.
59//
60// For Direct + canFlatten (where the coerced type is a multi-field struct), the
61// coerced struct is similarly passed as N separate arguments, one per field.
62// The callee stores the N scalar block args into one slot sized and aligned for
63// both the coerced struct and the original argument type, which the body reads.
64// The call site reads each field of the coerced struct out of memory holding
65// the operand as a separate call argument.
66
67namespace {
68
69/// A pointer to \p pointee in target address space \p addrSpace. Zero is the
70/// default address space, which carries no attribute.
71cir::PointerType getPtrTyInAddrSpace(mlir::Type pointee, unsigned addrSpace) {
72 mlir::ptr::MemorySpaceAttrInterface asAttr;
73 if (addrSpace)
74 asAttr = cir::TargetAddressSpaceAttr::get(context: pointee.getContext(), value: addrSpace);
75 return cir::PointerType::get(pointee, addrSpace: asAttr);
76}
77
78/// The pointer passed in place of an Indirect or byref value, or an sret
79/// return, of type \p pointee.
80cir::PointerType getIndirectPtrTy(mlir::Type pointee,
81 const ArgClassification &ac) {
82 assert(
83 (ac.kind == ArgKind::Indirect || ac.kind == ArgKind::IndirectAliased) &&
84 "expected an indirect classification");
85 return getPtrTyInAddrSpace(pointee, addrSpace: ac.indirectAddrSpace);
86}
87
88/// The struct a union is flattened as: its storage member, then its tail
89/// padding if it has any. This matches the union's LLVM type.
90cir::StructType getUnionFlattenType(cir::UnionType unionTy,
91 const mlir::DataLayout &dl) {
92 SmallVector<mlir::Type, 2> fields;
93 if (mlir::Type storage = unionTy.getUnionStorageType(dataLayout: dl))
94 fields.push_back(Elt: storage);
95 if (mlir::Type padding = unionTy.getPadding())
96 fields.push_back(Elt: padding);
97 return cir::StructType::get(context: unionTy.getContext(), members: fields, packed: unionTy.getPacked(),
98 /*is_class=*/false,
99 member_kinds: cir::RecordType::getAllDataKinds(members: fields));
100}
101
102/// Return the coerced RecordType for a Direct classification that should be
103/// flattened into individual scalar arguments, or a null type if the
104/// classification does not call for flattening.
105///
106/// Flattening applies when all four conditions hold:
107/// 1. The classification is Direct with a non-null coercedType.
108/// 2. canFlatten is set.
109/// 3. The coercedType is a struct, or a union, which is flattened as the
110/// struct getUnionFlattenType returns.
111/// 4. The struct has at least one field. A struct of one field is passed as
112/// that field.
113cir::RecordType getFlattenedCoercedType(const ArgClassification &ac,
114 const mlir::DataLayout &dl) {
115 if (ac.kind != ArgKind::Direct || !ac.coercedType || !ac.canFlatten)
116 return {};
117 auto recTy = dyn_cast<cir::RecordType>(Val: ac.coercedType);
118 if (auto unionTy = dyn_cast_if_present<cir::UnionType>(Val&: recTy))
119 recTy = getUnionFlattenType(unionTy, dl);
120 if (!recTy || !recTy.isStruct() || recTy.getMembers().empty())
121 return {};
122 return recTy;
123}
124
125/// Whether \p ac is an Extend of a record argument of type \p origTy. Such an
126/// argument is a transparent union, passed as its first member, which
127/// \p ac.coercedType holds.
128bool isRecordExtend(const ArgClassification &ac, mlir::Type origTy) {
129 if (ac.kind != ArgKind::Extend || !mlir::isa<cir::RecordType>(Val: origTy))
130 return false;
131 assert(ac.coercedType && "an Extend of a record names its first member type");
132 return true;
133}
134
135/// Build the new argument-type list for a function whose ABI classification
136/// is \p fc. Handles Direct (with or without coercion), Extend, Ignore,
137/// Indirect (byval and non-byval), IndirectAliased (byref), and Expand
138/// (struct flattening) arguments.
139/// The sret return pointer, when present, is prepended by
140/// rewriteFunctionDefinition rather than here.
141mlir::LogicalResult buildNewArgTypes(
142 ArrayRef<mlir::Type> oldArgTypes, const FunctionClassification &fc,
143 SmallVectorImpl<mlir::Type> &newArgTypes, const mlir::DataLayout &dl,
144 function_ref<mlir::InFlightDiagnostic()> emitError) {
145 assert(newArgTypes.empty() && "expected an empty output vector");
146 newArgTypes.reserve(N: oldArgTypes.size());
147 for (auto [idx, ac] : llvm::enumerate(First: fc.argInfos)) {
148 mlir::Type origTy = oldArgTypes[idx];
149 switch (ac.kind) {
150 case ArgKind::Direct:
151 // Direct with canFlatten and a struct coerced type: push one wire type
152 // per field of the coerced struct rather than the struct itself.
153 if (cir::RecordType flatTy = getFlattenedCoercedType(ac, dl)) {
154 llvm::append_range(C&: newArgTypes, R: flatTy.getMembers());
155 } else {
156 // Direct with a coerced type: the lowered signature uses the coerced
157 // type; the body still expects origTy and insertArgCoercion recovers it
158 // from a coercion slot. Direct without coercion is a pass-through.
159 newArgTypes.push_back(Elt: ac.coercedType ? ac.coercedType : origTy);
160 }
161 break;
162 case ArgKind::Ignore:
163 break;
164 case ArgKind::Expand: {
165 // Flatten the struct into one wire argument per field. The
166 // reassembly in the callee body and the decomposition at the call
167 // site are handled by insertArgCoercion and rewriteCallSite.
168 auto recTy = cast<cir::RecordType>(Val&: origTy);
169 assert(recTy.isStruct() &&
170 "Expand classification requires a struct type, not a union");
171 assert(!recTy.getMembers().empty() &&
172 "Expand classification requires at least one struct field");
173 llvm::append_range(C&: newArgTypes, R: recTy.getMembers());
174 break;
175 }
176 case ArgKind::Extend:
177 // Extend keeps the original (narrow) type in the signature; the
178 // sign/zero extension is communicated to LLVM via the llvm.signext /
179 // llvm.zeroext arg attribute, attached separately below. For an integer
180 // the coercedType the classifier set is informational (typically the
181 // register-width type the value gets extended to in registers) and does
182 // not change the CIR signature. A transparent union is passed as the
183 // type of its first member, which coercedType holds.
184 newArgTypes.push_back(Elt: isRecordExtend(ac, origTy) ? ac.coercedType
185 : origTy);
186 break;
187 case ArgKind::Indirect:
188 case ArgKind::IndirectAliased:
189 newArgTypes.push_back(Elt: getIndirectPtrTy(pointee: origTy, ac));
190 break;
191 }
192 }
193 return mlir::success();
194}
195
196/// Compute the new return type for a function whose return classification
197/// is \p retInfo. Direct returns keep (or coerce to) their type, Ignore and
198/// Indirect (sret) returns become void, Extend keeps its type; Expand and
199/// IndirectAliased emit an error.
200mlir::Type
201computeNewReturnType(mlir::Type origRetTy, const ArgClassification &retInfo,
202 mlir::MLIRContext *ctx,
203 function_ref<mlir::InFlightDiagnostic()> emitError) {
204 switch (retInfo.kind) {
205 case ArgKind::Direct:
206 // Direct return with a coerced type uses the coerced type on the wire;
207 // the rewriter inserts a coercion before each cir.return.
208 return retInfo.coercedType ? retInfo.coercedType : origRetTy;
209 case ArgKind::Ignore:
210 return cir::VoidType::get(ctx);
211 case ArgKind::Expand:
212 emitError() << "Expand return is not allowed (classic codegen rejects "
213 << "it in EmitFunctionEpilog)";
214 return nullptr;
215 case ArgKind::IndirectAliased:
216 emitError() << "IndirectAliased return is not supported";
217 return nullptr;
218 case ArgKind::Extend:
219 // Same convention as Extend args: keep the original return type in the
220 // signature; the sign/zero extension is communicated via the
221 // llvm.signext / llvm.zeroext res attribute attached separately below.
222 return origRetTy;
223 case ArgKind::Indirect:
224 // sret: the value is returned through a pointer argument that the ABI
225 // synthesizes (rewriteFunctionDefinition prepends it to the argument
226 // list); it is not part of the source-level signature, so the wire
227 // return type becomes void.
228 return cir::VoidType::get(ctx);
229 }
230 llvm_unreachable("all ArgKind cases handled");
231}
232
233/// Create a typed poison constant to stand in for a value the body of a
234/// function (or the result of a call) still references but whose ABI
235/// classification is Ignore. Using poison is honest -- the value is
236/// genuinely unused at the ABI boundary -- and avoids a fake alloca+load
237/// pattern that would suggest we have a value when we don't.
238mlir::Value createIgnoredValue(mlir::OpBuilder &builder, mlir::Location loc,
239 mlir::Type ty) {
240 return cir::ConstantOp::create(builder, location: loc, res: ty, value: cir::PoisonAttr::get(type: ty));
241}
242
243/// Whether \p coercedTy is wider than the type a value of type \p origTy has
244/// in memory, so the coerced value passes bits the value does not define and
245/// cannot be noundef. This is the comparison classic CodeGen makes. A bool
246/// vector's memory type is its storage integer, one bit per element and at
247/// least a byte wide, not its power-of-two size.
248static bool isCoercionWiderThanMemory(mlir::Type origTy, mlir::Type coercedTy,
249 const mlir::DataLayout &dl) {
250 uint64_t memoryBits = dl.getTypeSizeInBits(t: origTy).getFixedValue();
251 if (auto vecTy = mlir::dyn_cast<cir::VectorType>(Val&: origTy);
252 vecTy && mlir::isa<cir::BoolType>(Val: vecTy.getElementType()))
253 memoryBits = vecTy.getBoolStorageWidth();
254 return dl.getTypeSizeInBits(t: coercedTy).getFixedValue() > memoryBits;
255}
256
257/// Whether updateArgAttrs has anything to change: an arg that is Ignore (its
258/// slot is dropped), Extend (needs llvm.signext / llvm.zeroext), Indirect or
259/// IndirectAliased (needs the pointer attributes), Expand (changes the argument
260/// count), or a coerced Direct, which may be flattened or may lose
261/// llvm.noundef, or that has a stack-slot alignment, which needs
262/// llvm.alignstack.
263static bool needsArgAttrUpdate(const FunctionClassification &fc) {
264 return llvm::any_of(Range: fc.argInfos, P: [](const ArgClassification &ac) {
265 return ac.kind == ArgKind::Ignore || ac.kind == ArgKind::Extend ||
266 ac.kind == ArgKind::Indirect ||
267 ac.kind == ArgKind::IndirectAliased || ac.kind == ArgKind::Expand ||
268 (ac.kind == ArgKind::Direct && (ac.coercedType || ac.directAlign));
269 });
270}
271
272/// Build an updated arg_attrs ArrayAttr that drops Ignore'd args, adds
273/// llvm.signext / llvm.zeroext on Extend args, adds the pointer attributes
274/// for Indirect and IndirectAliased args in place of llvm.nofpclass, drops
275/// llvm.noundef from a coerced Direct arg that passes undefined bits,
276/// adds llvm.alignstack on a Direct arg with a stack-slot alignment, gives each
277/// field of a Direct+canFlatten argument the argument's attributes, and gives
278/// each field of an Expand argument an empty dict. When \p enableNoundefAttrs
279/// is set, Extend, Indirect and IndirectAliased args are also marked
280/// llvm.noundef. Direct and Extend slots keep their other existing attributes.
281/// \p origArgTypes provides the pre-rewrite type for each arg slot.
282mlir::ArrayAttr updateArgAttrs(mlir::MLIRContext *ctx, mlir::ModuleOp module,
283 ArrayRef<mlir::Type> origArgTypes,
284 mlir::ArrayAttr existingArgAttrs,
285 const FunctionClassification &fc,
286 const mlir::DataLayout &dl,
287 bool enableNoundefAttrs, bool isCallSite) {
288 mlir::Builder builder(ctx);
289 SmallVector<mlir::Attribute> newArgAttrs;
290 newArgAttrs.reserve(N: fc.argInfos.size());
291 for (auto [oldIdx, ac] : llvm::enumerate(First: fc.argInfos)) {
292 if (ac.kind == ArgKind::Ignore)
293 continue;
294 mlir::DictionaryAttr existing = builder.getDictionaryAttr(value: {});
295 if (existingArgAttrs && oldIdx < existingArgAttrs.size())
296 existing = mlir::cast<mlir::DictionaryAttr>(Val: existingArgAttrs[oldIdx]);
297 if (ac.kind == ArgKind::Direct && ac.coercedType &&
298 isCoercionWiderThanMemory(origTy: origArgTypes[oldIdx], coercedTy: ac.coercedType, dl)) {
299 mlir::NamedAttrList attrs(existing);
300 attrs.erase(name: mlir::LLVM::LLVMDialect::getNoUndefAttrName());
301 existing = attrs.getDictionary(context: ctx);
302 }
303 if (ac.kind == ArgKind::Direct && ac.directAlign) {
304 mlir::NamedAttrList attrs(existing);
305 attrs.set(name: mlir::LLVM::LLVMDialect::getStackAlignmentAttrName(),
306 value: builder.getI64IntegerAttr(value: ac.directAlign->value()));
307 existing = attrs.getDictionary(context: ctx);
308 }
309 if (cir::RecordType flatTy = getFlattenedCoercedType(ac, dl)) {
310 // Direct + canFlatten: as in classic CodeGen, each flattened field
311 // carries the attributes of the argument it came from.
312 newArgAttrs.append(NumInputs: flatTy.getNumElements(), Elt: existing);
313 } else if (ac.kind == ArgKind::Expand) {
314 // An expanded record has no attributes classic CodeGen would copy to its
315 // fields, so each field gets an empty dict.
316 auto recTy = cast<cir::RecordType>(Val: origArgTypes[oldIdx]);
317 newArgAttrs.append(NumInputs: recTy.getNumElements(), Elt: builder.getDictionaryAttr(value: {}));
318 } else if (ac.kind == ArgKind::Extend) {
319 // An extended value has no undefined bits. This covers a transparent
320 // union passed as its extended first member, which CIRGen does not
321 // mark noundef because it is a record.
322 StringRef attrName = ac.signExtend
323 ? mlir::LLVM::LLVMDialect::getSExtAttrName()
324 : mlir::LLVM::LLVMDialect::getZExtAttrName();
325 mlir::NamedAttrList attrs(existing);
326 // Conservative argument extends apply only at the call-site.
327 if (isCallSite || !ac.conservative)
328 attrs.set(name: attrName, value: builder.getUnitAttr());
329 if (enableNoundefAttrs)
330 attrs.set(name: mlir::LLVM::LLVMDialect::getNoUndefAttrName(),
331 value: builder.getUnitAttr());
332 newArgAttrs.push_back(Elt: attrs.getDictionary(context: ctx));
333 } else if (ac.kind == ArgKind::Indirect) {
334 // Indirect lowering hands the callee a pointer. llvm.align and
335 // llvm.noundef describe that pointer, not the bytes it points to, and
336 // llvm.nofpclass, which describes the value, does not apply to a
337 // pointer. Without byval the pointer points to the caller's own object.
338 // With byval the backend copies the pointee on the caller's side of the
339 // call, so the callee gets an object of its own even when the pointer
340 // points to the caller's own object.
341 mlir::Type pointeeTy = origArgTypes[oldIdx];
342 mlir::NamedAttrList attrs(existing);
343 attrs.erase(name: mlir::LLVM::LLVMDialect::getNoFPClassAttrName());
344 attrs.set(name: mlir::LLVM::LLVMDialect::getAlignAttrName(),
345 value: builder.getI64IntegerAttr(value: ac.indirectAlign.value()));
346 if (enableNoundefAttrs)
347 attrs.set(name: mlir::LLVM::LLVMDialect::getNoUndefAttrName(),
348 value: builder.getUnitAttr());
349 if (ac.byVal) {
350 // llvm.byval(T) records the pre-rewrite arg type because the opaque
351 // LLVM pointer cannot carry it. Classic adds llvm.noalias under
352 // -fpass-by-value-is-noalias, which CIR does not plumb through.
353 assert(!cir::MissingFeatures::noaliasOnByvalAttr());
354 attrs.set(name: mlir::LLVM::LLVMDialect::getByValAttrName(),
355 value: mlir::TypeAttr::get(type: pointeeTy));
356 } else {
357 // llvm.nofreeobj says the object cannot be freed while the callee
358 // runs, which holds because the caller owns it across the call.
359 attrs.set(name: mlir::LLVM::LLVMDialect::getNoFreeObjAttrName(),
360 value: builder.getUnitAttr());
361 // llvm.dead_on_return says the object's lifetime ends in the callee,
362 // which holds unless the caller still has a destructor to run on it.
363 // A record with no layout entry has no destructor.
364 auto recTy = mlir::dyn_cast<cir::RecordType>(Val&: pointeeTy);
365 cir::RecordLayoutAttr layout =
366 recTy ? cir::tryGetRecordLayout(mod: module, name: recTy.getName())
367 : cir::RecordLayoutAttr();
368 if (!layout || layout.getHasTrivialDtor())
369 attrs.set(name: mlir::LLVM::LLVMDialect::getDeadOnReturnAttrName(),
370 value: builder.getI64IntegerAttr(value: static_cast<int64_t>(
371 llvm::DeadOnReturnInfo().toIntValue())));
372 attrs.set(name: mlir::LLVM::LLVMDialect::getDereferenceableAttrName(),
373 value: builder.getI64IntegerAttr(
374 value: dl.getTypeAllocSize(t: pointeeTy).getFixedValue()));
375 }
376 newArgAttrs.push_back(Elt: attrs.getDictionary(context: ctx));
377 } else if (ac.kind == ArgKind::IndirectAliased) {
378 // llvm.byref(T) records the pre-rewrite arg type, as llvm.byval does.
379 // llvm.nofpclass describes the value and does not apply to a pointer.
380 mlir::NamedAttrList attrs(existing);
381 attrs.erase(name: mlir::LLVM::LLVMDialect::getNoFPClassAttrName());
382 attrs.set(name: mlir::LLVM::LLVMDialect::getAlignAttrName(),
383 value: builder.getI64IntegerAttr(value: ac.indirectAlign.value()));
384 if (enableNoundefAttrs)
385 attrs.set(name: mlir::LLVM::LLVMDialect::getNoUndefAttrName(),
386 value: builder.getUnitAttr());
387 attrs.set(name: mlir::LLVM::LLVMDialect::getByRefAttrName(),
388 value: mlir::TypeAttr::get(type: origArgTypes[oldIdx]));
389 newArgAttrs.push_back(Elt: attrs.getDictionary(context: ctx));
390 } else {
391 newArgAttrs.push_back(Elt: existing);
392 }
393 }
394 return builder.getArrayAttr(value: newArgAttrs);
395}
396
397/// Build an updated res_attrs ArrayAttr (single entry, since CIR funcs have
398/// at most one result) that adds llvm.signext / llvm.zeroext on an Extend
399/// return and drops llvm.noundef from a coerced return that passes undefined
400/// bits. Preserves any other existing res attributes. Returns
401/// \p existingResAttrs unchanged, which may be null, when there is nothing to
402/// add or drop.
403mlir::ArrayAttr updateResAttrs(mlir::MLIRContext *ctx,
404 mlir::ArrayAttr existingResAttrs,
405 const ArgClassification &retInfo,
406 mlir::Type origRetTy,
407 const mlir::DataLayout &dl) {
408 bool needsNoUndefDrop =
409 retInfo.kind == ArgKind::Direct && retInfo.coercedType &&
410 isCoercionWiderThanMemory(origTy: origRetTy, coercedTy: retInfo.coercedType, dl);
411 if (retInfo.kind != ArgKind::Extend && !needsNoUndefDrop)
412 return existingResAttrs;
413
414 mlir::NamedAttrList attrs;
415 if (existingResAttrs && !existingResAttrs.empty())
416 attrs.append(
417 newAttributes: mlir::cast<mlir::DictionaryAttr>(Val: existingResAttrs[0]).getValue());
418 if (retInfo.kind == ArgKind::Extend)
419 attrs.set(name: retInfo.signExtend ? "llvm.signext" : "llvm.zeroext",
420 value: mlir::UnitAttr::get(context: ctx));
421 if (needsNoUndefDrop)
422 attrs.erase(name: mlir::LLVM::LLVMDialect::getNoUndefAttrName());
423 return mlir::ArrayAttr::get(context: ctx, value: {attrs.getDictionary(context: ctx)});
424}
425
426/// The number of bytes a coercion memory slot needs to hold a value of type
427/// \p ty without truncating it. For most types this is the ordinary storage
428/// size. For a _BitInt it is deliberately the value's own literal byte
429/// footprint (ceil(width/8)) rather than the wider, ABI-alignment-padded
430/// footprint a _BitInt gets as a record member (see
431/// cir::IntType::getStorageTypeWidth): this coercion is about how many bytes
432/// the *value* needs to round-trip, not how a record would lay it out, and
433/// those are genuinely different questions for a _BitInt (e.g. _BitInt(33)
434/// only needs 5 bytes here, even though it occupies 8 padded bytes as a
435/// record member).
436static uint64_t coercionByteSize(mlir::Type ty, const mlir::DataLayout &dl) {
437 if (auto intTy = mlir::dyn_cast<cir::IntType>(Val&: ty))
438 return llvm::divideCeil(Numerator: intTy.getWidth(), Denominator: 8);
439 return dl.getTypeSize(t: ty);
440}
441
442/// A \p wantTy-typed view of the storage \p base points at. When \p wantTy is
443/// \p base's own pointee type, \p base is returned unchanged and \p offset must
444/// be 0. Any other type is reached by a bitcast, preceded by a byte stride
445/// of \p offset bytes when \p offset is nonzero. The view keeps \p base's
446/// address space. Any operations created are added to \p createdOps.
447static mlir::Value
448emitViewAtOffset(mlir::OpBuilder &builder, mlir::Location loc,
449 mlir::Type wantTy, mlir::Value base,
450 SmallPtrSetImpl<mlir::Operation *> &createdOps,
451 unsigned offset) {
452 auto basePtrTy = mlir::cast<cir::PointerType>(Val: base.getType());
453 if (wantTy == basePtrTy.getPointee()) {
454 assert(offset == 0 && "a view of the storage's own type has no offset");
455 return base;
456 }
457 mlir::ptr::MemorySpaceAttrInterface addrSpace = basePtrTy.getAddrSpace();
458 mlir::Value view = base;
459 if (offset != 0) {
460 auto u8Ty = cir::IntType::get(context: builder.getContext(), width: 8, /*isSigned=*/is_signed: false);
461 auto u8PtrTy = cir::PointerType::get(pointee: u8Ty, addrSpace);
462 auto u8Base = cir::CastOp::create(builder, location: loc, result: u8PtrTy,
463 kind: cir::CastKind::bitcast, src: base);
464 createdOps.insert(Ptr: u8Base);
465 auto strideTy =
466 cir::IntType::get(context: builder.getContext(), width: 64, /*isSigned=*/is_signed: true);
467 auto strideVal = cir::ConstantOp::create(
468 builder, location: loc, value: cir::IntAttr::get(type: strideTy, value: offset));
469 createdOps.insert(Ptr: strideVal);
470 view = cir::PtrStrideOp::create(builder, location: loc, result: u8PtrTy, base: u8Base, stride: strideVal);
471 createdOps.insert(Ptr: view.getDefiningOp());
472 }
473 auto typedView = cir::CastOp::create(builder, location: loc,
474 result: cir::PointerType::get(pointee: wantTy, addrSpace),
475 kind: cir::CastKind::bitcast, src: view);
476 createdOps.insert(Ptr: typedView);
477 return typedView;
478}
479
480/// A pointer to \p pointee in the target's alloca address space, which every
481/// temporary this pass allocates has to live in.
482static cir::PointerType getAllocaPtrTy(mlir::Type pointee,
483 const mlir::DataLayout &dl) {
484 mlir::ptr::MemorySpaceAttrInterface addrSpace;
485 if (auto asAttr = mlir::dyn_cast_if_present<mlir::IntegerAttr>(
486 Val: dl.getAllocaMemorySpace()))
487 addrSpace = cir::TargetAddressSpaceAttr::get(context: pointee.getContext(),
488 value: asAttr.getUInt());
489 return cir::PointerType::get(pointee, addrSpace);
490}
491
492/// A coercion slot for \p srcTy and \p dstTy, which must differ. It is typed
493/// as \p srcTy unless coercionByteSize says \p dstTy is larger, aligned for
494/// both, and placed at the start of \p slotBlock. \p offset, where the
495/// coerced side sits within the slot, is only checked by assertions. The
496/// alloca is added to \p createdOps.
497static cir::AllocaOp createCoercionSlot(
498 mlir::OpBuilder &builder, mlir::Location loc, mlir::Type srcTy,
499 mlir::Type dstTy, mlir::Block *slotBlock, const mlir::DataLayout &dl,
500 SmallPtrSetImpl<mlir::Operation *> &createdOps, unsigned offset) {
501 assert(srcTy != dstTy && "callers must pre-check that the types differ");
502
503 cir::CIRDataLayout cirDataLayout(
504 slotBlock->getParentOp()->getParentOfType<mlir::ModuleOp>());
505 uint64_t srcAlign = cirDataLayout.getTypeLanguageAlignment(ty: srcTy);
506 uint64_t dstAlign = cirDataLayout.getTypeLanguageAlignment(ty: dstTy);
507 uint64_t allocaAlign = std::max(a: srcAlign, b: dstAlign);
508 mlir::Type slotTy = coercionByteSize(ty: srcTy, dl) >= coercionByteSize(ty: dstTy, dl)
509 ? srcTy
510 : dstTy;
511
512 // Sizes are compared two ways on purpose: relative size in
513 // coercionByteSize terms, which is how slotTy was picked, and capacity in
514 // getTypeSize terms, which is what the alloca is given.
515 [[maybe_unused]] mlir::Type coercedTy = ((slotTy == srcTy) ? dstTy : srcTy);
516 assert((offset == 0 ||
517 coercionByteSize(coercedTy, dl) < coercionByteSize(slotTy, dl)) &&
518 "a direct offset must land on the coerced side, the smaller one");
519 assert((offset == 0 ||
520 offset + dl.getTypeSize(coercedTy) <= dl.getTypeSize(slotTy)) &&
521 "coerce slot too small for offset access");
522 assert((offset == 0 ||
523 offset % cirDataLayout.getTypeLanguageAlignment(coercedTy) == 0) &&
524 "a direct offset must be aligned for the coerced access");
525
526 mlir::OpBuilder::InsertionGuard guard(builder);
527 builder.setInsertionPointToStart(slotBlock);
528 auto alloca = cir::AllocaOp::create(builder, location: loc, addr: getAllocaPtrTy(pointee: slotTy, dl),
529 name: builder.getStringAttr(bytes: "coerce"),
530 alignment: builder.getI64IntegerAttr(value: allocaAlign));
531 createdOps.insert(Ptr: alloca);
532 return alloca;
533}
534
535/// Coerce \p src into a temporary memory slot typed for \p dstTy at the
536/// current builder insertion point, and return the destination-typed pointer
537/// to that slot without loading the value back out. This is the shared
538/// memory half of emitCoercion: callers that want the whole coerced value use
539/// emitCoercion (below); callers that want to read individual members of a
540/// coerced struct (the call-site struct flattening) take the returned pointer
541/// and emit their own cir.get_member + cir.load per field. Lowers uniformly
542/// for scalar, vector, and record types.
543///
544/// The slot is sized to the larger of the two types so that neither the store
545/// nor a later load ever runs past it: the coerced ABI type can be larger
546/// than the original (e.g. a 12-byte aggregate passed as `{i64, i64}`), so
547/// accessing the destination through a source-sized slot would over-read.
548/// Alignment is max(srcAlign, dstAlign) to satisfy both accesses. The slot
549/// is written through a source-typed view and returned as a destination-typed
550/// view.
551///
552/// The temporary alloca is placed at the start of \p slotBlock, which must
553/// dominate every use of the coerced value and must be a block that ends up
554/// inside the enclosing function's entry block after any later outlining.
555///
556/// \p offset is where the coerced value sits within the larger of the two
557/// types, non-zero when the ABI passes a value in a register read from
558/// partway into its storage. The coerced type must be the smaller of the
559/// two, so one that can exceed the value it coerces, such as a multi-field
560/// register tuple, must not be given an offset.
561///
562/// Any operations the helper creates are appended to \p createdOps so the
563/// caller can leave them out when it later rewires the source value's uses,
564/// which would otherwise clobber the store's value operand.
565mlir::Value emitCoercionToMemory(mlir::OpBuilder &builder, mlir::Location loc,
566 mlir::Type dstTy, mlir::Value src,
567 mlir::Block *slotBlock,
568 const mlir::DataLayout &dl,
569 SmallPtrSetImpl<mlir::Operation *> &createdOps,
570 unsigned offset) {
571 mlir::Type srcTy = src.getType();
572 cir::AllocaOp alloca = createCoercionSlot(builder, loc, srcTy, dstTy,
573 slotBlock, dl, createdOps, offset);
574 mlir::Type slotTy = alloca.getAllocaType();
575
576 // Store through a source-typed view of the slot.
577 mlir::Value srcSlot = emitViewAtOffset(
578 builder, loc, wantTy: srcTy, base: alloca, createdOps, offset: srcTy == slotTy ? 0 : offset);
579 auto store = cir::StoreOp::create(builder, location: loc, value: src, addr: srcSlot);
580 createdOps.insert(Ptr: store);
581
582 // Return a destination-typed view of the slot.
583 return emitViewAtOffset(builder, loc, wantTy: dstTy, base: alloca, createdOps,
584 offset: dstTy == slotTy ? 0 : offset);
585}
586
587/// Cast \p src to \p dstTy when both are pointers to the same pointee, so only
588/// the address space changes. Returns null when a real coercion is needed.
589static mlir::Value
590emitAddrSpaceCoercion(mlir::OpBuilder &builder, mlir::Location loc,
591 mlir::Type dstTy, mlir::Value src,
592 SmallPtrSetImpl<mlir::Operation *> &createdOps,
593 unsigned offset) {
594 auto srcPtrTy = dyn_cast<cir::PointerType>(Val: src.getType());
595 auto dstPtrTy = dyn_cast<cir::PointerType>(Val&: dstTy);
596 if (offset || !srcPtrTy || !dstPtrTy ||
597 srcPtrTy.getPointee() != dstPtrTy.getPointee())
598 return {};
599 assert(srcPtrTy.getAddrSpace() != dstPtrTy.getAddrSpace() &&
600 "pointers to the same pointee must differ in address space");
601 auto cast = cir::CastOp::create(builder, location: loc, result: dstTy,
602 kind: cir::CastKind::address_space, src);
603 createdOps.insert(Ptr: cast);
604 return cast;
605}
606
607/// Coerce \p src to type \p dstTy by going through memory and load the whole
608/// coerced value back out. Builds on emitCoercionToMemory, adding the final
609/// load of the destination-typed view. A pointer that only changes address
610/// space is cast instead.
611mlir::Value emitCoercion(mlir::OpBuilder &builder, mlir::Location loc,
612 mlir::Type dstTy, mlir::Value src,
613 mlir::Block *slotBlock, const mlir::DataLayout &dl,
614 SmallPtrSetImpl<mlir::Operation *> &createdOps,
615 unsigned offset) {
616 if (mlir::Value cast =
617 emitAddrSpaceCoercion(builder, loc, dstTy, src, createdOps, offset))
618 return cast;
619
620 mlir::Value dstSlot = emitCoercionToMemory(builder, loc, dstTy, src,
621 slotBlock, dl, createdOps, offset);
622 auto load = cir::LoadOp::create(builder, location: loc, addr: dstSlot);
623 createdOps.insert(Ptr: load);
624 return load;
625}
626
627/// Convenience overload for callers that don't need the createdOps set
628/// (e.g. call-site coercion where we don't replaceAllUsesExcept).
629mlir::Value emitCoercion(mlir::OpBuilder &builder, mlir::Location loc,
630 mlir::Type dstTy, mlir::Value src,
631 mlir::Block *slotBlock, const mlir::DataLayout &dl,
632 unsigned offset) {
633 SmallPtrSet<mlir::Operation *, 4> ignored;
634 return emitCoercion(builder, loc, dstTy, src, slotBlock, dl, createdOps&: ignored, offset);
635}
636
637/// The block a coercion slot's alloca belongs at the start of.
638///
639/// Normally the enclosing function's entry block, where HoistAllocas expects
640/// allocas to be. A body carrying a call is not always inside a function
641/// when this pass runs, though, because LoweringPrepare runs after it: a
642/// namespace-scope `T g = makeT();` is still in its cir.global ctor region,
643/// and an OpenACC recipe's init and destroy bodies are in regions the module
644/// owns. Those take the outermost region below the module, which dominates
645/// the whole body and travels with it when the body is outlined.
646mlir::Block *coercionSlotBlock(mlir::Operation *op) {
647 if (auto funcOp = op->getParentOfType<mlir::FunctionOpInterface>())
648 return &funcOp->getRegion(index: 0).front();
649 mlir::Region *region = op->getParentRegion();
650 while (mlir::Region *outer = region->getParentRegion()) {
651 if (mlir::isa<mlir::ModuleOp>(Val: outer->getParentOp()))
652 break;
653 region = outer;
654 }
655 assert(!region->empty() && "coercion slot needs a block to hold the alloca");
656 return &region->front();
657}
658
659/// \p val's defining load, if it is simple, meaning neither volatile nor
660/// atomic. Null otherwise: a non-simple load's access has to survive as
661/// written, and a call result or any other first-class value has no defining
662/// load at all.
663static cir::LoadOp maybeGetSimpleLoad(mlir::Value val) {
664 cir::LoadOp load = val.getDefiningOp<cir::LoadOp>();
665 if (!load || load.isVolatileOrAtomic())
666 return {};
667 return load;
668}
669
670/// \p val's defining load, if it loads a record and is not atomic. Null
671/// otherwise. The load may be volatile, and then its bytes may only be read
672/// through a volatile copy, never in place.
673static cir::LoadOp maybeGetCopyableRecordLoad(mlir::Value val) {
674 cir::LoadOp load = val.getDefiningOp<cir::LoadOp>();
675 if (!load || load.getMemOrder() ||
676 !mlir::isa<cir::RecordType>(Val: load.getType()))
677 return {};
678 return load;
679}
680
681/// Emits an error and returns failure if \p val, an operand of \p user, is an
682/// atomic load of a record. The record's bytes are read through a copy or in
683/// place, neither of which keeps the load atomic.
684static mlir::LogicalResult rejectAtomicRecordLoad(mlir::Operation *user,
685 mlir::Value val) {
686 auto load = val.getDefiningOp<cir::LoadOp>();
687 if (!load || !load.getMemOrder() ||
688 !mlir::isa<cir::RecordType>(Val: load.getType()))
689 return mlir::success();
690 return user->emitOpError()
691 << "atomic load of a record operand not yet implemented in "
692 "CallConvLowering";
693}
694
695/// Whether the storage the record load \p load reads is an alloca that holds
696/// \p coercedTy at \p offset. With no offset, the coerced type's alloc size
697/// has to fit in the record's. At an offset, the coerced value has to end
698/// within the record's alloc size.
699static bool canReadInPlace(cir::LoadOp load, mlir::Type coercedTy,
700 const cir::CIRDataLayout &dl, unsigned offset) {
701 if (!cir::getUnderlyingAlloca(addr: load.getAddr()))
702 return false;
703 uint64_t readEnd =
704 offset == 0 ? dl.getTypeAllocSize(ty: coercedTy).getFixedValue()
705 : offset + dl.getTypeStoreSize(ty: coercedTy).getFixedValue();
706 return readEnd <= dl.getTypeAllocSize(ty: load.getType()).getFixedValue();
707}
708
709/// Read \p coercedTy at \p offset straight out of the storage the record load
710/// \p load reads, which must be an alloca. The read goes just before \p load,
711/// so it reads the bytes \p load reads, and takes the alloca's alignment at
712/// \p offset.
713static mlir::Value emitInPlaceCoercedLoad(mlir::OpBuilder &builder,
714 mlir::Location loc, cir::LoadOp load,
715 mlir::Type coercedTy,
716 unsigned offset) {
717 cir::AllocaOp src = cir::getUnderlyingAlloca(addr: load.getAddr());
718 assert(src && "the record load must read an alloca");
719 mlir::OpBuilder::InsertionGuard guard(builder);
720 builder.setInsertionPoint(load);
721 SmallPtrSet<mlir::Operation *, 4> ignored;
722 mlir::Value view = emitViewAtOffset(builder, loc, wantTy: coercedTy, base: load.getAddr(),
723 createdOps&: ignored, offset);
724 auto coerced = cir::LoadOp::create(builder, location: loc, addr: view);
725 coerced.setAlignment(llvm::MinAlign(A: src.getAlignment(), B: offset));
726 return coerced;
727}
728
729/// Copy the bytes of the record \p load reads into a new coercion slot for
730/// \p coercedTy at \p offset, placed at the start of \p slotBlock, and return
731/// the slot. The slot is at least as aligned as the storage copied from. The
732/// copy goes just before \p load, so it reads the bytes \p load reads, and is
733/// volatile if \p load is.
734static cir::AllocaOp
735emitCopiedCoercionSlot(mlir::OpBuilder &builder, mlir::Location loc,
736 cir::LoadOp load, mlir::Type coercedTy,
737 mlir::Block *slotBlock, const mlir::DataLayout &dl,
738 unsigned offset) {
739 auto recTy = cast<cir::RecordType>(Val: load.getType());
740 uint64_t srcAlign;
741 if (cir::AllocaOp src = cir::getUnderlyingAlloca(addr: load.getAddr()))
742 srcAlign = src.getAlignment();
743 else
744 srcAlign = load.getAlignment().value_or(
745 u: cir::CIRDataLayout(load->getParentOfType<mlir::ModuleOp>())
746 .getTypeLanguageAlignment(ty: recTy));
747 SmallPtrSet<mlir::Operation *, 4> ignored;
748 cir::AllocaOp slot = createCoercionSlot(builder, loc, srcTy: recTy, dstTy: coercedTy,
749 slotBlock, dl, createdOps&: ignored, offset);
750 slot.setAlignment(std::max(a: slot.getAlignment(), b: srcAlign));
751 assert((offset == 0 || slot.getAllocaType() == recTy) &&
752 "a direct offset must leave the record as the slot's type");
753 mlir::OpBuilder::InsertionGuard guard(builder);
754 builder.setInsertionPoint(load);
755 mlir::Value recView =
756 emitViewAtOffset(builder, loc, wantTy: recTy, base: slot, createdOps&: ignored, /*offset=*/0);
757 auto copy =
758 cir::CopyOp::create(builder, location: loc, dst: recView, src: load.getAddr(),
759 dst_alignment: builder.getI64IntegerAttr(value: slot.getAlignment()),
760 src_alignment: builder.getI64IntegerAttr(value: srcAlign));
761 copy.setIsVolatile(load.getIsVolatile());
762 return slot;
763}
764
765/// Copy the bytes of the record \p load reads into a coercion slot, as
766/// emitCopiedCoercionSlot does, then read \p coercedTy out of the slot at
767/// \p offset. The read takes the slot's alignment at \p offset.
768static mlir::Value emitCopiedCoercedLoad(mlir::OpBuilder &builder,
769 mlir::Location loc, cir::LoadOp load,
770 mlir::Type coercedTy,
771 mlir::Block *slotBlock,
772 const mlir::DataLayout &dl,
773 unsigned offset) {
774 cir::AllocaOp slot = emitCopiedCoercionSlot(builder, loc, load, coercedTy,
775 slotBlock, dl, offset);
776 mlir::OpBuilder::InsertionGuard guard(builder);
777 builder.setInsertionPoint(load);
778 SmallPtrSet<mlir::Operation *, 4> ignored;
779 mlir::Value coercedView =
780 emitViewAtOffset(builder, loc, wantTy: coercedTy, base: slot, createdOps&: ignored, offset);
781 auto coerced = cir::LoadOp::create(builder, location: loc, addr: coercedView);
782 coerced.setAlignment(llvm::MinAlign(A: slot.getAlignment(), B: offset));
783 return coerced;
784}
785
786/// Read \p coercedTy at \p offset out of the bytes the record load \p load
787/// reads: straight out of its storage if \p load is not volatile and
788/// canReadInPlace accepts it, and otherwise out of a coercion slot at the start
789/// of \p slotBlock that those bytes are copied into. emitInPlaceCoercedLoad
790/// does not mark its read volatile, so a volatile load is read through the
791/// volatile copy instead. Either way no new load of the record type is
792/// emitted.
793static mlir::Value emitCoercedRecordLoad(
794 mlir::OpBuilder &builder, mlir::Location loc, cir::LoadOp load,
795 mlir::Type coercedTy, mlir::Block *slotBlock, const mlir::DataLayout &dl,
796 const cir::CIRDataLayout &cirDataLayout, unsigned offset) {
797 if (!load.getIsVolatile() &&
798 canReadInPlace(load, coercedTy, dl: cirDataLayout, offset))
799 return emitInPlaceCoercedLoad(builder, loc, load, coercedTy, offset);
800 return emitCopiedCoercedLoad(builder, loc, load, coercedTy, slotBlock, dl,
801 offset);
802}
803
804/// Whether a cir.copy of \p ty into \p dst that stands in for a store of it
805/// skips tail padding. Unless \p dst is an alloca, which holds a complete
806/// object, it can be a potentially-overlapping subobject, such as a
807/// [[no_unique_address]] member or a base, whose tail padding holds another
808/// object. Classic CodeGen writes only the data size into a destination it is
809/// handed (EmitCall) and into one that may overlap (EmitAggregateCopy).
810/// Copying the whole object into an alloca is equivalent, since nothing else
811/// lives in its tail padding. A struct has tail padding when its last members
812/// are pad members, which CIRGen emits for padding another object may reuse and
813/// for some other structs, such as an over-aligned one.
814static bool shouldSkipTailPadding(mlir::Value dst, mlir::Type ty) {
815 auto structTy = mlir::dyn_cast<cir::StructType>(Val&: ty);
816 if (!structTy || cir::getUnderlyingAlloca(addr: dst))
817 return false;
818 llvm::ArrayRef<mlir::Type> members = structTy.getMembers();
819 llvm::ArrayRef<cir::RecordMemberKind> kinds = structTy.getMemberKinds();
820 for (auto [member, kind] : llvm::reverse(C: llvm::zip_equal(t&: members, u&: kinds)))
821 if (cir::memberOwnsBytes(memberTy: member))
822 return kind == cir::RecordMemberKind::Pad;
823 return false;
824}
825
826/// Rewire the uses of the record value \p val, whose bytes sit at \p addr,
827/// which is aligned to \p addrAlign, so that each one finds the record in
828/// memory. \p val may already have been retyped to the type it is passed as, a
829/// coerced type or a pointer, but its uses still expect the record.
830///
831/// A store of \p val becomes a cir.copy from \p addr at the store, volatile if
832/// the store is and skipping tail padding as shouldSkipTailPadding says. A
833/// store of the record value writes only what its lowered type stores, so
834/// another union member's data past those bytes would be lost. A consumer may
835/// be rewritten before \p val's defining op, and has then already stored \p val
836/// into a slot of its own. That store becomes a copy here too.
837///
838/// Every other use is pointed at one cir.load of \p addr created at the
839/// builder's insertion point. Only a call argument, a return and a read of a
840/// member that is not itself a record may use it, since those either read
841/// \p addr when they are rewritten or read a field. Uses by an operation in
842/// \p exceptions are left alone. Fails, after reporting it, on an atomic store
843/// of \p val or a use by anything else, before anything is rewired.
844static mlir::LogicalResult
845rewireRecordUses(mlir::OpBuilder &builder, mlir::Location loc, mlir::Value val,
846 mlir::Value addr, uint64_t addrAlign,
847 const mlir::DataLayout &dl,
848 const SmallPtrSetImpl<mlir::Operation *> &exceptions) {
849 mlir::Type recTy = mlir::cast<cir::PointerType>(Val: addr.getType()).getPointee();
850 assert(mlir::isa<cir::RecordType>(recTy) && "only a record is rewired");
851 SmallVector<cir::StoreOp> stores;
852 SmallVector<mlir::OpOperand *> otherUses;
853 for (mlir::OpOperand &use : val.getUses()) {
854 mlir::Operation *user = use.getOwner();
855 if (exceptions.contains(Ptr: user))
856 continue;
857 if (auto store = mlir::dyn_cast<cir::StoreOp>(Val: user)) {
858 if (store.getMemOrder())
859 return store.emitOpError()
860 << "atomic store of a record argument or result not yet "
861 "implemented in CallConvLowering";
862 stores.push_back(Elt: store);
863 continue;
864 }
865 auto member = mlir::dyn_cast<cir::ExtractMemberOp>(Val: user);
866 bool isFieldRead = member && !mlir::isa<cir::RecordType>(Val: member.getType());
867 if (!isFieldRead &&
868 !mlir::isa<cir::CIRCallOpInterface, cir::ReturnOp>(Val: user))
869 return user->emitOpError()
870 << "use of a record argument or result other than by a store, "
871 "call, return or member read not yet implemented in "
872 "CallConvLowering";
873 otherUses.push_back(Elt: &use);
874 }
875
876 mlir::IntegerAttr srcAlign = builder.getI64IntegerAttr(value: addrAlign);
877 cir::CIRDataLayout cirDataLayout(
878 val.getParentRegion()->getParentOfType<mlir::ModuleOp>());
879 for (cir::StoreOp store : stores) {
880 mlir::OpBuilder::InsertionGuard guard(builder);
881 builder.setInsertionPoint(store);
882 uint64_t dstAlign = store.getAlignment().value_or(
883 u: cirDataLayout.getTypeLanguageAlignment(ty: recTy));
884 auto copy =
885 cir::CopyOp::create(builder, location: store.getLoc(), dst: store.getAddr(), src: addr,
886 dst_alignment: builder.getI64IntegerAttr(value: dstAlign), src_alignment: srcAlign);
887 copy.setIsVolatile(store.getIsVolatile());
888 copy.setSkipTailPadding(shouldSkipTailPadding(dst: store.getAddr(), ty: recTy));
889 store->erase();
890 }
891
892 if (otherUses.empty())
893 return mlir::success();
894 auto load = cir::LoadOp::create(builder, location: loc, addr);
895 load.setAlignment(addrAlign);
896 for (mlir::OpOperand *use : otherUses)
897 use->set(load);
898 return mlir::success();
899}
900
901/// Point the uses of \p val outside \p exceptions at the value of \p val's
902/// original type that \p view, a view of the start of a coercion slot, holds.
903/// For a record, rewireRecordUses points the uses at the slot. Any other type
904/// is loaded out of it at the builder's insertion point.
905static mlir::LogicalResult
906replaceUsesFromSlot(mlir::OpBuilder &builder, mlir::Location loc,
907 mlir::Value val, mlir::Value view,
908 const mlir::DataLayout &dl,
909 const SmallPtrSetImpl<mlir::Operation *> &exceptions) {
910 if (!mlir::isa<cir::RecordType>(
911 Val: mlir::cast<cir::PointerType>(Val: view.getType()).getPointee())) {
912 auto load = cir::LoadOp::create(builder, location: loc, addr: view);
913 val.replaceAllUsesExcept(newValue: load, exceptions);
914 return mlir::success();
915 }
916 cir::AllocaOp slot = cir::getUnderlyingAlloca(addr: view);
917 assert(slot && "the view must start at the coercion slot");
918 return rewireRecordUses(builder, loc, val, addr: view, addrAlign: slot.getAlignment(), dl,
919 exceptions);
920}
921
922/// Coerce \p src, which is \p val retyped or the value replacing it, back to
923/// \p origTy through a coercion slot at the start of \p slotBlock, and point
924/// the uses of \p val outside \p createdOps at the result with
925/// replaceUsesFromSlot. A direct \p offset lands on the coerced side, so the
926/// view of \p origTy starts at the slot. A pointer that only changes address
927/// space is cast instead. The coercion's operations are added to
928/// \p createdOps.
929static mlir::LogicalResult
930emitCoercionAndReplaceUses(mlir::OpBuilder &builder, mlir::Location loc,
931 mlir::Type origTy, mlir::Value src, mlir::Value val,
932 mlir::Block *slotBlock, const mlir::DataLayout &dl,
933 SmallPtrSetImpl<mlir::Operation *> &createdOps,
934 unsigned offset) {
935 if (mlir::Value cast = emitAddrSpaceCoercion(builder, loc, dstTy: origTy, src,
936 createdOps, offset)) {
937 val.replaceAllUsesExcept(newValue: cast, exceptions: createdOps);
938 return mlir::success();
939 }
940 mlir::Value view = emitCoercionToMemory(builder, loc, dstTy: origTy, src, slotBlock,
941 dl, createdOps, offset);
942 return replaceUsesFromSlot(builder, loc, val, view, dl, exceptions: createdOps);
943}
944
945/// Coerce the value each cir.return returns to the new (coerced) return type.
946///
947/// When the returned value is a non-atomic load of a record, the coerced value
948/// is read by emitCoercedRecordLoad, and the record load is erased once nothing
949/// else uses it. Any other value is stored into a coercion slot and read back.
950/// A record with no storage behind it, such as a constant, defines no bytes
951/// beyond its fields, so the store writes every byte it defines. A call
952/// result's store is turned into a copy when its call is rewritten later.
953/// Fails, after reporting it, on an atomic load of a record.
954mlir::LogicalResult
955insertReturnCoercion(mlir::FunctionOpInterface funcOp, mlir::Type origRetTy,
956 mlir::Type coercedRetTy, mlir::OpBuilder &builder,
957 const mlir::DataLayout &dl,
958 const cir::CIRDataLayout &cirDataLayout, unsigned offset) {
959 SmallVector<cir::ReturnOp> returns;
960 funcOp.walk(callback: [&](cir::ReturnOp r) { returns.push_back(Elt: r); });
961 for (cir::ReturnOp r : returns) {
962 if (r.getInput().empty())
963 continue;
964 mlir::Value origVal = r.getInput()[0];
965 if (origVal.getType() == coercedRetTy)
966 continue;
967
968 if (failed(Result: rejectAtomicRecordLoad(user: r, val: origVal)))
969 return mlir::failure();
970 if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(val: origVal)) {
971 mlir::Value coerced = emitCoercedRecordLoad(
972 builder, loc: r.getLoc(), load: recLoad, coercedTy: coercedRetTy,
973 slotBlock: &funcOp->getRegion(index: 0).front(), dl, cirDataLayout, offset);
974 r->setOperand(idx: 0, value: coerced);
975 if (recLoad->use_empty())
976 recLoad->erase();
977 continue;
978 }
979
980 builder.setInsertionPoint(r);
981 mlir::Value coerced =
982 emitCoercion(builder, loc: r.getLoc(), dstTy: coercedRetTy, src: origVal,
983 slotBlock: &funcOp->getRegion(index: 0).front(), dl, offset);
984 r->setOperand(idx: 0, value: coerced);
985 }
986 return mlir::success();
987}
988
989/// \p recordVal's defining load, if it is simple and its address resolves to
990/// an alloca. Null otherwise.
991static cir::LoadOp getWholeRecordLoad(mlir::Value recordVal) {
992 cir::LoadOp load = maybeGetSimpleLoad(val: recordVal);
993 if (!load || !cir::getUnderlyingAlloca(addr: load.getAddr()))
994 return {};
995 return load;
996}
997
998/// Returns the address \p load read from, as a pointer of type \p paramTy, if
999/// it can be passed in place of the loaded value for an indirect argument that
1000/// needs \p minAlign alignment, and null otherwise, including when \p load is
1001/// null. A byval or byref argument also needs storageUnwrittenBetween. On
1002/// null the caller reports a non-byval argument or fills a slot of its own.
1003///
1004/// The address has to have \p paramTy, the rewritten parameter's type, since a
1005/// cir.load pins the pointee type without pinning the address space. An
1006/// address_space cast of a \p paramTy pointer qualifies through its source.
1007/// The storage it names has to be a slot allocated here, reached through
1008/// storage-preserving casts, already stating at least \p minAlign. The
1009/// enclosing function's own non-byval parameter qualifies as well, since its
1010/// slot stands until finalizeParameterSlots and prepareNonByvalParameters has
1011/// restated it with the alignment that parameter promises rather than the one
1012/// CIRGen chose for a local copy. A byval or byref parameter's slot keeps
1013/// CIRGen's alignment, so it qualifies only where that already covers
1014/// \p minAlign.
1015///
1016/// Raising a slot's alignment here would not hold for one that stands in
1017/// for a parameter, since finalizeParameterSlots replaces it with the
1018/// incoming pointer, which promises only what the caller gave it.
1019static mlir::Value forwardableIndirectAddr(cir::LoadOp load,
1020 cir::PointerType paramTy,
1021 uint64_t minAlign) {
1022 if (!load)
1023 return {};
1024 mlir::Value addr = load.getAddr();
1025 // CIRGen reads an alloca outside the default address space through a cast.
1026 if (auto view = addr.getDefiningOp<cir::CastOp>();
1027 view && view.getKind() == cir::CastKind::address_space &&
1028 view.getSrc().getType() == paramTy)
1029 addr = view.getSrc();
1030 if (addr.getType() != paramTy)
1031 return {};
1032 cir::AllocaOp slot = cir::getUnderlyingAlloca(addr);
1033 return slot && slot.getAlignment() >= minAlign ? addr : mlir::Value();
1034}
1035
1036/// True when \p aliasAnalysis shows that nothing between \p load and \p call
1037/// can write the storage \p load read, so a byval argument may name that
1038/// storage rather than a copy of it. byval takes the callee's copy at the
1039/// call, while the operand carries the value as of the load, so a write in
1040/// between would reach the callee that the operand does not hold.
1041/// prepareNonByvalParameters reads a parameter at its spill, which leaves the
1042/// body's own stores in between.
1043///
1044/// A load in another block is not forwardable here, since the paths from it
1045/// to the call are not walked.
1046static bool storageUnwrittenBetween(cir::LoadOp load, mlir::Operation *call,
1047 mlir::AliasAnalysis &aliasAnalysis) {
1048 if (load->getBlock() != call->getBlock())
1049 return false;
1050 for (mlir::Operation *op = load->getNextNode(); op != call;
1051 op = op->getNextNode()) {
1052 assert(op && "the load must precede the call in the block they share");
1053 if (op->getNumRegions() != 0 ||
1054 aliasAnalysis.getModRef(op, location: load.getAddr()).isMod())
1055 return false;
1056 }
1057 return true;
1058}
1059
1060/// The alignment a copy may claim for the storage \p load read, which is the
1061/// one \p load states, or the underlying slot's when it states none. A load
1062/// with neither names storage this pass cannot reason about, so the copy is
1063/// left to assume nothing.
1064static mlir::IntegerAttr loadSourceAlignment(cir::LoadOp load,
1065 mlir::OpBuilder &builder) {
1066 if (mlir::IntegerAttr stated = load.getAlignmentAttr())
1067 return stated;
1068 if (cir::AllocaOp slot = cir::getUnderlyingAlloca(addr: load.getAddr()))
1069 return builder.getI64IntegerAttr(value: slot.getAlignment());
1070 return builder.getI64IntegerAttr(value: 1);
1071}
1072
1073/// Decompose a struct value into one scalar call argument per field of \p
1074/// recTy, appending the field values to \p newArgs. When \p structVal is a
1075/// simple load from an alloca, read each field with cir.get_member +
1076/// cir.load from the address the load used, emitted at the original load's
1077/// position so they observe the same memory state, and record the now-dead
1078/// whole-struct load in \p deadRecordLoads for later erasure. Otherwise (a
1079/// call result, compound literal, or a volatile or atomic load) extract each
1080/// field from the value with cir.extract_member. Loading the members from
1081/// memory rather than extracting from a whole-struct value keeps the result in
1082/// a form SROA can promote (it does not reason about extractvalue). Shared by
1083/// the Expand and Direct+canFlatten argument paths.
1084static void emitStructFieldArgs(mlir::OpBuilder &builder, mlir::Location loc,
1085 mlir::Value structVal, cir::RecordType recTy,
1086 SmallVectorImpl<mlir::Value> &newArgs,
1087 SmallVectorImpl<cir::LoadOp> &deadRecordLoads) {
1088 cir::LoadOp srcLoad = getWholeRecordLoad(recordVal: structVal);
1089
1090 if (srcLoad) {
1091 mlir::OpBuilder::InsertionGuard guard(builder);
1092 builder.setInsertionPoint(srcLoad);
1093 cir::PointerType baseTy = srcLoad.getAddr().getType();
1094 for (auto [f, fieldTy] : llvm::enumerate(First: recTy.getMembers())) {
1095 mlir::Type fieldPtrTy =
1096 cir::PointerType::get(pointee: fieldTy, addrSpace: baseTy.getAddrSpace());
1097 mlir::Value fieldPtr = cir::GetMemberOp::create(
1098 builder, location: loc, type: fieldPtrTy, value: srcLoad.getAddr(), /*name=*/"",
1099 /*index=*/f);
1100 newArgs.push_back(Elt: cir::LoadOp::create(builder, location: loc, addr: fieldPtr));
1101 }
1102 deadRecordLoads.push_back(Elt: srcLoad);
1103 } else {
1104 for (unsigned f = 0; f < recTy.getNumElements(); ++f)
1105 newArgs.push_back(
1106 Elt: cir::ExtractMemberOp::create(builder, location: loc, record: structVal, index: f));
1107 }
1108}
1109
1110/// Erase the loads that a rewritten call left unused. The old call must
1111/// already be erased, since until then it still counts as a user. One load can
1112/// feed two operands of the same call, as in f(s, s), so \p loads can hold the
1113/// same load twice. A load that another op still reads is left alone.
1114static void eraseDeadRecordLoads(ArrayRef<cir::LoadOp> loads) {
1115 llvm::SmallSetVector<mlir::Operation *, 4> uniqueLoads(llvm::from_range,
1116 loads);
1117 for (mlir::Operation *load : uniqueLoads)
1118 if (load->use_empty())
1119 load->erase();
1120}
1121
1122/// The old call, their only user, must already be erased.
1123static void eraseDeadFreezes(ArrayRef<cir::FreezeOp> freezes) {
1124 for (cir::FreezeOp freeze : freezes)
1125 freeze->erase();
1126}
1127
1128/// The store that spills indirect parameter \p blockArg, if the parameter's
1129/// only use is that store. Null otherwise.
1130static cir::StoreOp maybeFindParamSpill(mlir::BlockArgument blockArg) {
1131 if (!blockArg.hasOneUse())
1132 return {};
1133 auto store = dyn_cast<cir::StoreOp>(Val: *blockArg.user_begin());
1134 if (!store || store.getValue() != blockArg)
1135 return {};
1136 return store;
1137}
1138
1139/// The alloca a parameter spill writes to through \p addr, either directly or
1140/// through an address_space cast of it. Null otherwise.
1141static cir::AllocaOp getSpillSlot(mlir::Value addr) {
1142 if (auto view = addr.getDefiningOp<cir::CastOp>();
1143 view && view.getKind() == cir::CastKind::address_space)
1144 addr = view.getSrc();
1145 return addr.getDefiningOp<cir::AllocaOp>();
1146}
1147
1148/// The store that spills non-byval indirect parameter \p blockArg, and the
1149/// slot it spills into. prepareNonByvalParameters has already established
1150/// that the spill is the block argument's only use and that it stores into
1151/// an alloca, which is what the assertions below rest on. Both results are
1152/// null when the parameter has no spill, which is so only when nothing uses
1153/// it at all.
1154static std::pair<cir::StoreOp, cir::AllocaOp>
1155findParamSpill(mlir::BlockArgument blockArg) {
1156 if (blockArg.use_empty())
1157 return {};
1158 cir::StoreOp store = maybeFindParamSpill(blockArg);
1159 assert(store && "non-byval arg's only use must be its CIRGen param spill");
1160 cir::AllocaOp slot = getSpillSlot(addr: store.getAddr());
1161 assert(slot && "non-byval arg's spill must store to an alloca");
1162 return {store, slot};
1163}
1164
1165/// For each Direct arg with a coerced type, change the block argument's type to
1166/// the coerced type and insert a coercion at function entry that maps it back
1167/// to the original type for body uses, which find a record in the coercion slot
1168/// through rewireRecordUses. For each Indirect byval arg and each
1169/// IndirectAliased arg, change the block argument's type to a pointer, then
1170/// replace a record's single param spill with a copy from that pointer.
1171/// Without such a spill, a record's uses find it at the pointer through
1172/// rewireRecordUses, and a value of any other type is loaded from the pointer
1173/// at entry. For each Indirect non-byval arg, change the block argument to a
1174/// pointer and queue the param-slot alloca to be replaced by it (no entry load
1175/// or byte-copy) so the body operates on the caller's storage in place. For
1176/// each Expand arg, replace the single struct block argument with N scalar
1177/// block arguments (one per field) and store each field directly into the
1178/// parameter's own alloca (the CIRGen spill slot), erasing the original
1179/// whole-struct store.
1180///
1181/// \p hasSRetArg is true when the function has an sret return (a hidden return
1182/// pointer is prepended as block argument 0). Expand arguments expand the
1183/// block argument count, so a running index tracks the current block argument
1184/// position rather than computing the classification index + \p hasSRetArg
1185/// directly. Fails, after reporting it, when rewireRecordUses does.
1186mlir::LogicalResult
1187insertArgCoercion(mlir::FunctionOpInterface funcOp,
1188 const FunctionClassification &fc, mlir::OpBuilder &builder,
1189 const mlir::DataLayout &dl, bool hasSRetArg,
1190 SmallVectorImpl<std::pair<cir::AllocaOp, mlir::BlockArgument>>
1191 &pendingParamSlots) {
1192 mlir::Region &body = funcOp->getRegion(index: 0);
1193 if (body.empty())
1194 return mlir::success();
1195 mlir::Block &entry = body.front();
1196
1197 // Running block argument index. Each non-Expand classification occupies
1198 // one block argument slot; each Expand classification occupies N slots
1199 // (one per struct field), so the running index must be incremented by N
1200 // rather than 1 after processing an Expand arg.
1201 unsigned blockArgIdx = hasSRetArg ? 1 : 0;
1202
1203 for (const ArgClassification &ac : fc.argInfos) {
1204 assert(blockArgIdx < entry.getNumArguments() &&
1205 "classification count must not exceed entry block arguments");
1206
1207 if (ac.kind == ArgKind::Expand) {
1208 // The block arg at blockArgIdx currently has the original struct type.
1209 // Replace it with N scalar args (one per field) and store each field
1210 // directly into the parameter's own alloca.
1211 mlir::BlockArgument origArg = entry.getArgument(i: blockArgIdx);
1212 auto recTy = cast<cir::RecordType>(Val: origArg.getType());
1213 assert(recTy.isStruct() &&
1214 "Expand classification requires a struct type, not a union");
1215 unsigned numFields = recTy.getNumElements();
1216 assert(numFields > 0 &&
1217 "Expand classification requires at least one struct field");
1218 mlir::Location loc = funcOp.getLoc();
1219
1220 // CIRGen spills every by-value struct parameter into its local alloca
1221 // with a single store before any other use, so the struct block arg's
1222 // only use is that spill. Capture it and the destination alloca so the
1223 // expanded fields can be stored straight into that alloca, preserving
1224 // the alloca's variable name and `init` flag and avoiding a
1225 // reassemble-then-reload roundtrip. DCE may have run earlier and
1226 // removed the spill (leaving the block arg unused); tolerate that by
1227 // only flattening the signature and emitting no field stores.
1228 cir::StoreOp paramStore;
1229 cir::AllocaOp destAlloca;
1230 if (!origArg.use_empty()) {
1231 assert(origArg.hasOneUse() &&
1232 "Expand arg must have exactly one use (the CIRGen param spill)");
1233 paramStore = cast<cir::StoreOp>(Val: *origArg.user_begin());
1234 assert(paramStore.getValue() == origArg &&
1235 "Expand arg's use must be the value operand of its store");
1236 destAlloca = cast<cir::AllocaOp>(Val: paramStore.getAddr().getDefiningOp());
1237 }
1238
1239 // Erase the original whole-struct spill before retyping the block
1240 // argument, so the store is never left feeding a type-mismatched value.
1241 // The field stores take its place, just before the following operation
1242 // (the spill always precedes the entry block's terminator).
1243 mlir::Operation *fieldStoreInsertPt = nullptr;
1244 if (paramStore) {
1245 fieldStoreInsertPt = paramStore->getNextNode();
1246 assert(fieldStoreInsertPt &&
1247 "param spill must be followed by a block terminator");
1248 paramStore->erase();
1249 }
1250
1251 // Split the single struct block arg into N scalar field block args (slot
1252 // 0 reuses the original; slots 1..N-1 are inserted after it). The
1253 // reshape needs no insertion point. The field stores are gated on the
1254 // same destAlloca condition: when the spill survived we set the insert
1255 // point to its old slot (which sits after the CIRGen allocas) and store
1256 // each field there; when DCE removed the spill the parameter is dead, so
1257 // we only reshape the signature and emit no stores.
1258 if (destAlloca)
1259 builder.setInsertionPoint(fieldStoreInsertPt);
1260 for (auto [f, fieldTy] : llvm::enumerate(First: recTy.getMembers())) {
1261 if (f == 0)
1262 origArg.setType(fieldTy);
1263 else
1264 entry.insertArgument(index: blockArgIdx + f, type: fieldTy, loc);
1265 if (!destAlloca)
1266 continue;
1267 mlir::Type fieldPtrTy = cir::PointerType::get(
1268 pointee: fieldTy,
1269 addrSpace: mlir::cast<cir::PointerType>(Val: destAlloca.getType()).getAddrSpace());
1270 auto fieldPtr = cir::GetMemberOp::create(builder, location: loc, type: fieldPtrTy,
1271 value: destAlloca, /*name=*/"",
1272 /*index=*/f);
1273 cir::StoreOp::create(builder, location: loc, value: entry.getArgument(i: blockArgIdx + f),
1274 addr: fieldPtr);
1275 }
1276
1277 blockArgIdx += numFields;
1278 continue;
1279 }
1280
1281 mlir::BlockArgument blockArg = entry.getArgument(i: blockArgIdx);
1282
1283 if (cir::RecordType flatTy = getFlattenedCoercedType(ac, dl)) {
1284 // Direct + canFlatten: the coerced type is a struct whose fields are
1285 // passed as separate arguments. As in the Expand path, the single block
1286 // arg becomes N scalar block args, and each is stored straight into a
1287 // slot sized and aligned for both the coerced struct and the original
1288 // argument type, which the body then reads.
1289 unsigned numFields = flatTy.getNumElements();
1290 Type origTy = blockArg.getType();
1291 Location loc = funcOp.getLoc();
1292
1293 // Change slot 0 to field 0's type; insert slots 1..N-1 after it.
1294 blockArg.setType(flatTy.getElementType(idx: 0));
1295 for (unsigned f = 1; f < numFields; ++f)
1296 entry.insertArgument(index: blockArgIdx + f, type: flatTy.getElementType(idx: f), loc);
1297
1298 // setInsertionPointToStart: see comment in the Expand arm above.
1299 builder.setInsertionPointToStart(&entry);
1300 assert(!ac.directOffset &&
1301 "the coerced struct starts at slot offset 0 here, so a flattened "
1302 "coercion cannot honor a direct offset");
1303 SmallPtrSet<Operation *, 8> flattenOps;
1304 cir::AllocaOp slot;
1305 if (origTy == flatTy) {
1306 slot = cir::AllocaOp::create(
1307 builder, location: loc, addr: getAllocaPtrTy(pointee: flatTy, dl),
1308 name: builder.getStringAttr(bytes: "coerce"),
1309 alignment: builder.getI64IntegerAttr(
1310 value: cir::CIRDataLayout(funcOp->getParentOfType<mlir::ModuleOp>())
1311 .getTypeLanguageAlignment(ty: flatTy)));
1312 flattenOps.insert(Ptr: slot);
1313 } else {
1314 slot = createCoercionSlot(builder, loc, srcTy: flatTy, dstTy: origTy, slotBlock: &entry, dl,
1315 createdOps&: flattenOps, /*offset=*/0);
1316 }
1317 Value flatView = emitViewAtOffset(builder, loc, wantTy: flatTy, base: slot, createdOps&: flattenOps,
1318 /*offset=*/0);
1319 auto flatPtrTy = cast<cir::PointerType>(Val: flatView.getType());
1320 for (auto [f, fieldTy] : llvm::enumerate(First: flatTy.getMembers())) {
1321 Type fieldPtrTy =
1322 cir::PointerType::get(pointee: fieldTy, addrSpace: flatPtrTy.getAddrSpace());
1323 auto fieldPtr = cir::GetMemberOp::create(builder, location: loc, type: fieldPtrTy,
1324 value: flatView, /*name=*/"",
1325 /*index=*/f);
1326 flattenOps.insert(Ptr: fieldPtr);
1327 auto storeOp = cir::StoreOp::create(
1328 builder, location: loc, value: entry.getArgument(i: blockArgIdx + f), addr: fieldPtr);
1329 // A packed slot can be less aligned than the field's own type.
1330 uint64_t fieldAlign =
1331 llvm::MinAlign(A: slot.getAlignment(), B: flatTy.getElementOffset(dataLayout: dl, idx: f));
1332 if (fieldAlign < dl.getTypeABIAlignment(t: fieldTy))
1333 storeOp.setAlignment(fieldAlign);
1334 flattenOps.insert(Ptr: storeOp);
1335 }
1336
1337 // Point the original body uses of the struct block arg, now field 0, at
1338 // the original-type view of the slot.
1339 Value origView = emitViewAtOffset(builder, loc, wantTy: origTy, base: slot, createdOps&: flattenOps,
1340 /*offset=*/0);
1341 if (failed(Result: replaceUsesFromSlot(builder, loc, val: blockArg, view: origView, dl,
1342 exceptions: flattenOps)))
1343 return mlir::failure();
1344
1345 blockArgIdx += numFields;
1346 continue;
1347 }
1348
1349 if ((ac.kind == ArgKind::Direct && ac.coercedType) ||
1350 isRecordExtend(ac, origTy: blockArg.getType())) {
1351 mlir::Type oldArgTy = blockArg.getType();
1352 mlir::Type newArgTy = ac.coercedType;
1353 if (oldArgTy == newArgTy) {
1354 ++blockArgIdx;
1355 continue;
1356 }
1357 blockArg.setType(newArgTy);
1358
1359 // The coercion's own operations are left out of the rewiring, since they
1360 // read blockArg as the coerced type.
1361 builder.setInsertionPointToStart(&entry);
1362 SmallPtrSet<mlir::Operation *, 4> coercionOps;
1363 if (failed(Result: emitCoercionAndReplaceUses(builder, loc: funcOp.getLoc(), origTy: oldArgTy,
1364 src: blockArg, val: blockArg, slotBlock: &entry, dl,
1365 createdOps&: coercionOps, offset: ac.directOffset)))
1366 return mlir::failure();
1367 } else if (ac.kind == ArgKind::Indirect ||
1368 ac.kind == ArgKind::IndirectAliased) {
1369 // byval and non-byval both lower to a pointer to T in the classified
1370 // address space, and which it is shows up only in the attrs
1371 // updateArgAttrs applies. Body lowering differs: the body addresses a
1372 // byval parameter through the CIRGen slot it was spilled into, so that
1373 // slot has to be filled from the incoming pointer, while a non-byval
1374 // parameter's slot is replaced by the pointer so the body works on the
1375 // caller's storage in place. A byref parameter takes the byval path.
1376 bool byRef = ac.kind == ArgKind::IndirectAliased;
1377 cir::PointerType ptrTy = getIndirectPtrTy(pointee: blockArg.getType(), ac);
1378
1379 if (!byRef && !ac.byVal) {
1380 // Without byval, drop the spill store and let the slot's uses read the
1381 // incoming pointer, so the body operates on the caller's storage in
1382 // place. A byte-copy would be wrong for non-trivially-copyable
1383 // aggregates (e.g. libstdc++ SSO std::string, where it would leave
1384 // `_M_p` aliasing the source's `_M_local_buf`).
1385 auto [paramStore, destAlloca] = findParamSpill(blockArg);
1386
1387 if (paramStore)
1388 paramStore->erase();
1389
1390 // Update the block argument to point to its original type.
1391 blockArg.setType(ptrTy);
1392
1393 // Pointing the slot's uses at the incoming pointer waits until every
1394 // call site has been rewritten. A call that hands this parameter
1395 // straight on recognizes it by the slot its operand was loaded from,
1396 // and collapsing the slot here would leave that call reading a block
1397 // argument with no defining operation to inspect. A dead spill DCE
1398 // already removed leaves nothing to collapse.
1399 if (destAlloca)
1400 pendingParamSlots.emplace_back(Args&: destAlloca, Args&: blockArg);
1401 } else {
1402 // A record's spill slot is filled by a byte copy of the incoming
1403 // object rather than by a store of the loaded value: its padding
1404 // bytes can hold another union member's live data, which a value
1405 // store would leave unwritten. A _BitInt keeps its load and spill
1406 // store, which are its conversion between value and in-memory form.
1407 cir::StoreOp paramStore;
1408 if (mlir::isa<cir::RecordType>(Val: blockArg.getType()))
1409 paramStore = maybeFindParamSpill(blockArg);
1410
1411 // Erasing the spill before the block argument is retyped keeps the
1412 // store from being left with a value operand of the wrong type.
1413 mlir::Value destAddr;
1414 mlir::IntegerAttr destAlign;
1415 mlir::Operation *spillPos = nullptr;
1416 bool spillIsVolatile = false;
1417 if (paramStore) {
1418 if (paramStore.getMemOrder())
1419 return paramStore.emitOpError()
1420 << "atomic store of a record argument or result not yet "
1421 "implemented in CallConvLowering";
1422 spillIsVolatile = paramStore.getIsVolatile();
1423 destAddr = paramStore.getAddr();
1424 cir::AllocaOp destAlloca = cir::getUnderlyingAlloca(addr: destAddr);
1425 destAlign = paramStore.getAlignmentAttr();
1426 if (!destAlign && destAlloca)
1427 destAlign = destAlloca.getAlignmentAttr();
1428 spillPos = paramStore->getNextNode();
1429 assert(spillPos && "param spill must be followed by a block "
1430 "terminator");
1431 paramStore->erase();
1432 }
1433
1434 blockArg.setType(ptrTy);
1435
1436 if (destAddr) {
1437 mlir::OpBuilder::InsertionGuard guard(builder);
1438 builder.setInsertionPoint(spillPos);
1439 auto copy = cir::CopyOp::create(
1440 builder, location: funcOp.getLoc(), dst: destAddr, src: blockArg, dst_alignment: destAlign,
1441 src_alignment: builder.getI64IntegerAttr(value: ac.indirectAlign.value()));
1442 copy.setIsVolatile(spillIsVolatile);
1443 copy.setSkipTailPadding(
1444 shouldSkipTailPadding(dst: destAddr, ty: ptrTy.getPointee()));
1445 } else if (!blockArg.use_empty()) {
1446 builder.setInsertionPointToStart(&entry);
1447 if (mlir::isa<cir::RecordType>(Val: ptrTy.getPointee())) {
1448 SmallPtrSet<mlir::Operation *, 1> noExceptions;
1449 if (failed(Result: rewireRecordUses(builder, loc: funcOp.getLoc(), val: blockArg,
1450 addr: blockArg, addrAlign: ac.indirectAlign.value(), dl,
1451 exceptions: noExceptions)))
1452 return mlir::failure();
1453 } else {
1454 auto loadOp =
1455 cir::LoadOp::create(builder, location: funcOp.getLoc(), addr: blockArg);
1456 SmallPtrSet<mlir::Operation *, 1> loadOps = {loadOp};
1457 blockArg.replaceAllUsesExcept(newValue: loadOp.getResult(), exceptions: loadOps);
1458 }
1459 }
1460 }
1461 }
1462 // Ignore, Extend, and Direct-without-coerce need no block-level changes.
1463
1464 ++blockArgIdx;
1465 }
1466 return mlir::success();
1467}
1468
1469/// Replace each address_space cast of \p slot to the type of \p ptr with
1470/// \p ptr.
1471void replaceAddrSpaceViews(cir::AllocaOp slot, mlir::Value ptr) {
1472 for (mlir::Operation *user : llvm::make_early_inc_range(Range: slot->getUsers())) {
1473 auto view = mlir::dyn_cast<cir::CastOp>(Val: user);
1474 if (view && view.getKind() == cir::CastKind::address_space &&
1475 view.getType() == ptr.getType()) {
1476 view.getResult().replaceAllUsesWith(newValue: ptr);
1477 view->erase();
1478 }
1479 }
1480}
1481
1482/// Rewrite each cir.return so the return value flows through the sret
1483/// pointer (the prepended first block argument) and the function returns
1484/// void.
1485///
1486/// CIRGen emits a local `__retval` alloca and emits `cir.return %loaded`
1487/// where `%loaded = cir.load __retval`. The naive lowering -- store the
1488/// loaded SSA value through the sret pointer -- byte-copies the record,
1489/// which is wrong for non-trivially-copyable types: e.g. libstdc++'s SSO
1490/// `std::string` has a `_M_p` pointer that aliases the source's internal
1491/// `_M_local_buf`, so a byte-copy leaves the destination pointing at the
1492/// source's (now-dying) stack storage and the destination's destructor
1493/// later `free()`s a stack pointer.
1494///
1495/// Instead, route construction directly into the sret slot: find the
1496/// `__retval` alloca, replace its uses with the sret pointer, and drop the
1497/// trailing `cir.load __retval` so the rewritten return has no operand.
1498/// The CIRGen-emitted constructor / store-into-`__retval` then targets the
1499/// sret slot uniformly, matching classic CodeGen's "construct directly into
1500/// `%agg.result`" pattern.
1501///
1502/// CIRGen emits one `%v = cir.load %__retval` / `cir.return %v` pair per
1503/// return statement, and every such load reads the single `__retval`
1504/// alloca (CIR does not merge returns into a shared epilogue block). The
1505/// alloca is therefore rewired to the sret pointer once; each cir.return is
1506/// then collapsed to a bare return and its now-dead load erased. This
1507/// `cir.return (cir.load <alloca>)` shape is an invariant guaranteed by
1508/// CIRGen, so it is asserted via `cast<>` rather than guarded with a
1509/// fallback.
1510///
1511/// A `__retval` outside the sret pointer's address space is not replaced
1512/// itself. Its address_space casts to the sret pointer's type are.
1513void insertSRetStores(mlir::FunctionOpInterface funcOp, mlir::Type origRetTy,
1514 mlir::OpBuilder &builder) {
1515 mlir::Value sretPtr = funcOp.getArguments()[0];
1516
1517 SmallVector<cir::ReturnOp> returnOps;
1518 funcOp->walk(callback: [&](cir::ReturnOp retOp) { returnOps.push_back(Elt: retOp); });
1519
1520 cir::AllocaOp retAlloca = nullptr;
1521 for (cir::ReturnOp retOp : returnOps) {
1522 // Every cir.return in an sret function must carry the loaded return
1523 // value -- a bare return would mean the sret slot was never written.
1524 assert(!retOp.getInput().empty() &&
1525 "cir.return in sret function must have an operand");
1526
1527 cir::LoadOp retLoad =
1528 mlir::cast<cir::LoadOp>(Val: retOp.getInput()[0].getDefiningOp());
1529
1530 // Rewire the shared `__retval` alloca to the sret pointer once.
1531 // replaceAllUsesWith updates every load of the alloca (including those
1532 // feeding the other cir.return ops) to read from sretPtr instead, so
1533 // all returns are covered by this single rewiring.
1534 if (!retAlloca) {
1535 retAlloca = mlir::cast<cir::AllocaOp>(Val: retLoad.getAddr().getDefiningOp());
1536 if (retAlloca.getType() == sretPtr.getType())
1537 retAlloca.getResult().replaceAllUsesWith(newValue: sretPtr);
1538 else
1539 replaceAddrSpaceViews(slot: retAlloca, ptr: sretPtr);
1540 }
1541
1542 // The sret slot now holds the return value directly; replace the
1543 // value-carrying return with a void return (no operand).
1544 builder.setInsertionPoint(retOp);
1545 cir::ReturnOp::create(builder, location: retOp.getLoc());
1546 retOp->erase();
1547 if (retLoad.use_empty())
1548 retLoad->erase();
1549 }
1550
1551 if (retAlloca) {
1552 assert(retAlloca->use_empty() &&
1553 "__retval used other than by its returns and address_space views");
1554 retAlloca->erase();
1555 }
1556}
1557
1558/// Build the attribute dictionary for the sret slot (slot 0 of an
1559/// sret-returning function or call). Matches classic CodeGen's
1560/// `sret(T) align A [noalias] writable dead_on_unwind`. noalias is only
1561/// valid on the callee's parameter, not at the call site, so it is gated by
1562/// \p withNoalias. Key order is irrelevant: DictionaryAttr sorts by name.
1563SmallVector<mlir::NamedAttribute> buildSretSlotAttrs(mlir::OpBuilder &builder,
1564 mlir::Type retTy,
1565 uint64_t align,
1566 bool withNoalias) {
1567 SmallVector<mlir::NamedAttribute> attrs;
1568 // The sret type must be carried explicitly: LLVM's sret attribute requires
1569 // it, and once the CIR `!cir.ptr<retTy>` lowers to an opaque LLVM `ptr` the
1570 // pointee type can no longer be recovered from the pointer.
1571 attrs.push_back(
1572 Elt: builder.getNamedAttr(name: "llvm.sret", val: mlir::TypeAttr::get(type: retTy)));
1573 attrs.push_back(
1574 Elt: builder.getNamedAttr(name: "llvm.align", val: builder.getI64IntegerAttr(value: align)));
1575 if (withNoalias)
1576 attrs.push_back(
1577 Elt: builder.getNamedAttr(name: "llvm.noalias", val: builder.getUnitAttr()));
1578 attrs.push_back(Elt: builder.getNamedAttr(name: "llvm.writable", val: builder.getUnitAttr()));
1579 attrs.push_back(
1580 Elt: builder.getNamedAttr(name: "llvm.dead_on_unwind", val: builder.getUnitAttr()));
1581 return attrs;
1582}
1583
1584/// Prepend the sret slot's attrs at position 0 of newCall's arg_attrs.
1585/// Called after the call has been rewritten with the sret pointer at
1586/// operand 0, so the operand count now includes the sret slot. \p argAttrs
1587/// must already be shaped for the rewritten argument list (Extend slots
1588/// carry signext/zeroext, Ignore slots dropped); it is shifted to slots
1589/// 1..N behind the sret slot.
1590void applySretSlotAttrs(cir::CallOp newCall, mlir::ArrayAttr argAttrs,
1591 mlir::Type retTy, uint64_t align,
1592 mlir::OpBuilder &builder) {
1593 mlir::MLIRContext *ctx = newCall->getContext();
1594 SmallVector<mlir::NamedAttribute> sretAttrs =
1595 buildSretSlotAttrs(builder, retTy, align, /*withNoalias=*/false);
1596
1597 SmallVector<mlir::Attribute> newArgAttrs;
1598 newArgAttrs.reserve(N: newCall.getArgOperands().size());
1599 newArgAttrs.push_back(Elt: mlir::DictionaryAttr::get(context: ctx, value: sretAttrs));
1600 if (argAttrs)
1601 llvm::append_range(C&: newArgAttrs, R&: argAttrs);
1602 assert(newArgAttrs.size() <= newCall.getArgOperands().size() &&
1603 "arg_attrs wider than the rewritten call's operand list");
1604 newArgAttrs.resize(N: newCall.getArgOperands().size(),
1605 NV: mlir::DictionaryAttr::get(context: ctx));
1606 newCall->setAttr(name: "arg_attrs", value: mlir::ArrayAttr::get(context: ctx, value: newArgAttrs));
1607}
1608
1609/// Copy the call attributes from \p source to the rebuilt \p target. The
1610/// callee is already established by CallOp::create and may have been rewritten,
1611/// so it is not copied. Existing discardable attributes on \p target take
1612/// precedence over attributes from \p source.
1613static void copyCallAttributes(cir::CallOp source, cir::CallOp target) {
1614 source->getName().walkInherentAttrs(op: source, visitor: [&](llvm::StringRef name,
1615 mlir::Attribute &attr) {
1616 if (name != cir::CIRDialect::getCalleeAttrName())
1617 target->setInherentAttr(name: mlir::StringAttr::get(context: source->getContext(), bytes: name),
1618 value: attr);
1619 });
1620 for (mlir::NamedAttribute attr : source->getDiscardableAttrs())
1621 if (!target->hasDiscardableAttr(name: attr.getName()))
1622 target->setDiscardableAttr(name: attr.getName(), value: attr.getValue());
1623}
1624
1625/// For an indirect call, prepend the callee function pointer as operand 0 so
1626/// CallOp::create rebuilds it as an indirect call, bitcasting it to a function
1627/// pointer that takes \p args, less the last \p numEllipsisArgs, and returns
1628/// \p retTy. Those last arguments were passed through the callee's ellipsis
1629/// and stay out of the function type, as in classic CodeGen's
1630/// GetFunctionType: LLVM treats operands past a call's function type as
1631/// variadic, and some backends pass variadic arguments differently. No-op
1632/// for direct calls.
1633static void prependIndirectCallee(cir::CallOp call,
1634 SmallVectorImpl<mlir::Value> &args,
1635 mlir::Type retTy, unsigned numEllipsisArgs,
1636 mlir::OpBuilder &builder) {
1637 if (!call.isIndirect())
1638 return;
1639 assert(numEllipsisArgs <= args.size() &&
1640 "more ellipsis arguments than arguments");
1641 mlir::Value calleePtr = call.getIndirectCall();
1642 SmallVector<mlir::Type> paramTypes;
1643 paramTypes.reserve(N: args.size() - numEllipsisArgs);
1644 llvm::transform(Range: ArrayRef(args).drop_back(N: numEllipsisArgs),
1645 d_first: std::back_inserter(x&: paramTypes),
1646 F: [](mlir::Value v) { return v.getType(); });
1647 // Lowering builds an indirect call's LLVM function type from the callee
1648 // pointer's pointee and takes the call's result from that type, so the
1649 // pointee's return type has to track the rewrite: an sret return would
1650 // leave a result the call no longer produces, and a coerced return one of
1651 // the wrong type. The ellipsis has to survive for the same reason: the
1652 // rebuilt pointee is what makes the lowered call variadic, and only a
1653 // variadic call gets the vector-register count that the x86_64 SysV ABI
1654 // passes in AL and that the callee's va_arg reads back.
1655 cir::FuncType calleeFnTy = getIndirectCalleeType(call);
1656 assert((calleeFnTy.isVarArg() || numEllipsisArgs == 0) &&
1657 "only a variadic callee takes arguments through an ellipsis");
1658 auto newPtrTy = cir::PointerType::get(
1659 pointee: cir::FuncType::get(inputs: paramTypes, returnType: retTy, isVarArg: calleeFnTy.isVarArg()));
1660 if (calleePtr.getType() != newPtrTy)
1661 calleePtr = cir::CastOp::create(builder, location: call.getLoc(), result: newPtrTy,
1662 kind: cir::CastKind::bitcast, src: calleePtr);
1663 args.insert(I: args.begin(), Elt: calleePtr);
1664}
1665
1666/// Rewrite an indirect-return (sret) call site: prepend a return-slot pointer
1667/// as operand 0, make the call return void, and either reuse a dominating
1668/// single-use store destination as the slot (so construction flows directly
1669/// into it) or allocate a fresh slot and point the result's uses at it, a
1670/// record's through rewireRecordUses and any other type's through a load of the
1671/// slot after the call. \p newArgs is the already-shaped (Ignore-dropped,
1672/// coercion-applied) non-sret argument list, the last \p numEllipsisArgs of
1673/// which were passed through the callee's ellipsis. The caller guarantees the
1674/// call has a result and an indirect-return classification. Fails, after
1675/// reporting it, when rewireRecordUses does.
1676mlir::LogicalResult
1677rewriteIndirectReturnCall(cir::CallOp call, const FunctionClassification &fc,
1678 ArrayRef<mlir::Value> newArgs,
1679 unsigned numEllipsisArgs, mlir::Type origRetTy,
1680 ArrayRef<mlir::Type> origCallArgTypes,
1681 mlir::OpBuilder &builder, const mlir::DataLayout &dl,
1682 bool enableNoundefAttrs) {
1683 mlir::MLIRContext *ctx = call->getContext();
1684 cir::PointerType ptrTy = getIndirectPtrTy(pointee: origRetTy, ac: fc.returnInfo);
1685 builder.setInsertionPoint(call);
1686 uint64_t sretAlign = fc.returnInfo.indirectAlign.value();
1687
1688 // CIRGen emits `cir.store %callResult, %dest` when the call's result is
1689 // bound to a local (e.g. `T s = make();`). Allocating a fresh sret slot
1690 // and copying into %dest would byte-copy the record, which is wrong for
1691 // non-trivially-copyable types (the libstdc++ SSO `_M_p` pointer
1692 // survives a byte-copy but ends up pointing at the dying temp's local
1693 // buffer, so the destination's destructor later `free()`s a stack
1694 // pointer). When the result has a single store-into-%dest use, use
1695 // %dest as the sret slot directly so construction flows into it,
1696 // matching classic CodeGen's "pass %s as sret" pattern. %dest must
1697 // dominate the call so the rewritten call (which takes it as operand 0)
1698 // does not use a value before its definition. An address_space cast of a
1699 // slot that already has the sret pointer's type stands for that slot.
1700 mlir::Value sretSlot = nullptr;
1701 cir::StoreOp reuseStore = nullptr;
1702 if (call.getResult().hasOneUse()) {
1703 mlir::Operation *user = *call.getResult().getUsers().begin();
1704 auto store = mlir::dyn_cast<cir::StoreOp>(Val: user);
1705 if (store && store.getValue() == call.getResult()) {
1706 mlir::Value dest = store.getAddr();
1707 if (auto view = dest.getDefiningOp<cir::CastOp>();
1708 view && view.getKind() == cir::CastKind::address_space &&
1709 view.getSrc().getType() == ptrTy)
1710 dest = view.getSrc();
1711 if (dest.getType() == ptrTy &&
1712 mlir::DominanceInfo().properlyDominates(a: dest, b: call)) {
1713 sretSlot = dest;
1714 reuseStore = store;
1715 }
1716 }
1717 }
1718 if (!sretSlot) {
1719 cir::PointerType allocaPtrTy = getAllocaPtrTy(pointee: origRetTy, dl);
1720 sretSlot = cir::AllocaOp::create(
1721 builder, location: call.getLoc(), addr: allocaPtrTy,
1722 /*name=*/builder.getStringAttr(bytes: "sret"),
1723 /*alignment=*/builder.getI64IntegerAttr(value: sretAlign));
1724 // The sret pointer can be outside the alloca address space.
1725 if (allocaPtrTy != ptrTy)
1726 sretSlot = cir::CastOp::create(builder, location: call.getLoc(), result: ptrTy,
1727 kind: cir::CastKind::address_space, src: sretSlot);
1728 }
1729
1730 SmallVector<mlir::Value> sretArgs;
1731 sretArgs.push_back(Elt: sretSlot);
1732 sretArgs.append(in_start: newArgs.begin(), in_end: newArgs.end());
1733
1734 // numEllipsisArgs counts from the end, so the sret slot prepended here stays
1735 // in the retyped signature, as it does in classic CodeGen.
1736 mlir::Type sretVoidTy = cir::VoidType::get(ctx);
1737 prependIndirectCallee(call, args&: sretArgs, retTy: sretVoidTy, numEllipsisArgs, builder);
1738 auto newCall = cir::CallOp::create(
1739 builder, location: call.getLoc(), callee: call.getCalleeAttr(), resType: sretVoidTy, operands: sretArgs);
1740 copyCallAttributes(source: call, target: newCall);
1741 newCall->removeAttr(name: "res_attrs");
1742
1743 // Shape the per-argument attrs exactly as the non-sret path does, through
1744 // updateArgAttrs, before prepending the sret slot, so sret composes with
1745 // every argument classification.
1746 mlir::ArrayAttr argAttrs = call->getAttrOfType<mlir::ArrayAttr>(name: "arg_attrs");
1747 if (needsArgAttrUpdate(fc))
1748 argAttrs =
1749 updateArgAttrs(ctx, module: call->getParentOfType<mlir::ModuleOp>(),
1750 origArgTypes: origCallArgTypes, existingArgAttrs: argAttrs, fc, dl, enableNoundefAttrs,
1751 /*isCallSite=*/true);
1752 applySretSlotAttrs(newCall, argAttrs, retTy: origRetTy, align: sretAlign, builder);
1753
1754 if (reuseStore) {
1755 // The callee now constructs directly into the destination slot, so the
1756 // original store-from-result is redundant; dropping it avoids a
1757 // byte-copy of the record.
1758 reuseStore->erase();
1759 } else if (mlir::isa<cir::RecordType>(Val: origRetTy)) {
1760 builder.setInsertionPointAfter(newCall);
1761 SmallPtrSet<mlir::Operation *, 1> noExceptions;
1762 if (failed(Result: rewireRecordUses(builder, loc: call.getLoc(), val: call.getResult(),
1763 addr: sretSlot, addrAlign: sretAlign, dl, exceptions: noExceptions)))
1764 return mlir::failure();
1765 } else {
1766 builder.setInsertionPointAfter(newCall);
1767 auto load = cir::LoadOp::create(builder, location: call.getLoc(), result: origRetTy, addr: sretSlot,
1768 /*isDeref=*/mlir::UnitAttr(),
1769 /*isVolatile=*/is_volatile: mlir::UnitAttr(),
1770 /*is_nontemporal=*/mlir::UnitAttr(),
1771 /*alignment=*/mlir::IntegerAttr(),
1772 /*sync_scope=*/cir::SyncScopeKindAttr(),
1773 /*mem_order=*/cir::MemOrderAttr(),
1774 /*invariant=*/mlir::UnitAttr());
1775 call.getResult().replaceAllUsesWith(newValue: load);
1776 }
1777 call->erase();
1778 return mlir::success();
1779}
1780
1781/// Whether \p ty, a type the classifier named for one register of a
1782/// coercion, is carried in a vector register.
1783bool isSSERegisterClass(mlir::Type ty) {
1784 return mlir::isa<cir::VectorType, cir::FPTypeInterface>(Val: ty);
1785}
1786
1787} // namespace
1788
1789cir::FuncType cir::getIndirectCalleeType(cir::CIRCallOpInterface call) {
1790 assert(call.isIndirect() && "expected an indirect call");
1791 return cast<cir::FuncType>(
1792 Val: cast<cir::PointerType>(Val: call.getIndirectCall().getType()).getPointee());
1793}
1794
1795/// Bring \p funcOp's non-byval indirect parameter \p argNo into the shape the
1796/// rest of the rewrite assumes. \p claimedSlots carries the slots \p funcOp's
1797/// earlier non-byval indirect parameters took. The shape itself, and the
1798/// cases reported rather than repaired, are documented on
1799/// CIRABIRewriteContext::prepareNonByvalParameters in the header.
1800static mlir::LogicalResult prepareOneNonByvalParameter(
1801 cir::FuncOp funcOp, unsigned argNo, const ArgClassification &ac,
1802 mlir::BlockArgument blockArg, mlir::DominanceInfo &dom,
1803 SmallPtrSetImpl<mlir::Operation *> &claimedSlots) {
1804 cir::PointerType paramPtrTy = getIndirectPtrTy(pointee: blockArg.getType(), ac);
1805
1806 // The spill is the store that writes the parameter itself. Any other use
1807 // consumes the record value. At -O1 and above such a use comes from
1808 // cir-simplify: CIRGen marks a const-qualified parameter's slot const, so
1809 // the load of it folds to the stored parameter.
1810 cir::StoreOp spill;
1811 cir::StoreOp extraSpill;
1812 mlir::Operation *otherUse = nullptr;
1813 SmallVector<mlir::OpOperand *> callArgs;
1814 for (mlir::OpOperand &use : blockArg.getUses()) {
1815 auto store = dyn_cast<cir::StoreOp>(Val: use.getOwner());
1816 if (store && store.getValue() == blockArg) {
1817 // Which of two stores is the spill decides which slot stands in for the
1818 // parameter, and the use list is in no particular order, so there is
1819 // nothing to prefer between them.
1820 if (spill)
1821 extraSpill = store;
1822 else
1823 spill = store;
1824 continue;
1825 }
1826 // Only an argument is served by a load of the slot. Any other consumer
1827 // belongs to a rewrite that reads the parameter its own way: a returned
1828 // record, for one, is rewritten through the sret slot, which assumes the
1829 // returned load names the return slot and not this one.
1830 auto call = dyn_cast<cir::CIRCallOpInterface>(Val: use.getOwner());
1831 if (call && llvm::is_contained(Range: call.getArgOperands(), Element: blockArg))
1832 callArgs.push_back(Elt: &use);
1833 else if (!otherUse)
1834 otherUse = use.getOwner();
1835 }
1836
1837 if (extraSpill)
1838 return extraSpill->emitOpError()
1839 << "non-byval parameter " << argNo
1840 << " spilled more than once is not yet implemented in "
1841 "CallConvLowering";
1842
1843 if (otherUse)
1844 return otherUse->emitOpError()
1845 << "non-byval parameter " << argNo
1846 << " consumed other than as a call argument is not yet implemented "
1847 "in CallConvLowering";
1848
1849 if (!spill) {
1850 // Every other kind of use was reported above, so the parameter has no
1851 // uses at all and needs neither a slot nor a read.
1852 if (callArgs.empty())
1853 return mlir::success();
1854
1855 // Without a spill the parameter becomes the incoming pointer directly,
1856 // which is the storage an argument taken from it has to name. Giving it
1857 // the spill it lacks lets the checks and the read below apply unchanged,
1858 // and neither survives the pass: insertArgCoercion erases the store and
1859 // finalizeParameterSlots replaces the slot.
1860 mlir::OpBuilder builder(funcOp.getContext());
1861 builder.setInsertionPointToStart(blockArg.getOwner());
1862 auto synthesized = cir::AllocaOp::create(
1863 builder, location: funcOp.getLoc(), addr: paramPtrTy,
1864 name: builder.getStringAttr(bytes: "nonbyval.param"),
1865 alignment: builder.getI64IntegerAttr(value: ac.indirectAlign.value()));
1866 spill =
1867 cir::StoreOp::create(builder, location: funcOp.getLoc(), value: blockArg, addr: synthesized);
1868 }
1869
1870 // The incoming pointer replaces the slot itself, which also covers a spill
1871 // through an address_space cast of it. Retargeting only the cast would
1872 // leave the allocation's other views reading storage nothing writes.
1873 // Storage in another address space than the incoming pointer, or storage
1874 // that is not a local alloca at all, is not something the incoming pointer
1875 // can be substituted for.
1876 cir::AllocaOp slot = getSpillSlot(addr: spill.getAddr());
1877 if (!slot || slot.getType() != paramPtrTy)
1878 return spill->emitOpError()
1879 << "non-byval parameter " << argNo
1880 << " spilled to storage that cannot take the incoming pointer is "
1881 "not yet implemented in CallConvLowering";
1882
1883 // One slot stands in for one non-byval parameter, since the incoming
1884 // pointer replaces it. Two of them spilled to the same slot are each
1885 // spilled once, so the check above cannot see the collision and only
1886 // comparing the slots can.
1887 if (!claimedSlots.insert(Ptr: slot).second)
1888 return spill->emitOpError()
1889 << "non-byval parameter " << argNo
1890 << " sharing its spill slot with another parameter is not yet "
1891 "implemented in CallConvLowering";
1892
1893 // CIRGen picked the slot's alignment for a local copy of the record, but the
1894 // slot is about to stand in for the parameter, and a call forwarding it may
1895 // only promise what the incoming pointer does.
1896 slot.setAlignment(ac.indirectAlign.value());
1897
1898 if (callArgs.empty())
1899 return mlir::success();
1900
1901 // A reader the spill does not dominate could read the slot before anything
1902 // wrote it, and a load placed at the spill would not dominate it either.
1903 for (mlir::OpOperand *callArg : callArgs)
1904 if (!dom.properlyDominates(a: spill, b: callArg->getOwner()))
1905 return callArg->getOwner()->emitOpError()
1906 << "non-byval parameter " << argNo
1907 << " read before its spill is not yet implemented in "
1908 "CallConvLowering";
1909
1910 // Read the record back out of the slot instead, so every reader sees the
1911 // shape CIRGen emits without cir-simplify: a load naming the storage the
1912 // argument would have to name anyway. Reading at the spill, not at the
1913 // reader, keeps the value the parameter's own whatever the body later
1914 // stores into the slot, and the load promises only the alignment the
1915 // classification gives the incoming pointer.
1916 //
1917 // A call that forwards the argument consumes the load once it recognizes
1918 // the slot. One that wants a copy keeps it, reading the incoming pointer
1919 // once finalizeParameterSlots replaces the slot.
1920 mlir::OpBuilder builder(spill);
1921 builder.setInsertionPointAfter(spill);
1922 auto reload = cir::LoadOp::create(builder, location: spill.getLoc(), addr: slot);
1923 reload.setAlignment(ac.indirectAlign.value());
1924 for (mlir::OpOperand *callArg : callArgs)
1925 callArg->set(reload.getResult());
1926 return mlir::success();
1927}
1928
1929mlir::LogicalResult CIRABIRewriteContext::prepareNonByvalParameters(
1930 cir::FuncOp funcOp, const FunctionClassification &fc) {
1931 if (!funcOp.isDefinition())
1932 return mlir::success();
1933 mlir::Region &body = funcOp->getRegion(index: 0);
1934 if (body.empty())
1935 return mlir::success();
1936 mlir::Block &entry = body.front();
1937 mlir::DominanceInfo dom;
1938 SmallPtrSet<mlir::Operation *, 4> claimedSlots;
1939
1940 // No signature has been rewritten yet, so no sret pointer has been prepended
1941 // and no Expand argument has been split into its fields. Every
1942 // classification therefore still maps to the entry block argument at its own
1943 // index.
1944 for (auto [idx, ac] : llvm::enumerate(First: fc.argInfos)) {
1945 if (ac.kind != ArgKind::Indirect || ac.byVal)
1946 continue;
1947 assert(idx < entry.getNumArguments() &&
1948 "classification count must not exceed entry block arguments");
1949 if (failed(Result: prepareOneNonByvalParameter(
1950 funcOp, argNo: idx, ac, blockArg: entry.getArgument(i: idx), dom, claimedSlots)))
1951 return mlir::failure();
1952 }
1953 return mlir::success();
1954}
1955
1956void CIRABIRewriteContext::finalizeParameterSlots() {
1957 for (auto [slot, incoming] : pendingParamSlots) {
1958 slot.getResult().replaceAllUsesWith(newValue: incoming);
1959 slot->erase();
1960 }
1961 pendingParamSlots.clear();
1962}
1963
1964mlir::LogicalResult CIRABIRewriteContext::rewriteFunctionDefinition(
1965 mlir::FunctionOpInterface funcOpInterface, const FunctionClassification &fc,
1966 mlir::OpBuilder &builder) {
1967 // The pass driver (CallConvLoweringPass) only ever hands us cir.func ops.
1968 // Cast once at the top so the rest of the function reads in CIR's own
1969 // vocabulary, and so we can dispatch to the CIRGlobalValueInterface for
1970 // isDefinition() (FunctionOpInterface alone does not inherit from
1971 // CIRGlobalValueInterface).
1972 cir::FuncOp funcOp = mlir::cast<cir::FuncOp>(Val&: funcOpInterface);
1973
1974 if (!fc.needsRewrite())
1975 return mlir::success();
1976
1977 ArrayRef<mlir::Type> oldArgTypes = funcOp.getArgumentTypes();
1978 ArrayRef<mlir::Type> oldResultTypes = funcOp.getResultTypes();
1979 mlir::MLIRContext *ctx = funcOp->getContext();
1980
1981 // CIR follows LLVM IR's single-result rule: a function returns either
1982 // zero or one value. Document the invariant so a future multi-result
1983 // change forces us to revisit the return-handling below.
1984 assert(oldResultTypes.size() <= 1 &&
1985 "CIR functions return zero or one value");
1986
1987 SmallVector<mlir::Type> newArgTypes;
1988 if (mlir::failed(Result: buildNewArgTypes(oldArgTypes, fc, newArgTypes, dl,
1989 emitError: [&]() { return funcOp.emitOpError(); })))
1990 return mlir::failure();
1991
1992 mlir::Type voidTy = cir::VoidType::get(ctx);
1993 mlir::Type origRetTy = oldResultTypes.empty() ? voidTy : oldResultTypes[0];
1994 mlir::Type newRetTy = computeNewReturnType(
1995 origRetTy, retInfo: fc.returnInfo, ctx, emitError: [&]() { return funcOp.emitOpError(); });
1996 if (!newRetTy)
1997 return mlir::failure();
1998 SmallVector<mlir::Type> newResultTypes = {newRetTy};
1999
2000 // sret return: the value is returned through a pointer the ABI inserts as
2001 // argument 0. This pointer is not part of the function's source-level
2002 // signature -- it is synthesized here -- and the wire return type was
2003 // already set to void by computeNewReturnType. Every classification index
2004 // therefore maps to a block argument shifted by one in the body handling
2005 // below.
2006 bool hasSRet =
2007 fc.returnInfo.kind == ArgKind::Indirect && !oldResultTypes.empty();
2008 cir::PointerType sretPtrTy;
2009 if (hasSRet) {
2010 sretPtrTy = getIndirectPtrTy(pointee: origRetTy, ac: fc.returnInfo);
2011 newArgTypes.insert(I: newArgTypes.begin(), Elt: sretPtrTy);
2012 }
2013
2014 if (funcOp.isDefinition()) {
2015 mlir::Region &body = funcOp->getRegion(index: 0);
2016 if (!body.empty()) {
2017 // Prepend the sret pointer block argument and route every cir.return
2018 // through it before any index-based argument handling below (which
2019 // then accounts for the +1 offset).
2020 if (hasSRet) {
2021 body.front().insertArgument(index: 0u, type: sretPtrTy, loc: funcOp.getLoc());
2022 insertSRetStores(funcOp, origRetTy, builder);
2023 }
2024
2025 // Reshape the entry block for the classifications that change what a
2026 // parameter looks like to the body: a Direct arg with a coerced type
2027 // gets a memory roundtrip back to its original type, an Indirect arg
2028 // becomes a pointer, and an Expand arg becomes one block argument per
2029 // field. Done before the Ignore-drop below so the entry block
2030 // argument indices used here still refer to the original positions.
2031 if (failed(Result: insertArgCoercion(funcOp, fc, builder, dl, hasSRetArg: hasSRet,
2032 pendingParamSlots)))
2033 return mlir::failure();
2034
2035 // Direct return with coerced type: insert a coercion at every
2036 // cir.return so the returned value matches the (coerced) return
2037 // type in the new function signature set below.
2038 if (fc.returnInfo.kind == ArgKind::Direct && fc.returnInfo.coercedType &&
2039 !oldResultTypes.empty() && fc.returnInfo.coercedType != origRetTy &&
2040 failed(Result: insertReturnCoercion(
2041 funcOp, origRetTy, coercedRetTy: fc.returnInfo.coercedType, builder, dl,
2042 cirDataLayout, offset: fc.returnInfo.directOffset)))
2043 return mlir::failure();
2044
2045 mlir::Block &entry = body.front();
2046
2047 // Drop each Ignored argument's block argument, replacing any remaining
2048 // body uses with a poison constant (an Ignore arg is not passed at the
2049 // ABI level, so any use is vacuous; poison says exactly that). Walk
2050 // forward with a running block-argument index that mirrors
2051 // insertArgCoercion: an Expand arg or a Direct+canFlatten arg occupies N
2052 // slots, every other kept kind one. On erase, do not advance the index
2053 // -- the next block argument shifts into the vacated slot.
2054 unsigned blockArgIdx = hasSRet ? 1 : 0;
2055 for (auto [i, ac] : llvm::enumerate(First: fc.argInfos)) {
2056 if (blockArgIdx >= entry.getNumArguments())
2057 break;
2058 if (ac.kind == ArgKind::Ignore) {
2059 mlir::BlockArgument arg = entry.getArgument(i: blockArgIdx);
2060 if (!arg.use_empty()) {
2061 builder.setInsertionPointToStart(&entry);
2062 mlir::Value poison =
2063 createIgnoredValue(builder, loc: funcOp.getLoc(), ty: arg.getType());
2064 arg.replaceAllUsesWith(newValue: poison);
2065 }
2066 entry.eraseArgument(index: blockArgIdx);
2067 continue;
2068 }
2069 if (cir::RecordType flatTy = getFlattenedCoercedType(ac, dl))
2070 blockArgIdx += flatTy.getNumElements();
2071 else if (ac.kind == ArgKind::Expand)
2072 blockArgIdx += cast<cir::RecordType>(Val: oldArgTypes[i]).getNumElements();
2073 else
2074 ++blockArgIdx;
2075 }
2076 }
2077
2078 // When the return is classified Ignore but the original function had
2079 // a non-void return type, every cir.return becomes a naked return.
2080 // This relies on the invariant that computeNewReturnType has set
2081 // newRetTy = void for Ignore above, and that the function type is
2082 // updated below to match. Asserting this keeps the dependency
2083 // explicit.
2084 if (fc.returnInfo.kind == ArgKind::Ignore && !oldResultTypes.empty()) {
2085 assert(mlir::isa<cir::VoidType>(newRetTy) &&
2086 "Ignore-return path requires the new return type to be void");
2087 SmallVector<cir::ReturnOp> returns;
2088 funcOp.walk(callback: [&](cir::ReturnOp r) { returns.push_back(Elt: r); });
2089 for (cir::ReturnOp r : returns) {
2090 if (r.getNumOperands() == 0)
2091 continue;
2092 builder.setInsertionPoint(r);
2093 cir::ReturnOp::create(builder, location: r.getLoc());
2094 r.erase();
2095 }
2096 }
2097 }
2098
2099 mlir::Type newFnTy = funcOp.cloneTypeWith(inputs: newArgTypes, results: newResultTypes);
2100 funcOp.setFunctionTypeAttr(mlir::TypeAttr::get(type: newFnTy));
2101
2102 // Rebuild arg_attrs when the function has an sret slot (slot 0 needs the
2103 // sret attribute set) or any arg needs updateArgAttrs.
2104 if (hasSRet || needsArgAttrUpdate(fc)) {
2105 auto existing = funcOp->getAttrOfType<mlir::ArrayAttr>(name: "arg_attrs");
2106 mlir::ArrayAttr updated = updateArgAttrs(ctx, module, origArgTypes: oldArgTypes, existingArgAttrs: existing,
2107 fc, dl, enableNoundefAttrs,
2108 /*isCallSite=*/false);
2109 if (hasSRet) {
2110 // Prepend the sret slot's attribute dict (slot 0); the per-argument
2111 // dicts shift to slots 1..N. noalias is valid only on the callee's
2112 // parameter, so it is added only for definitions.
2113 SmallVector<mlir::NamedAttribute> sretAttrs = buildSretSlotAttrs(
2114 builder, retTy: origRetTy, align: fc.returnInfo.indirectAlign.value(),
2115 /*withNoalias=*/funcOp.isDefinition());
2116 SmallVector<mlir::Attribute> withSret;
2117 withSret.push_back(Elt: mlir::DictionaryAttr::get(context: ctx, value: sretAttrs));
2118 llvm::append_range(C&: withSret, R&: updated);
2119 funcOp->setAttr(name: "arg_attrs", value: mlir::ArrayAttr::get(context: ctx, value: withSret));
2120 } else {
2121 funcOp->setAttr(name: "arg_attrs", value: updated);
2122 }
2123 }
2124
2125 if (mlir::isa<cir::VoidType>(Val: newRetTy)) {
2126 funcOp->removeAttr(name: "res_attrs");
2127 } else if (fc.returnInfo.kind == ArgKind::Extend ||
2128 fc.returnInfo.kind == ArgKind::Direct) {
2129 auto existing = funcOp->getAttrOfType<mlir::ArrayAttr>(name: "res_attrs");
2130 if (mlir::ArrayAttr updated =
2131 updateResAttrs(ctx, existingResAttrs: existing, retInfo: fc.returnInfo, origRetTy, dl))
2132 funcOp->setAttr(name: "res_attrs", value: updated);
2133 }
2134
2135 return mlir::success();
2136}
2137
2138mlir::LogicalResult
2139CIRABIRewriteContext::rewriteCallSite(mlir::Operation *callOp,
2140 const FunctionClassification &fc,
2141 mlir::OpBuilder &builder) {
2142 // The rewrite below pairs the classification with the call's operands one
2143 // for one, so a call passing arguments through an ellipsis needs a
2144 // classification built from its own operands rather than from the callee's
2145 // declared parameters. Both directions of a mismatch have to be reported
2146 // before the pass-through early return, or a call whose declared parameters
2147 // happen to be pass-through is left as written with its surplus operands
2148 // never classified.
2149 //
2150 // A surplus operand means fc was built from the callee's declared
2151 // parameters alone. A shortfall means the callee was declared no_proto,
2152 // which turns off the verifier's argument-count check altogether.
2153 unsigned numOperands =
2154 mlir::cast<cir::CIRCallOpInterface>(Val: callOp).getNumArgOperands();
2155 if (numOperands > fc.argInfos.size())
2156 return callOp->emitOpError()
2157 << "variadic arguments not yet implemented in CallConvLowering";
2158 if (numOperands < fc.argInfos.size())
2159 return callOp->emitOpError()
2160 << "call passes fewer arguments than the callee declares, which is "
2161 "not yet implemented in CallConvLowering";
2162
2163 if (!fc.needsRewrite())
2164 return mlir::success();
2165
2166 if (mlir::isa<cir::TryCallOp>(Val: callOp))
2167 return callOp->emitOpError()
2168 << "TryCallOp not yet implemented in CallConvLowering";
2169
2170 auto call = mlir::cast<cir::CallOp>(Val: callOp);
2171 mlir::MLIRContext *ctx = callOp->getContext();
2172 mlir::Block *slotBlock = coercionSlotBlock(op: call);
2173
2174 builder.setInsertionPoint(call);
2175
2176 SmallVector<mlir::Value> newArgs;
2177 mlir::ValueRange argOperands = call.getArgOperands();
2178 newArgs.reserve(N: argOperands.size());
2179
2180 // Loads that the new call leaves unused: a Direct (coerced or flattened) or
2181 // Expand record is read out of its load's storage or a copy of it, and an
2182 // Indirect argument passes or copies from the address the load read from.
2183 // The old call still uses them, so erase them only after it is gone.
2184 SmallVector<cir::LoadOp> deadRecordLoads;
2185 // Erased before deadRecordLoads, since a freeze keeps its load alive.
2186 SmallVector<cir::FreezeOp> deadFreezes;
2187
2188 // Capture original arg types before building newArgs (an Indirect argument
2189 // changes the wire argument from T to !cir.ptr<T>, so we save the
2190 // pre-rewrite types here for use in updateArgAttrs).
2191 SmallVector<mlir::Type> origCallArgTypes;
2192 llvm::append_range(C&: origCallArgTypes, R: argOperands.getTypes());
2193
2194 // Operands past an indirect callee's declared parameters were passed
2195 // through its ellipsis. Note where they begin in newArgs, so
2196 // prependIndirectCallee can leave them out of the retyped callee type. A
2197 // direct call has no callee pointer to retype, so it records no boundary.
2198 unsigned numDeclared = fc.argInfos.size();
2199 if (call.isIndirect()) {
2200 numDeclared = getIndirectCalleeType(call).getNumInputs();
2201 assert(numDeclared <= fc.argInfos.size() &&
2202 "the classification covers at least the declared parameters");
2203 }
2204 // Recorded before the Ignore skip below, so an ignored first ellipsis
2205 // argument still marks where the ellipsis arguments begin in newArgs.
2206 std::optional<unsigned> ellipsisStart;
2207 // Where each argument starts in newArgs, and the freezes to reapply there
2208 // once every argument is rewritten.
2209 SmallVector<unsigned> argBegin;
2210 SmallVector<std::pair<unsigned, cir::FreezeOp>> refreezes;
2211 for (auto [idx, ac] : llvm::enumerate(First: fc.argInfos)) {
2212 if (idx == numDeclared)
2213 ellipsisStart = newArgs.size();
2214 argBegin.push_back(Elt: newArgs.size());
2215 mlir::Value arg = argOperands[idx];
2216 auto freeze = arg.getDefiningOp<cir::FreezeOp>();
2217 if (freeze && !freeze->hasOneUse())
2218 freeze = {};
2219 if (ac.kind == ArgKind::Ignore) {
2220 if (freeze)
2221 deadFreezes.push_back(Elt: freeze);
2222 continue;
2223 }
2224
2225 // Refreeze after the ABI rewrite, as classic CodeGen does, so padding is
2226 // frozen too. Indirect freezes only the pointer, so keep non-load freezes.
2227 if (freeze && ((ac.kind != ArgKind::Indirect &&
2228 ac.kind != ArgKind::IndirectAliased) ||
2229 freeze.getInput().getDefiningOp<cir::LoadOp>())) {
2230 arg = freeze.getInput();
2231 deadFreezes.push_back(Elt: freeze);
2232 refreezes.push_back(Elt: {idx, freeze});
2233 }
2234 if (cir::RecordType flatTy = getFlattenedCoercedType(ac, dl)) {
2235 // Direct + canFlatten: pass one scalar call argument per field of the
2236 // ABI-coerced struct. A non-atomic record load is read in place or
2237 // through a copy of its storage, as emitCoercedRecordLoad decides, and
2238 // any other operand is stored into a coercion slot first. When the types
2239 // already match, the fields come from the struct value directly (reading
2240 // from its source alloca when possible).
2241 if (arg.getType() != flatTy) {
2242 SmallPtrSet<mlir::Operation *, 4> coercionOps;
2243 assert(!ac.directOffset &&
2244 "the coerced struct starts at slot offset 0 here, so a "
2245 "flattened coercion cannot honor a direct offset");
2246 if (failed(Result: rejectAtomicRecordLoad(user: call, val: arg)))
2247 return mlir::failure();
2248 mlir::OpBuilder::InsertionGuard guard(builder);
2249 mlir::Value coercedPtr;
2250 uint64_t baseAlign = 0;
2251 if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(val: arg)) {
2252 mlir::Value base = recLoad.getAddr();
2253 if (!recLoad.getIsVolatile() &&
2254 canReadInPlace(load: recLoad, coercedTy: flatTy, dl: cirDataLayout, /*offset=*/0)) {
2255 cir::AllocaOp baseAlloca = cir::getUnderlyingAlloca(addr: base);
2256 assert(baseAlloca && "canReadInPlace requires an alloca");
2257 baseAlign = baseAlloca.getAlignment();
2258 } else {
2259 base = emitCopiedCoercionSlot(builder, loc: call.getLoc(), load: recLoad,
2260 coercedTy: flatTy, slotBlock, dl, /*offset=*/0);
2261 }
2262 builder.setInsertionPoint(recLoad);
2263 coercedPtr = emitViewAtOffset(builder, loc: call.getLoc(), wantTy: flatTy, base,
2264 createdOps&: coercionOps, /*offset=*/0);
2265 deadRecordLoads.push_back(Elt: recLoad);
2266 } else {
2267 coercedPtr =
2268 emitCoercionToMemory(builder, loc: call.getLoc(), dstTy: flatTy, src: arg,
2269 slotBlock, dl, createdOps&: coercionOps, /*offset=*/0);
2270 }
2271 auto coercedPtrTy = mlir::cast<cir::PointerType>(Val: coercedPtr.getType());
2272 for (auto [f, fieldTy] : llvm::enumerate(First: flatTy.getMembers())) {
2273 mlir::Type fieldPtrTy =
2274 cir::PointerType::get(pointee: fieldTy, addrSpace: coercedPtrTy.getAddrSpace());
2275 auto fieldPtr =
2276 cir::GetMemberOp::create(builder, location: call.getLoc(), type: fieldPtrTy,
2277 value: coercedPtr, /*name=*/"", /*index=*/f);
2278 auto field = cir::LoadOp::create(builder, location: call.getLoc(), result: fieldTy,
2279 addr: fieldPtr.getResult());
2280 // Storage read in place may be less aligned than the coerced struct,
2281 // so each field claims only what its offset in it allows.
2282 if (baseAlign)
2283 field.setAlignment(
2284 llvm::MinAlign(A: baseAlign, B: flatTy.getElementOffset(dataLayout: dl, idx: f)));
2285 newArgs.push_back(Elt: field);
2286 }
2287 } else {
2288 emitStructFieldArgs(builder, loc: call.getLoc(), structVal: arg, recTy: flatTy, newArgs,
2289 deadRecordLoads);
2290 }
2291 } else if (ac.kind == ArgKind::Expand) {
2292 // Decompose the struct value into its constituent scalar fields and
2293 // pass each as a separate argument.
2294 auto recTy = cast<cir::RecordType>(Val: arg.getType());
2295 assert(recTy.isStruct() &&
2296 "Expand classification requires a struct type, not a union");
2297 emitStructFieldArgs(builder, loc: call.getLoc(), structVal: arg, recTy, newArgs,
2298 deadRecordLoads);
2299 } else if ((ac.kind == ArgKind::Direct && ac.coercedType &&
2300 arg.getType() != ac.coercedType) ||
2301 isRecordExtend(ac, origTy: arg.getType())) {
2302 if (failed(Result: rejectAtomicRecordLoad(user: call, val: arg)))
2303 return mlir::failure();
2304 if (cir::LoadOp recLoad = maybeGetCopyableRecordLoad(val: arg)) {
2305 newArgs.push_back(Elt: emitCoercedRecordLoad(
2306 builder, loc: call.getLoc(), load: recLoad, coercedTy: ac.coercedType, slotBlock, dl,
2307 cirDataLayout, offset: ac.directOffset));
2308 deadRecordLoads.push_back(Elt: recLoad);
2309 } else {
2310 newArgs.push_back(Elt: emitCoercion(builder, loc: call.getLoc(), dstTy: ac.coercedType,
2311 src: arg, slotBlock, dl, offset: ac.directOffset));
2312 }
2313 } else if (ac.kind == ArgKind::Indirect ||
2314 ac.kind == ArgKind::IndirectAliased) {
2315 uint64_t minAlign = ac.indirectAlign.value();
2316 // The callee copies a byref object at entry, so the caller treats it
2317 // like byval.
2318 bool byRef = ac.kind == ArgKind::IndirectAliased;
2319 bool byCopy = byRef || ac.byVal;
2320 cir::PointerType ptrTy = getIndirectPtrTy(pointee: arg.getType(), ac);
2321
2322 // A record operand that names storage is passed from that storage
2323 // rather than from the loaded value: a record value carries only the
2324 // fields of its type, so a store leaves the type's padding bytes
2325 // unwritten, and for a union those can be another member's live data.
2326 // A byval _BitInt is passed from its value, since the load and the
2327 // store below are its conversion between value and in-memory form.
2328 cir::LoadOp srcLoad;
2329 if (!byCopy || mlir::isa<cir::RecordType>(Val: arg.getType()))
2330 srcLoad = maybeGetSimpleLoad(val: arg);
2331
2332 // A non-byval argument must name the caller's storage, so that the
2333 // object the callee operates on is the one the caller destroys. That
2334 // means passing the address the operand was loaded from rather than the
2335 // loaded value, so a store to that storage after the load is visible to
2336 // the callee. byval fills a slot below, this cannot.
2337 if (!byCopy) {
2338 mlir::Value fwdAddr = forwardableIndirectAddr(load: srcLoad, paramTy: ptrTy, minAlign);
2339 if (!fwdAddr)
2340 return call->emitOpError()
2341 << "non-byval indirect argument that does not name the "
2342 "caller's storage is not yet implemented in "
2343 "CallConvLowering";
2344 newArgs.push_back(Elt: fwdAddr);
2345 deadRecordLoads.push_back(Elt: srcLoad);
2346 continue;
2347 }
2348
2349 // byval hands the callee a copy taken at the call, so naming the object's
2350 // own storage leaves it with the same copy and saves this one. That
2351 // holds only while nothing writes that storage between the load and the
2352 // call, since the operand carries the value as of the load. A volatile
2353 // load is not forwarded, since the copy byval takes at the call is not
2354 // volatile.
2355 if (mlir::Value fwdAddr =
2356 forwardableIndirectAddr(load: srcLoad, paramTy: ptrTy, minAlign);
2357 fwdAddr && storageUnwrittenBetween(load: srcLoad, call: call.getOperation(),
2358 aliasAnalysis)) {
2359 newArgs.push_back(Elt: fwdAddr);
2360 deadRecordLoads.push_back(Elt: srcLoad);
2361 continue;
2362 }
2363
2364 // A byref slot has to be an alloca of the parameter's own type, which
2365 // only the alloca address space can hold.
2366 if (byRef && ptrTy != getAllocaPtrTy(pointee: arg.getType(), dl))
2367 return call->emitOpError()
2368 << "byref argument outside the alloca address space that does "
2369 "not name forwardable storage is not yet implemented in "
2370 "CallConvLowering";
2371
2372 // Otherwise fill a slot, at the load's position when there is one so
2373 // the copy reads the memory state the load read.
2374 if (failed(Result: rejectAtomicRecordLoad(user: call, val: arg)))
2375 return mlir::failure();
2376 if (!srcLoad)
2377 srcLoad = maybeGetCopyableRecordLoad(val: arg);
2378 mlir::IntegerAttr slotAlign = builder.getI64IntegerAttr(value: minAlign);
2379 mlir::OpBuilder::InsertionGuard guard(builder);
2380 if (srcLoad)
2381 builder.setInsertionPoint(srcLoad);
2382
2383 auto slot = cir::AllocaOp::create(
2384 builder, location: call.getLoc(), addr: ptrTy,
2385 name: builder.getStringAttr(bytes: byRef ? "byref" : "byval"), alignment: slotAlign);
2386 if (srcLoad) {
2387 auto copy = cir::CopyOp::create(builder, location: call.getLoc(), dst: slot,
2388 src: srcLoad.getAddr(), dst_alignment: slotAlign,
2389 src_alignment: loadSourceAlignment(load: srcLoad, builder));
2390 copy.setIsVolatile(srcLoad.getIsVolatile());
2391 deadRecordLoads.push_back(Elt: srcLoad);
2392 } else {
2393 // A record with no storage behind it, such as a constant, defines no
2394 // bytes beyond its fields. A producer rewritten later turns this store
2395 // into a copy through rewireRecordUses.
2396 cir::StoreOp::create(builder, location: call.getLoc(), value: arg, addr: slot);
2397 }
2398 newArgs.push_back(Elt: slot);
2399 } else {
2400 newArgs.push_back(Elt: arg);
2401 }
2402 }
2403 argBegin.push_back(Elt: newArgs.size());
2404 builder.setInsertionPoint(call);
2405 for (auto [idx, freeze] : refreezes)
2406 for (unsigned i = argBegin[idx]; i != argBegin[idx + 1]; ++i)
2407 newArgs[i] = cir::FreezeOp::create(builder, location: freeze.getLoc(), input: newArgs[i]);
2408 unsigned numEllipsisArgs =
2409 ellipsisStart ? newArgs.size() - *ellipsisStart : 0;
2410
2411 bool hasResult = call.getNumResults() > 0;
2412 mlir::Type origRetTy =
2413 hasResult ? call.getResult().getType() : cir::VoidType::get(ctx);
2414
2415 // An indirect (sret) return has a different call shape than the coerce /
2416 // extend / ignore return handling further down (the value is returned
2417 // through a prepended pointer slot, not as a result), so dispatch to a
2418 // dedicated helper for it; everything below handles the by-value returns.
2419 if (fc.returnInfo.kind == ArgKind::Indirect && hasResult) {
2420 if (failed(Result: rewriteIndirectReturnCall(call, fc, newArgs, numEllipsisArgs,
2421 origRetTy, origCallArgTypes, builder,
2422 dl, enableNoundefAttrs)))
2423 return mlir::failure();
2424 eraseDeadFreezes(freezes: deadFreezes);
2425 eraseDeadRecordLoads(loads: deadRecordLoads);
2426 return mlir::success();
2427 }
2428
2429 mlir::Type callRetTy = origRetTy;
2430 if (fc.returnInfo.kind == ArgKind::Ignore && hasResult)
2431 callRetTy = cir::VoidType::get(ctx);
2432 bool returnNeedsCoercion =
2433 hasResult && fc.returnInfo.kind == ArgKind::Direct &&
2434 fc.returnInfo.coercedType && fc.returnInfo.coercedType != origRetTy;
2435 if (returnNeedsCoercion)
2436 callRetTy = fc.returnInfo.coercedType;
2437
2438 builder.setInsertionPoint(call);
2439 prependIndirectCallee(call, args&: newArgs, retTy: callRetTy, numEllipsisArgs, builder);
2440 auto newCall = cir::CallOp::create(builder, location: call.getLoc(),
2441 callee: call.getCalleeAttr(), resType: callRetTy, operands: newArgs);
2442 copyCallAttributes(source: call, target: newCall);
2443
2444 // Direct return with coercion: the new call returns the coerced type, which
2445 // is stored into a coercion slot where the old call's uses find the original
2446 // type.
2447 if (returnNeedsCoercion) {
2448 builder.setInsertionPointAfter(newCall);
2449 SmallPtrSet<mlir::Operation *, 4> coercionOps;
2450 if (failed(Result: emitCoercionAndReplaceUses(builder, loc: call.getLoc(), origTy: origRetTy,
2451 src: newCall.getResult(), val: call.getResult(),
2452 slotBlock, dl, createdOps&: coercionOps,
2453 offset: fc.returnInfo.directOffset)))
2454 return mlir::failure();
2455 }
2456
2457 if (needsArgAttrUpdate(fc)) {
2458 auto existing = call->getAttrOfType<mlir::ArrayAttr>(name: "arg_attrs");
2459 newCall->setAttr(name: "arg_attrs",
2460 value: updateArgAttrs(ctx, module, origArgTypes: origCallArgTypes, existingArgAttrs: existing, fc,
2461 dl, enableNoundefAttrs,
2462 /*isCallSite=*/true));
2463 }
2464 // Add llvm.signext / llvm.zeroext to an Extend return, and drop llvm.noundef
2465 // from a coerced Direct return that passes undefined bits.
2466 if (fc.returnInfo.kind == ArgKind::Extend ||
2467 (hasResult && fc.returnInfo.kind == ArgKind::Direct)) {
2468 auto existing = call->getAttrOfType<mlir::ArrayAttr>(name: "res_attrs");
2469 if (mlir::ArrayAttr updated =
2470 updateResAttrs(ctx, existingResAttrs: existing, retInfo: fc.returnInfo, origRetTy, dl))
2471 newCall->setAttr(name: "res_attrs", value: updated);
2472 } else if (hasResult && mlir::isa<cir::VoidType>(Val: callRetTy)) {
2473 newCall->removeAttr(name: "res_attrs");
2474 }
2475
2476 if (hasResult && fc.returnInfo.kind == ArgKind::Ignore) {
2477 // The new call returns void, but the original call's result may still
2478 // have uses. Substitute a poison constant of the original type so
2479 // those uses remain well-formed without pretending we have a real
2480 // value at the ABI boundary.
2481 if (!call.getResult().use_empty()) {
2482 builder.setInsertionPointAfter(newCall);
2483 mlir::Value poison =
2484 createIgnoredValue(builder, loc: call.getLoc(), ty: origRetTy);
2485 call.getResult().replaceAllUsesWith(newValue: poison);
2486 }
2487 } else if (hasResult && !returnNeedsCoercion) {
2488 // returnNeedsCoercion already wired up the coerced result above.
2489 call.getResult().replaceAllUsesWith(newValue: newCall.getResult());
2490 }
2491
2492 call->erase();
2493 eraseDeadFreezes(freezes: deadFreezes);
2494 eraseDeadRecordLoads(loads: deadRecordLoads);
2495
2496 return mlir::success();
2497}
2498
2499void CIRABIRewriteContext::rewriteFunctionAddress(cir::GetGlobalOp addrOp,
2500 cir::FuncOp funcOp,
2501 mlir::OpBuilder &builder) {
2502 auto oldPtrTy = mlir::cast<cir::PointerType>(Val: addrOp.getAddr().getType());
2503 cir::FuncType newFuncTy = funcOp.getFunctionType();
2504 // An extension rides on an argument attribute and leaves the signature
2505 // alone, so such a callee still matches the written type.
2506 if (newFuncTy == oldPtrTy.getPointee())
2507 return;
2508
2509 // The verifier requires the retype even when nothing reads the address.
2510 addrOp.getAddr().setType(cir::PointerType::get(pointee: newFuncTy));
2511 if (addrOp.getAddr().use_empty())
2512 return;
2513
2514 // A later indirect call through the written type is classified and coerced
2515 // on its own, so casting back leaves it to that rewrite.
2516 mlir::OpBuilder::InsertionGuard guard(builder);
2517 builder.setInsertionPointAfter(addrOp);
2518 auto bitcast = cir::CastOp::create(builder, location: addrOp.getLoc(), result: oldPtrTy,
2519 kind: cir::CastKind::bitcast, src: addrOp.getAddr());
2520 addrOp.getAddr().replaceAllUsesExcept(newValue: bitcast.getResult(), exceptedUser: bitcast);
2521}
2522
2523namespace {
2524
2525/// What one `va_arg` expansion needs from the op it rewrites. The x86-64
2526/// cursor fields are reached through `vaFields` by index: 0 gp_offset,
2527/// 1 fp_offset, 2 overflow_arg_area, 3 reg_save_area.
2528struct VAArgFetch {
2529 VAArgFetch(mlir::Location loc, mlir::Value valist,
2530 llvm::ArrayRef<mlir::Type> vaFields, mlir::Type resultTy,
2531 const ArgClassification &ac, const mlir::DataLayout &dl,
2532 mlir::ModuleOp module)
2533 : loc(loc), valist(valist), vaFields(vaFields), resultTy(resultTy),
2534 ac(ac), dl(dl), module(module) {
2535 assert(vaFields.size() == 4 &&
2536 "the x86-64 va_list is a four-field cursor, checked by the caller");
2537 }
2538
2539 mlir::Location loc;
2540 mlir::Value valist;
2541 llvm::ArrayRef<mlir::Type> vaFields;
2542 mlir::Type resultTy;
2543 const ArgClassification &ac;
2544 const mlir::DataLayout &dl;
2545 mlir::ModuleOp module;
2546};
2547
2548/// The cursor fields a register fetch reads, and the predicate saying every
2549/// class it needs still has room. An offset is null when the fetch needs no
2550/// register of that class.
2551struct RegisterCursor {
2552 mlir::Value gpOffsetP;
2553 mlir::Value fpOffsetP;
2554 mlir::Value gpOffset;
2555 mlir::Value fpOffset;
2556 mlir::Value inRegs;
2557};
2558
2559mlir::LogicalResult reportVAArgNYI(cir::VAArgOp op, llvm::StringRef what) {
2560 op->emitOpError() << "va_arg of " << what
2561 << " not yet implemented in CallConvLowering";
2562 return mlir::failure();
2563}
2564
2565/// Sets \p isRegPair when a two-register argument is a coerced pair, one
2566/// register per member, rather than a single wide scalar, and records in
2567/// \p pairIsSse which element is SSE class. Fails on a coercion that is not
2568/// two eightbytes.
2569mlir::LogicalResult classifyRegisterPair(cir::VAArgOp op,
2570 const ArgClassification &ac,
2571 std::array<bool, 2> &pairIsSse,
2572 bool &isRegPair) {
2573 auto pairTy = mlir::dyn_cast<cir::RecordType>(Val: ac.coercedType);
2574 if (!pairTy)
2575 return mlir::success();
2576
2577 if (pairTy.getNumElements() != 2)
2578 return reportVAArgNYI(op, what: "a register coercion that is not two eightbytes");
2579
2580 assert(!ac.directOffset &&
2581 "a pair already spans both eightbytes, so it cannot also start "
2582 "partway into the value");
2583 for (auto [i, memberTy] : llvm::enumerate(First: pairTy.getMembers()))
2584 pairIsSse[i] = isSSERegisterClass(ty: memberTy);
2585 isRegPair = true;
2586 return mlir::success();
2587}
2588
2589/// The alignment an argument is placed at, which is the language alignment of
2590/// its type.
2591uint64_t argumentAreaAlign(mlir::Type ty, mlir::ModuleOp modOp) {
2592 return cir::CIRDataLayout(modOp).getTypeLanguageAlignment(ty);
2593}
2594
2595mlir::Value roundPointerUpToAlignment(CIRBaseBuilderTy &b, mlir::Location loc,
2596 mlir::Value bytePtr, uint64_t align,
2597 const mlir::DataLayout &dl) {
2598 assert(mlir::cast<cir::PointerType>(bytePtr.getType()).getPointee() ==
2599 b.getUIntNTy(8) &&
2600 "the bump strides in bytes, so the pointee must be u8");
2601 assert(llvm::isPowerOf2_64(align) &&
2602 "mask rounding needs a power-of-two alignment");
2603 mlir::Value bumped =
2604 b.createPtrStride(loc, base: bytePtr, stride: b.getSignedInt(loc, val: align - 1, numBits: 32));
2605 std::optional<uint64_t> indexWidth =
2606 dl.getTypeIndexBitwidth(t: bytePtr.getType());
2607 assert(indexWidth && "a pointer in the argument area has an index width");
2608 mlir::Value mask = b.getSignedInt(loc, val: -static_cast<int64_t>(align),
2609 numBits: static_cast<unsigned>(*indexWidth));
2610 return cir::PtrMaskOp::create(builder&: b, location: loc, result: bytePtr.getType(), ptr: bumped, mask);
2611}
2612
2613/// Reads the argument's address out of the overflow area, which also advances
2614/// the cursor past the argument.
2615mlir::Value buildOverflowAddrAndAdvance(CIRBaseBuilderTy &b,
2616 const VAArgFetch &f) {
2617 cir::IntType byteTy = b.getUIntNTy(n: 8);
2618 mlir::Value overflowP = b.createGetMember(
2619 loc: f.loc, resultTy: b.getPointerTo(ty: f.vaFields[2]), base: f.valist, name: "overflow_arg_area", index: 2);
2620 mlir::Value overflow = b.createLoad(loc: f.loc, ptr: overflowP);
2621 mlir::Value bytePtr = b.createPtrBitcast(src: overflow, newPointeeTy: byteTy);
2622
2623 uint64_t tyAlign = argumentAreaAlign(ty: f.resultTy, modOp: f.module);
2624 if (tyAlign > 8)
2625 bytePtr = roundPointerUpToAlignment(b, loc: f.loc, bytePtr, align: tyAlign, dl: f.dl);
2626
2627 uint64_t stride =
2628 llvm::alignTo(Value: f.dl.getTypeAllocSize(t: f.resultTy).getFixedValue(), Align: 8);
2629 mlir::Value strideVal = b.getSignedInt(loc: f.loc, val: stride, numBits: 32);
2630 mlir::Value next = b.createPtrStride(loc: f.loc, base: bytePtr, stride: strideVal);
2631 b.createStore(loc: f.loc, val: next, dst: overflowP);
2632 return bytePtr;
2633}
2634
2635/// Loads the cursor offsets and builds the predicate that sends the fetch to
2636/// the register-save area. Both offsets count from the start of that area, so
2637/// the integer limit is the six GP registers at 48 and the vector limit is
2638/// those plus the eight SSE registers at 176.
2639RegisterCursor buildRegisterGate(CIRBaseBuilderTy &b, const VAArgFetch &f,
2640 unsigned neededInt, unsigned neededSse) {
2641 RegisterCursor cursor;
2642 if (neededInt) {
2643 cursor.gpOffsetP = b.createGetMember(loc: f.loc, resultTy: b.getPointerTo(ty: f.vaFields[0]),
2644 base: f.valist, name: "gp_offset", index: 0);
2645 cursor.gpOffset = b.createLoad(loc: f.loc, ptr: cursor.gpOffsetP);
2646 mlir::Value limit =
2647 b.getConstantInt(loc: f.loc, ty: cursor.gpOffset.getType(), value: 48 - neededInt * 8);
2648 cursor.inRegs =
2649 b.createCompare(loc: f.loc, kind: cir::CmpOpKind::le, lhs: cursor.gpOffset, rhs: limit);
2650 }
2651 if (neededSse) {
2652 cursor.fpOffsetP = b.createGetMember(loc: f.loc, resultTy: b.getPointerTo(ty: f.vaFields[1]),
2653 base: f.valist, name: "fp_offset", index: 1);
2654 cursor.fpOffset = b.createLoad(loc: f.loc, ptr: cursor.fpOffsetP);
2655 mlir::Value limit = b.getConstantInt(loc: f.loc, ty: cursor.fpOffset.getType(),
2656 value: 176 - neededSse * 16);
2657 mlir::Value fitsInFp =
2658 b.createCompare(loc: f.loc, kind: cir::CmpOpKind::le, lhs: cursor.fpOffset, rhs: limit);
2659 cursor.inRegs = cursor.inRegs
2660 ? b.createLogicalAnd(loc: f.loc, lhs: cursor.inRegs, rhs: fitsInFp)
2661 : fitsInFp;
2662 }
2663 return cursor;
2664}
2665
2666/// Copies each half of a non-contiguous pair out of the register-save area
2667/// into \p regPairTemp, laid out as the coerced pair.
2668void reassembleRegisterPair(CIRBaseBuilderTy &b, mlir::Location loc,
2669 const ArgClassification &ac,
2670 const RegisterCursor &cursor,
2671 const std::array<bool, 2> &pairIsSse,
2672 mlir::Value regSaveArea, mlir::Value regPairTemp) {
2673 auto pairTy = mlir::cast<cir::RecordType>(Val: ac.coercedType);
2674 // Both halves of an all-SSE pair sit in 16-byte slots, which classic
2675 // CodeGen tells the load about. It leaves a mixed pair to the element's
2676 // own alignment, so match that rather than claiming the slot there.
2677 bool bothSse = pairIsSse[0] && pairIsSse[1];
2678 // Track how many of each class came before, since a slot is reached from
2679 // its class's own cursor.
2680 unsigned seenOfClass[2] = {0, 0};
2681 for (unsigned i = 0; i < 2; ++i) {
2682 bool isSse = pairIsSse[i];
2683 mlir::Value base = isSse ? cursor.fpOffset : cursor.gpOffset;
2684 unsigned regSize = isSse ? 16 : 8;
2685 unsigned prior = seenOfClass[isSse];
2686 ++seenOfClass[isSse];
2687 mlir::Value off = base;
2688 if (prior) {
2689 off = b.createAdd(loc, lhs: base,
2690 rhs: b.getConstantInt(loc, ty: base.getType(), value: prior * regSize));
2691 }
2692 mlir::Value src = b.createPtrStride(loc, base: regSaveArea, stride: off);
2693 mlir::Type elemTy = pairTy.getElementType(idx: i);
2694 mlir::Value elemPtr = b.createPtrBitcast(src, newPointeeTy: elemTy);
2695 mlir::Value val = bothSse ? b.createAlignedLoad(loc, ptr: elemPtr, alignment: 16)
2696 : b.createLoad(loc, ptr: elemPtr);
2697 b.createStore(
2698 loc, val,
2699 dst: b.createGetMember(loc, resultTy: b.getPointerTo(ty: elemTy), base: regPairTemp, name: "", index: i));
2700 }
2701}
2702
2703/// The bytes carried by the registers of this fetch's one class.
2704uint64_t registerSlotSize(unsigned neededInt, unsigned neededSse) {
2705 assert(!(neededInt && neededSse) &&
2706 "a fetch needing both classes is a pair, reassembled elsewhere");
2707 return neededSse ? neededSse * 16 : neededInt * 8;
2708}
2709
2710/// Whether the register cannot be read in place, either because it carries
2711/// less than the whole result or because its slot is under-aligned for it.
2712bool needsTempCopy(const VAArgFetch &f, unsigned neededInt,
2713 unsigned neededSse) {
2714 uint64_t tySize = f.dl.getTypeSize(t: f.resultTy).getFixedValue();
2715 if (f.ac.coercedType &&
2716 (f.ac.directOffset || registerSlotSize(neededInt, neededSse) < tySize))
2717 return true;
2718 // A slot is only as aligned as its class, 8 for a GP register and 16 for an
2719 // SSE one, so a fetch the ABI places more strictly is copied through a temp.
2720 uint64_t slotAlign = neededSse ? 16 : 8;
2721 return argumentAreaAlign(ty: f.resultTy, modOp: f.module) > slotAlign;
2722}
2723
2724/// Copies what the registers carry into \p temp, which is the size of the
2725/// whole result, and returns the address to read the result from.
2726mlir::Value copyRegisterToTemp(CIRBaseBuilderTy &b, mlir::Location loc,
2727 const VAArgFetch &f, mlir::Value regAddr,
2728 mlir::Value temp, unsigned neededInt,
2729 unsigned neededSse) {
2730 cir::IntType byteTy = b.getUIntNTy(n: 8);
2731 uint64_t tySize = f.dl.getTypeSize(t: f.resultTy).getFixedValue();
2732
2733 if (f.ac.coercedType &&
2734 (f.ac.directOffset || registerSlotSize(neededInt, neededSse) < tySize)) {
2735 // The registers carry less than the whole result, either because the
2736 // eightbytes below directOffset hold no field or because the result is
2737 // wider than the registers carrying it. Copy only what they carry, so
2738 // that reading the temp cannot run on into the neighboring slot.
2739 mlir::Value val = b.createAlignedLoad(
2740 loc, ptr: b.createPtrBitcast(src: regAddr, newPointeeTy: f.ac.coercedType), alignment: 8);
2741 mlir::Value dst = temp;
2742 if (f.ac.directOffset) {
2743 mlir::Value tempByte = b.createPtrBitcast(src: temp, newPointeeTy: byteTy);
2744 mlir::Value offset = b.getSignedInt(loc, val: f.ac.directOffset, numBits: 32);
2745 dst = b.createPtrStride(loc, base: tempByte, stride: offset);
2746 }
2747 b.createStore(loc, val, dst: b.createPtrBitcast(src: dst, newPointeeTy: f.ac.coercedType));
2748 return b.createPtrBitcast(src: temp, newPointeeTy: byteTy);
2749 }
2750
2751 mlir::Value regView = b.createPtrBitcast(src: regAddr, newPointeeTy: f.resultTy);
2752 if (mlir::isa<cir::RecordType>(Val: f.resultTy))
2753 cir::CopyOp::create(
2754 builder&: b, location: loc, dst: temp, src: regView,
2755 dst_alignment: b.getI64IntegerAttr(value: argumentAreaAlign(ty: f.resultTy, modOp: f.module)),
2756 src_alignment: b.getI64IntegerAttr(value: 8));
2757 else
2758 b.createStore(loc, val: b.createAlignedLoad(loc, ptr: regView, alignment: 8), dst: temp);
2759 return b.createPtrBitcast(src: temp, newPointeeTy: byteTy);
2760}
2761
2762void advanceRegisterCursors(CIRBaseBuilderTy &b, mlir::Location loc,
2763 const RegisterCursor &cursor, unsigned neededInt,
2764 unsigned neededSse) {
2765 if (neededInt) {
2766 b.createStore(loc,
2767 val: b.createAdd(loc, lhs: cursor.gpOffset,
2768 rhs: b.getConstantInt(loc, ty: cursor.gpOffset.getType(),
2769 value: neededInt * 8)),
2770 dst: cursor.gpOffsetP);
2771 }
2772 if (neededSse) {
2773 b.createStore(loc,
2774 val: b.createAdd(loc, lhs: cursor.fpOffset,
2775 rhs: b.getConstantInt(loc, ty: cursor.fpOffset.getType(),
2776 value: neededSse * 16)),
2777 dst: cursor.fpOffsetP);
2778 }
2779}
2780
2781} // namespace
2782
2783mlir::LogicalResult
2784CIRABIRewriteContext::rewriteVAArg(mlir::Operation *vaArgOp,
2785 const ArgClassification &ac,
2786 mlir::OpBuilder &opBuilder) {
2787 auto op = mlir::cast<cir::VAArgOp>(Val: vaArgOp);
2788 CIRBaseBuilderTy builder(opBuilder);
2789 mlir::Location loc = op.getLoc();
2790 mlir::Type resultTy = op.getType();
2791 mlir::Value valist = op.getArgList();
2792
2793 // An ignored type travels in no register and no stack slot, so the fetch
2794 // reads nothing and must leave the cursor unchanged. The value holds no
2795 // bytes, so poison stands in for it.
2796 if (ac.kind == ArgKind::Ignore) {
2797 builder.setInsertionPoint(op);
2798 op.getResult().replaceAllUsesWith(
2799 newValue: createIgnoredValue(builder, loc, ty: resultTy));
2800 op->erase();
2801 return mlir::success();
2802 }
2803
2804 if (ac.kind == ArgKind::Indirect && !ac.byVal)
2805 return reportVAArgNYI(op, what: "a non-trivially-copyable type");
2806 if (ac.kind == ArgKind::IndirectAliased)
2807 return reportVAArgNYI(op, what: "a type passed byref");
2808
2809 // neededInt counts 8-byte integer slots and neededSse counts 16-byte vector
2810 // slots. Zero of both means the type travels in memory and is read
2811 // straight from the overflow area.
2812 unsigned neededInt = ac.neededIntRegs;
2813 unsigned neededSse = ac.neededSseRegs;
2814
2815 // Which coerced-pair element (0 = low eightbyte, 1 = high) is SSE rather
2816 // than INTEGER class. Only meaningful when isRegPair is set.
2817 std::array<bool, 2> pairIsSse = {false, false};
2818 bool isRegPair = false;
2819 if (ac.kind == ArgKind::Direct && neededInt + neededSse == 2 &&
2820 ac.coercedType) {
2821 if (classifyRegisterPair(op, ac, pairIsSse, isRegPair).failed())
2822 return mlir::failure();
2823 }
2824
2825 auto vaListRecTy = mlir::dyn_cast<cir::RecordType>(
2826 Val: mlir::cast<cir::PointerType>(Val: valist.getType()).getPointee());
2827 if (!vaListRecTy || vaListRecTy.getNumElements() != 4) {
2828 return reportVAArgNYI(op,
2829 what: "a va_list that is not the four-field gp_offset / "
2830 "fp_offset / overflow_arg_area / reg_save_area "
2831 "cursor");
2832 }
2833
2834 const VAArgFetch fetch{loc, valist, vaListRecTy.getMembers(), resultTy, ac,
2835 dl, module};
2836 cir::IntType byteTy = builder.getUIntNTy(n: 8);
2837
2838 builder.setInsertionPoint(op);
2839
2840 mlir::Value addr;
2841 if (neededInt == 0 && neededSse == 0) {
2842 addr = buildOverflowAddrAndAdvance(b&: builder, f: fetch);
2843 } else {
2844 RegisterCursor cursor =
2845 buildRegisterGate(b&: builder, f: fetch, neededInt, neededSse);
2846
2847 // A two-eightbyte pair that is purely INTEGER class is contiguous in the
2848 // register-save area, since GP slots are 8-byte packed, so the address of
2849 // its low eightbyte is already the address of the whole value. A pure
2850 // SSE pair or a mixed pair is not contiguous, since SSE slots are 16-byte
2851 // spaced and a mixed pair's halves live in disjoint areas, so each half is
2852 // copied into a temp laid out as the coerced pair.
2853 bool pairNeedsReassembly = isRegPair && neededSse != 0;
2854
2855 // Both temps are allocated here, since a ternary arm yields their address.
2856 mlir::Value regPairTemp;
2857 if (pairNeedsReassembly) {
2858 // The temp is written one member at a time through the coerced pair, so
2859 // it has to meet that type's alignment, and read back as the result, so
2860 // it has to meet the result's alignment too.
2861 regPairTemp = builder.createAlloca(
2862 loc, addrType: builder.getPointerTo(ty: ac.coercedType), name: "vaarg.reg",
2863 alignment: clang::CharUnits::fromQuantity(
2864 Quantity: std::max(a: cirDataLayout.getTypeLanguageAlignment(ty: ac.coercedType),
2865 b: argumentAreaAlign(ty: resultTy, modOp: module))));
2866 }
2867
2868 mlir::Value regTemp;
2869 bool copyThroughTemp =
2870 !pairNeedsReassembly && needsTempCopy(f: fetch, neededInt, neededSse);
2871 if (copyThroughTemp) {
2872 regTemp = builder.createAlloca(
2873 loc, addrType: builder.getPointerTo(ty: resultTy), name: "vaarg.reg",
2874 alignment: clang::CharUnits::fromQuantity(Quantity: argumentAreaAlign(ty: resultTy, modOp: module)));
2875 }
2876
2877 addr = cir::TernaryOp::create(
2878 builder, location: loc, cond: cursor.inRegs,
2879 /*trueBuilder=*/
2880 [&](mlir::OpBuilder &ob, mlir::Location l) {
2881 CIRBaseBuilderTy b(ob);
2882 mlir::Value regSaveArea = b.createLoad(
2883 loc: l, ptr: b.createGetMember(loc: l, resultTy: b.getPointerTo(ty: fetch.vaFields[3]),
2884 base: valist, name: "reg_save_area", index: 3));
2885 regSaveArea = b.createPtrBitcast(src: regSaveArea, newPointeeTy: byteTy);
2886
2887 mlir::Value regAddr;
2888 if (pairNeedsReassembly) {
2889 reassembleRegisterPair(b, loc: l, ac, cursor, pairIsSse,
2890 regSaveArea, regPairTemp);
2891 regAddr = b.createPtrBitcast(src: regPairTemp, newPointeeTy: byteTy);
2892 } else {
2893 mlir::Value off =
2894 neededSse ? cursor.fpOffset : cursor.gpOffset;
2895 regAddr = b.createPtrStride(loc: l, base: regSaveArea, stride: off);
2896 if (copyThroughTemp) {
2897 regAddr = copyRegisterToTemp(b, loc: l, f: fetch, regAddr, temp: regTemp,
2898 neededInt, neededSse);
2899 }
2900 }
2901
2902 advanceRegisterCursors(b, loc: l, cursor, neededInt, neededSse);
2903 cir::YieldOp::create(builder&: b, location: l, args: regAddr);
2904 },
2905 /*falseBuilder=*/
2906 [&](mlir::OpBuilder &ob, mlir::Location l) {
2907 CIRBaseBuilderTy b(ob);
2908 mlir::Value memAddr = buildOverflowAddrAndAdvance(b, f: fetch);
2909 cir::YieldOp::create(builder&: b, location: l, args: memAddr);
2910 })
2911 .getResult();
2912 }
2913
2914 // Every path above places the result at least this well aligned.
2915 mlir::Value resultAddr = builder.createPtrBitcast(src: addr, newPointeeTy: resultTy);
2916 uint64_t resultAlign = argumentAreaAlign(ty: resultTy, modOp: module);
2917 if (mlir::isa<cir::RecordType>(Val: resultTy)) {
2918 SmallPtrSet<mlir::Operation *, 1> noExceptions;
2919 if (failed(Result: rewireRecordUses(builder, loc, val: op.getResult(), addr: resultAddr,
2920 addrAlign: resultAlign, dl, exceptions: noExceptions)))
2921 return mlir::failure();
2922 } else {
2923 op.getResult().replaceAllUsesWith(
2924 newValue: builder.createAlignedLoad(loc, ptr: resultAddr, alignment: resultAlign));
2925 }
2926 op->erase();
2927 return mlir::success();
2928}
2929