1//===- CallConvLoweringPass.cpp - Lower CIR to ABI calling convention ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass walks every cir.func and cir.call in the module, computes a
10// FunctionClassification for it (via either an ABI target or a pre-built
11// classification injected as a function attribute), and dispatches to
12// CIRABIRewriteContext to perform the actual IR rewriting.
13//
14// Two driver modes (mutually exclusive):
15//
16// target=test
17// Use the MLIR test ABI target (mlir/lib/ABI/Targets/Test/) to classify
18// each function. Predictable rules that approximate x86_64 SysV. Real
19// targets (x86_64, AArch64) will be added once the LLVM ABI library
20// ships them.
21//
22// classification-attr=<name>
23// Read a DictionaryAttr named <name> from each cir.func and parse it via
24// mlir::abi::test::parseClassificationAttr. Used by tests to inject any
25// classification (including shapes the test target itself does not
26// produce) without depending on a real ABI target.
27//
28// The pass requires a `dlti.dl_spec` attribute on the module so the
29// classifier can query type sizes and alignments.
30//
31//===----------------------------------------------------------------------===//
32
33#include "CIRABIClassifier.h"
34#include "PassDetail.h"
35#include "TargetLowering/CIRABIRewriteContext.h"
36
37#include "mlir/ABI/ABIRewriteContext.h"
38#include "mlir/Dialect/DLTI/DLTI.h"
39#include "mlir/IR/Builders.h"
40#include "mlir/IR/BuiltinOps.h"
41#include "mlir/IR/SymbolTable.h"
42#include "mlir/Interfaces/DataLayoutInterfaces.h"
43#include "mlir/Pass/Pass.h"
44#include "clang/CIR/Dialect/IR/CIRDialect.h"
45#include "clang/CIR/Dialect/Passes.h"
46#include "llvm/ABI/TargetInfo.h"
47#include "llvm/ADT/ScopeExit.h"
48
49#include <algorithm>
50
51using namespace mlir;
52using namespace mlir::abi;
53using namespace cir;
54
55namespace mlir {
56#define GEN_PASS_DEF_CALLCONVLOWERING
57#include "clang/CIR/Dialect/Passes.h.inc"
58} // namespace mlir
59
60/// Whether \p ty, or an aggregate member/element reached by value (never
61/// through a pointer), is an incomplete record. Such a record has no known
62/// layout, so no classification can be built for it.
63static bool hasIncompleteRecordByValue(mlir::Type ty) {
64 if (auto recTy = dyn_cast<cir::RecordType>(Val&: ty))
65 return !recTy.isComplete() ||
66 llvm::any_of(Range: recTy.getMembers(), P: hasIncompleteRecordByValue);
67 if (auto arrTy = dyn_cast<cir::ArrayType>(Val&: ty))
68 return hasIncompleteRecordByValue(ty: arrTy.getElementType());
69 return false;
70}
71
72/// Whether \p fc gives the callee access to memory through a pointer the ABI
73/// introduced: an sret slot for an indirect return, or an indirect or byref
74/// argument.
75static bool hasIndirectSlot(const FunctionClassification &fc) {
76 if (fc.returnInfo.kind == ArgKind::Indirect)
77 return true;
78 return llvm::any_of(Range: fc.argInfos, P: [](const ArgClassification &ac) {
79 return ac.kind == ArgKind::Indirect || ac.kind == ArgKind::IndirectAliased;
80 });
81}
82
83/// Record on \p op that the memory its pointer arguments address may be read
84/// and written. Absent effects already allow that, since an absent attribute
85/// means the effects are unknown, so they are left absent.
86template <typename OpTy> static void widenForArgMemory(OpTy op) {
87 cir::MemoryEffectsAttr effects = op.getMemoryEffectsAttr();
88 if (!effects)
89 return;
90 op.setMemoryEffectsAttr(cir::MemoryEffectsAttr::get(
91 context: effects.getContext(), other: effects.getOther(), arg_mem: cir::ModRefInfo::ModRef,
92 inaccessible_mem: effects.getInaccessibleMem(), errno_mem: effects.getErrnoMem(),
93 target_mem0: effects.getTargetMem0(), target_mem1: effects.getTargetMem1()));
94}
95
96#ifndef NDEBUG
97/// Whether \p callFc classifies a call's leading arguments and its return
98/// exactly as \p calleeFc classifies the callee's declared parameters and
99/// return. A function definition and its call sites are rewritten from
100/// separate classifications, so the two would silently disagree on the wire
101/// format if an ellipsis argument could ever change how a declared parameter
102/// is passed.
103static bool classifiesSamePrefix(const FunctionClassification &calleeFc,
104 const FunctionClassification &callFc) {
105 if (callFc.argInfos.size() < calleeFc.argInfos.size())
106 return false;
107 return calleeFc.returnInfo == callFc.returnInfo &&
108 std::equal(calleeFc.argInfos.begin(), calleeFc.argInfos.end(),
109 callFc.argInfos.begin());
110}
111#endif
112
113namespace {
114
115struct CallConvLoweringPass
116 : public impl::CallConvLoweringBase<CallConvLoweringPass> {
117 using CallConvLoweringBase::CallConvLoweringBase;
118
119 CallConvLoweringPass(const CallConvLoweringOptions &options,
120 const llvm::abi::X86ABICompatInfo &x86AbiCompat)
121 : CallConvLoweringBase(options), x86AbiCompat(x86AbiCompat) {}
122
123 void runOnOperation() override;
124
125 /// The x86_64 flags whose value depends on the target and the requested ABI
126 /// compatibility version. Carried outside the pass options because the
127 /// struct has no command-line parser, so a cir-opt run gets the library
128 /// defaults rather than a target's values.
129 llvm::abi::X86ABICompatInfo x86AbiCompat;
130};
131
132} // namespace
133
134/// Build the classifier the pass options select. Exactly one of \p target
135/// and \p classificationAttrName is set, which runOnOperation checks before
136/// calling.
137static std::unique_ptr<CIRABIClassifier>
138createABIClassifier(cir::CallConvTarget target,
139 StringRef classificationAttrName, const DataLayout &dl,
140 ModuleOp modOp, const ABIClassifierOptions &options) {
141 if (!classificationAttrName.empty())
142 return createInjectedABIClassifier(dl, attrName: classificationAttrName);
143
144 switch (target) {
145 case cir::CallConvTarget::Test:
146 return createTestTargetABIClassifier(dl);
147 case cir::CallConvTarget::X86_64:
148 return createX86_64ABIClassifier(dl, modOp, options);
149 case cir::CallConvTarget::AMDGPU:
150 return createAMDGPUABIClassifier(dl, modOp);
151 case cir::CallConvTarget::SPIRV:
152 return createSPIRVABIClassifier(dl, modOp);
153 case cir::CallConvTarget::None:
154 break;
155 }
156 llvm_unreachable("an unset driver mode is rejected before this point");
157}
158
159namespace {
160
161/// Find the cir.func declaration matching a direct cir.call / cir.try_call
162/// callee, if any. Returns nullptr if the callee is indirect or the symbol
163/// cannot be resolved. Takes a SymbolTable instead of a ModuleOp so the
164/// symbol lookup is amortized across all the call sites the driver walks
165/// (ModuleOp::lookupSymbol is linear per call).
166cir::FuncOp lookupCallee(Operation *callOp, SymbolTable &symbolTable) {
167 FlatSymbolRefAttr callee;
168 if (auto call = dyn_cast<cir::CallOp>(Val: callOp))
169 callee = call.getCalleeAttr();
170 else if (auto tryCall = dyn_cast<cir::TryCallOp>(Val: callOp))
171 callee = tryCall.getCalleeAttr();
172 else
173 return nullptr;
174 if (!callee)
175 return nullptr;
176 return symbolTable.lookup<cir::FuncOp>(name: callee.getValue());
177}
178
179void CallConvLoweringPass::runOnOperation() {
180 ModuleOp moduleOp = getOperation();
181 MLIRContext *ctx = &getContext();
182
183 bool haveTarget = target != cir::CallConvTarget::None;
184 bool haveAttr = !classificationAttr.empty();
185 if (haveTarget == haveAttr) {
186 moduleOp.emitOpError() << "CallConvLowering requires exactly one of "
187 "'target' or 'classification-attr' pass options";
188 signalPassFailure();
189 return;
190 }
191
192 if (!moduleOp->hasAttr(name: DLTIDialect::kDataLayoutAttrName)) {
193 moduleOp.emitOpError()
194 << "CallConvLowering requires a DataLayout (dlti.dl_spec attribute "
195 "on the module)";
196 signalPassFailure();
197 return;
198 }
199
200 // A module without #cir.lowering_options gets the CodeGenOptions
201 // default, which marks arguments noundef.
202 auto loweringOpts = moduleOp->getAttrOfType<cir::LoweringOptionsAttr>(
203 name: cir::CIRDialect::getLoweringOptionsAttrName());
204 bool enableNoundefAttrs =
205 !loweringOpts || loweringOpts.getEnableNoundefAttrs();
206
207 DataLayout dl(moduleOp);
208 CIRABIRewriteContext rewriteCtx(moduleOp, dl, enableNoundefAttrs);
209 // A non-byval indirect parameter's slot outlives the rewrite that retypes
210 // the parameter, so that a call forwarding the parameter can still recognize
211 // it. Draining on scope exit collapses those slots whichever way this
212 // function returns.
213 llvm::scope_exit drainParamSlots(
214 [&] { rewriteCtx.finalizeParameterSlots(); });
215 SymbolTable symbolTable(moduleOp);
216
217 cir::ABIClassifierOptions classifierOptions;
218 classifierOptions.x86AvxAbiLevel = x86AvxAbiLevel;
219 classifierOptions.allowsX86TargetAttrAvx = allowsX86TargetAttrAvx;
220 classifierOptions.x86AbiCompat = x86AbiCompat;
221 std::unique_ptr<cir::CIRABIClassifier> classifier = createABIClassifier(
222 target, classificationAttrName: classificationAttr, dl, modOp: moduleOp, options: classifierOptions);
223
224 // Classify every cir.func up front. No IR mutation happens here, so
225 // later walks can consult any function's classification regardless of
226 // visitation order.
227 llvm::MapVector<cir::FuncOp, FunctionClassification> classifications;
228 bool anyFailed = false;
229 moduleOp.walk(callback: [&](cir::FuncOp f) {
230 // A lowered signature uses ordinary CIR types and cannot be told apart
231 // from one that still needs lowering, so only a function that says it
232 // needs lowering is lowered.
233 if (!f.getRequiresCallConvLowering())
234 return;
235 // A complete type is required at any call or definition, so only a
236 // declaration can carry an incomplete-by-value parameter or return type,
237 // and no translation unit can ever call or define it with real argument
238 // data. Leave it unclassified.
239 cir::FuncType fnTy = f.getFunctionType();
240 if (f.isDeclaration() &&
241 (hasIncompleteRecordByValue(ty: fnTy.getReturnType()) ||
242 llvm::any_of(Range: fnTy.getInputs(), P: hasIncompleteRecordByValue)))
243 return;
244 mlir::FailureOr<FunctionClassification> fc =
245 classifier->classifyFunction(func: f);
246 if (failed(Result: fc)) {
247 anyFailed = true;
248 return;
249 }
250 classifications.insert(KV: {f, std::move(*fc)});
251 });
252 if (anyFailed) {
253 signalPassFailure();
254 return;
255 }
256
257 // Build a callee-to-callers index. One module walk collects every direct
258 // cir.call / cir.try_call to each cir.func; the loop below rewrites a
259 // function and all of its call sites together. Indirect or unresolved
260 // callees are skipped here; rewriteCallSite errors on those at the end.
261 //
262 // A call that passes arguments through an ellipsis gets its own
263 // classification, recorded here while every signature is still in its
264 // original form. The callee's classification covers only its declared
265 // parameters and cannot describe those extra arguments.
266 llvm::DenseMap<cir::FuncOp, SmallVector<Operation *>> callers;
267 // Keyed on the call op collected below, looked up once when that same op is
268 // rewritten. A key must never come from an op created during the rewrite:
269 // a recycled address could match an unrelated entry.
270 llvm::DenseMap<Operation *, FunctionClassification> variadicCallSites;
271 moduleOp.walk(callback: [&](Operation *op) {
272 auto call = dyn_cast<cir::CIRCallOpInterface>(Val: op);
273 if (!call)
274 return;
275 cir::FuncOp callee = lookupCallee(callOp: op, symbolTable);
276 if (!callee || !callee.getRequiresCallConvLowering())
277 return;
278 callers[callee].push_back(Elt: op);
279
280 cir::FuncType calleeTy = callee.getFunctionType();
281 if (call.getNumArgOperands() <= calleeTy.getNumInputs())
282 return;
283 // A callee declared without a prototype also takes more operands than it
284 // declares, and the verifier allows it. Those extra arguments are named
285 // rather than passed through an ellipsis, so the accounting below does not
286 // describe them.
287 if (!calleeTy.isVarArg()) {
288 op->emitOpError() << "extra arguments to a callee without a prototype "
289 "not yet implemented in CallConvLowering";
290 anyFailed = true;
291 return;
292 }
293 mlir::FailureOr<FunctionClassification> fc =
294 classifier->classifyVariadicCall(call, callee);
295 if (failed(Result: fc)) {
296 anyFailed = true;
297 return;
298 }
299 variadicCallSites.insert(KV: {op, std::move(*fc)});
300 });
301 if (anyFailed) {
302 signalPassFailure();
303 return;
304 }
305
306 // A cir.get_global holding a function's address carries the signature the
307 // source wrote, which the verifier ties to the callee, so it goes stale when
308 // that callee is rewritten. A function address in a global initializer is a
309 // GlobalViewAttr instead, whose recorded type the verifier does not tie to
310 // the callee and which is dropped at opaque-pointer lowering, so it needs no
311 // counterpart.
312 llvm::DenseMap<cir::FuncOp, SmallVector<cir::GetGlobalOp>> addressTakers;
313 moduleOp.walk(callback: [&](cir::GetGlobalOp getGlobal) {
314 auto ptrTy = cast<cir::PointerType>(Val: getGlobal.getAddr().getType());
315 if (!isa<cir::FuncType>(Val: ptrTy.getPointee()))
316 return;
317 // A get_global's pointee must equal the named symbol's type, and the
318 // GlobalOp verifier rejects a function type there, so this names a
319 // cir.func.
320 auto callee = cast<cir::FuncOp>(Val: symbolTable.lookup(name: getGlobal.getName()));
321 addressTakers[callee].push_back(Elt: getGlobal);
322 });
323
324 // Restate every non-byval indirect parameter's slot alignment and route any
325 // use of such a parameter as a call argument through a load of that slot,
326 // before any definition or call site is rewritten. Doing this up front
327 // makes the forwarding decision independent of the order the callee and its
328 // caller were declared in.
329 for (auto &kv : classifications) {
330 if (failed(Result: rewriteCtx.prepareNonByvalParameters(funcOp: kv.first, fc: kv.second))) {
331 signalPassFailure();
332 return;
333 }
334 }
335
336 // An sret slot or an indirect argument is a pointer the source never
337 // wrote, and an ellipsis can carry one the declared parameters do not
338 // describe. The callee reads and writes that memory whatever const or
339 // pure recorded about it, so the recorded effects have to widen.
340 for (auto &kv : classifications) {
341 cir::FuncOp func = kv.first;
342 if (hasIndirectSlot(fc: kv.second) || func.getFunctionType().isVarArg())
343 widenForArgMemory(op: func);
344 }
345
346 // Rewrite each function together with every direct call to it and every op
347 // holding its address. By the time we move on to function F+1, F's
348 // signature and every reference to F have already been brought into
349 // alignment, and F+1..FN are still in their original (mutually consistent)
350 // form, so the IR is verifier-clean at every outer-iteration boundary.
351 //
352 // There is still a brief inner window where F's signature has been
353 // rewritten but its references have not yet caught up -- we have no way to
354 // mutate both sides of a call atomically. No verifier runs inside the
355 // pass, and at pass exit the module is verifier-clean. Fusing the inner
356 // loops here keeps the invalid window per-function rather than module-wide.
357 OpBuilder builder(ctx);
358 for (auto &kv : classifications) {
359 cir::FuncOp func = kv.first;
360 const FunctionClassification &fc = kv.second;
361 if (failed(Result: rewriteCtx.rewriteFunctionDefinition(funcOp: func, fc, builder))) {
362 signalPassFailure();
363 return;
364 }
365 for (Operation *callOp : callers.lookup(Val: func)) {
366 const FunctionClassification *callFc = &fc;
367 if (auto it = variadicCallSites.find(Val: callOp);
368 it != variadicCallSites.end()) {
369 callFc = &it->second;
370 assert(classifiesSamePrefix(fc, *callFc) &&
371 "a call site's declared parameters must be classified the same "
372 "way as the callee's");
373 }
374 if (hasIndirectSlot(fc: *callFc))
375 widenForArgMemory(op: cast<cir::CIRCallOpInterface>(Val: callOp));
376 if (failed(Result: rewriteCtx.rewriteCallSite(callOp, fc: *callFc, builder))) {
377 signalPassFailure();
378 return;
379 }
380 }
381 for (cir::GetGlobalOp addrOp : addressTakers.lookup(Val: func))
382 rewriteCtx.rewriteFunctionAddress(addrOp, funcOp: func, builder);
383 }
384
385 // Rewrite indirect call sites. The callee is opaque, so classify from the
386 // function pointer's pointee FuncType and let rewriteCallSite retype the
387 // callee pointer to match the coerced signature. Collect the calls first:
388 // when an sret rewrite reuses a single-use store's destination as the return
389 // slot it erases that store, which is the operation a live walk has already
390 // cached as the next one to visit.
391 SmallVector<cir::CIRCallOpInterface> indirectCalls;
392 // An indirect call has no callee to say whether it is lowered, but it is
393 // lowered together with the function it is in.
394 auto inFunctionToLower = [](Operation *op) {
395 auto func = op->getParentOfType<cir::FuncOp>();
396 return func && func.getRequiresCallConvLowering();
397 };
398 moduleOp.walk(callback: [&](cir::CIRCallOpInterface c) {
399 if (c.isIndirect() && inFunctionToLower(c))
400 indirectCalls.push_back(Elt: c);
401 });
402 for (cir::CIRCallOpInterface c : indirectCalls) {
403 mlir::FailureOr<FunctionClassification> fc =
404 classifier->classifyIndirectCall(call: c);
405 if (failed(Result: fc)) {
406 signalPassFailure();
407 return;
408 }
409 if (hasIndirectSlot(fc: *fc))
410 widenForArgMemory(op: c);
411 if (failed(Result: rewriteCtx.rewriteCallSite(callOp: c.getOperation(), fc: *fc, builder))) {
412 signalPassFailure();
413 return;
414 }
415 }
416
417 // Collect the fetches before rewriting any, because rewriting one erases
418 // it, and a live walk holds the op it is about to visit next.
419 SmallVector<cir::VAArgOp> vaArgs;
420 moduleOp.walk(callback: [&](cir::VAArgOp v) {
421 if (inFunctionToLower(v))
422 vaArgs.push_back(Elt: v);
423 });
424 for (cir::VAArgOp v : vaArgs) {
425 mlir::FailureOr<ArgClassification> ac = classifier->classifyVAArg(vaArg: v);
426 if (failed(Result: ac)) {
427 signalPassFailure();
428 return;
429 }
430 if (failed(Result: rewriteCtx.rewriteVAArg(vaArgOp: v.getOperation(), ac: *ac, builder))) {
431 signalPassFailure();
432 return;
433 }
434 }
435
436 // Everything in and around these functions now uses the target ABI.
437 for (auto &kv : classifications)
438 kv.first.setRequiresCallConvLowering(false);
439}
440
441} // namespace
442
443std::unique_ptr<Pass> mlir::createCallConvLoweringPass() {
444 return std::make_unique<CallConvLoweringPass>();
445}
446
447std::unique_ptr<Pass> mlir::createCallConvLoweringPass(
448 cir::CallConvTarget target, llvm::abi::X86AVXABILevel x86AvxAbiLevel,
449 bool allowsX86TargetAttrAvx,
450 const llvm::abi::X86ABICompatInfo &x86AbiCompat) {
451 CallConvLoweringOptions options;
452 options.target = target;
453 options.x86AvxAbiLevel = x86AvxAbiLevel;
454 options.allowsX86TargetAttrAvx = allowsX86TargetAttrAvx;
455 return std::make_unique<CallConvLoweringPass>(args&: options, args: x86AbiCompat);
456}
457