1//===-- AArch64Arm64ECCallLowering.cpp - Lower Arm64EC calls ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file contains the IR transform to lower external or indirect calls for
11/// the ARM64EC calling convention. Such calls must go through the runtime, so
12/// we can translate the calling convention for calls into the emulator.
13///
14/// This subsumes Control Flow Guard handling.
15///
16//===----------------------------------------------------------------------===//
17
18#include "AArch64.h"
19#include "llvm/ADT/SetVector.h"
20#include "llvm/ADT/SmallString.h"
21#include "llvm/ADT/SmallVector.h"
22#include "llvm/ADT/Statistic.h"
23#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/GlobalAlias.h"
25#include "llvm/IR/IRBuilder.h"
26#include "llvm/IR/Instruction.h"
27#include "llvm/IR/Mangler.h"
28#include "llvm/IR/Module.h"
29#include "llvm/Object/COFF.h"
30#include "llvm/Pass.h"
31#include "llvm/Support/CommandLine.h"
32#include "llvm/TargetParser/Triple.h"
33
34using namespace llvm;
35using namespace llvm::COFF;
36
37using OperandBundleDef = OperandBundleDefT<Value *>;
38
39#define DEBUG_TYPE "arm64eccalllowering"
40
41STATISTIC(Arm64ECCallsLowered, "Number of Arm64EC calls lowered");
42
43static cl::opt<bool> LowerDirectToIndirect("arm64ec-lower-direct-to-indirect",
44 cl::Hidden, cl::init(Val: true));
45static cl::opt<bool> GenerateThunks("arm64ec-generate-thunks", cl::Hidden,
46 cl::init(Val: true));
47
48namespace {
49
50enum ThunkArgTranslation : uint8_t {
51 Direct,
52 Bitcast,
53 PointerIndirection,
54};
55
56struct ThunkArgInfo {
57 Type *Arm64Ty;
58 Type *X64Ty;
59 ThunkArgTranslation Translation;
60};
61
62class AArch64Arm64ECCallLowering : public ModulePass {
63public:
64 static char ID;
65 AArch64Arm64ECCallLowering() : ModulePass(ID) {}
66
67 Function *buildExitThunk(FunctionType *FnTy, AttributeList Attrs);
68 Function *buildEntryThunk(Function *F);
69 void lowerCall(CallBase *CB);
70 Function *buildGuestExitThunk(Function *F);
71 Function *buildPatchableThunk(GlobalAlias *UnmangledAlias,
72 GlobalAlias *MangledAlias);
73 bool processFunction(Function &F, SetVector<GlobalValue *> &DirectCalledFns,
74 DenseMap<GlobalAlias *, GlobalAlias *> &FnsMap);
75 bool runOnModule(Module &M) override;
76
77private:
78 ControlFlowGuardMode CFGuardModuleFlag = ControlFlowGuardMode::Disabled;
79 FunctionType *GuardFnType = nullptr;
80 FunctionType *DispatchFnType = nullptr;
81 Constant *GuardFnCFGlobal = nullptr;
82 Constant *GuardFnGlobal = nullptr;
83 Constant *DispatchFnGlobal = nullptr;
84 Module *M = nullptr;
85
86 Type *PtrTy;
87 Type *I64Ty;
88 Type *VoidTy;
89
90 void getThunkType(FunctionType *FT, AttributeList AttrList,
91 Arm64ECThunkType TT, raw_ostream &Out,
92 FunctionType *&Arm64Ty, FunctionType *&X64Ty,
93 SmallVector<ThunkArgTranslation> &ArgTranslations);
94 void getThunkRetType(FunctionType *FT, AttributeList AttrList,
95 raw_ostream &Out, Type *&Arm64RetTy, Type *&X64RetTy,
96 SmallVectorImpl<Type *> &Arm64ArgTypes,
97 SmallVectorImpl<Type *> &X64ArgTypes,
98 SmallVector<ThunkArgTranslation> &ArgTranslations,
99 bool &HasSretPtr);
100 void getThunkArgTypes(FunctionType *FT, AttributeList AttrList,
101 Arm64ECThunkType TT, raw_ostream &Out,
102 SmallVectorImpl<Type *> &Arm64ArgTypes,
103 SmallVectorImpl<Type *> &X64ArgTypes,
104 SmallVectorImpl<ThunkArgTranslation> &ArgTranslations,
105 bool HasSretPtr);
106 ThunkArgInfo canonicalizeThunkType(Type *T, Align Alignment, bool Ret,
107 uint64_t ArgSizeBytes, raw_ostream &Out);
108};
109
110} // end anonymous namespace
111
112void AArch64Arm64ECCallLowering::getThunkType(
113 FunctionType *FT, AttributeList AttrList, Arm64ECThunkType TT,
114 raw_ostream &Out, FunctionType *&Arm64Ty, FunctionType *&X64Ty,
115 SmallVector<ThunkArgTranslation> &ArgTranslations) {
116 Out << (TT == Arm64ECThunkType::Entry ? "$ientry_thunk$cdecl$"
117 : "$iexit_thunk$cdecl$");
118
119 Type *Arm64RetTy;
120 Type *X64RetTy;
121
122 SmallVector<Type *> Arm64ArgTypes;
123 SmallVector<Type *> X64ArgTypes;
124
125 // The first argument to a thunk is the called function, stored in x9.
126 // For exit thunks, we pass the called function down to the emulator;
127 // for entry/guest exit thunks, we just call the Arm64 function directly.
128 if (TT == Arm64ECThunkType::Exit)
129 Arm64ArgTypes.push_back(Elt: PtrTy);
130 X64ArgTypes.push_back(Elt: PtrTy);
131
132 bool HasSretPtr = false;
133 getThunkRetType(FT, AttrList, Out, Arm64RetTy, X64RetTy, Arm64ArgTypes,
134 X64ArgTypes, ArgTranslations, HasSretPtr);
135
136 getThunkArgTypes(FT, AttrList, TT, Out, Arm64ArgTypes, X64ArgTypes,
137 ArgTranslations, HasSretPtr);
138
139 Arm64Ty = FunctionType::get(Result: Arm64RetTy, Params: Arm64ArgTypes, isVarArg: false);
140
141 X64Ty = FunctionType::get(Result: X64RetTy, Params: X64ArgTypes, isVarArg: false);
142}
143
144void AArch64Arm64ECCallLowering::getThunkArgTypes(
145 FunctionType *FT, AttributeList AttrList, Arm64ECThunkType TT,
146 raw_ostream &Out, SmallVectorImpl<Type *> &Arm64ArgTypes,
147 SmallVectorImpl<Type *> &X64ArgTypes,
148 SmallVectorImpl<ThunkArgTranslation> &ArgTranslations, bool HasSretPtr) {
149
150 Out << "$";
151 if (FT->isVarArg()) {
152 // We treat the variadic function's thunk as a normal function
153 // with the following type on the ARM side:
154 // rettype exitthunk(
155 // ptr x9, ptr x0, i64 x1, i64 x2, i64 x3, ptr x4, i64 x5)
156 //
157 // that can coverage all types of variadic function.
158 // x9 is similar to normal exit thunk, store the called function.
159 // x0-x3 is the arguments be stored in registers.
160 // x4 is the address of the arguments on the stack.
161 // x5 is the size of the arguments on the stack.
162 //
163 // On the x64 side, it's the same except that x5 isn't set.
164 //
165 // If both the ARM and X64 sides are sret, there are only three
166 // arguments in registers.
167 //
168 // If the X64 side is sret, but the ARM side isn't, we pass an extra value
169 // to/from the X64 side, and let SelectionDAG transform it into a memory
170 // location.
171 Out << "varargs";
172
173 // x0-x3
174 for (int i = HasSretPtr ? 1 : 0; i < 4; i++) {
175 Arm64ArgTypes.push_back(Elt: I64Ty);
176 X64ArgTypes.push_back(Elt: I64Ty);
177 ArgTranslations.push_back(Elt: ThunkArgTranslation::Direct);
178 }
179
180 // x4
181 Arm64ArgTypes.push_back(Elt: PtrTy);
182 X64ArgTypes.push_back(Elt: PtrTy);
183 ArgTranslations.push_back(Elt: ThunkArgTranslation::Direct);
184 // x5
185 Arm64ArgTypes.push_back(Elt: I64Ty);
186 if (TT != Arm64ECThunkType::Entry) {
187 // FIXME: x5 isn't actually used by the x64 side; revisit once we
188 // have proper isel for varargs
189 X64ArgTypes.push_back(Elt: I64Ty);
190 ArgTranslations.push_back(Elt: ThunkArgTranslation::Direct);
191 }
192 return;
193 }
194
195 unsigned I = 0;
196 if (HasSretPtr)
197 I++;
198
199 if (I == FT->getNumParams()) {
200 Out << "v";
201 return;
202 }
203
204 for (unsigned E = FT->getNumParams(); I != E; ++I) {
205#if 0
206 // FIXME: Need more information about argument size; see
207 // https://reviews.llvm.org/D132926
208 uint64_t ArgSizeBytes = AttrList.getParamArm64ECArgSizeBytes(I);
209 Align ParamAlign = AttrList.getParamAlignment(I).valueOrOne();
210#else
211 uint64_t ArgSizeBytes = 0;
212 Align ParamAlign = Align();
213#endif
214 auto [Arm64Ty, X64Ty, ArgTranslation] =
215 canonicalizeThunkType(T: FT->getParamType(i: I), Alignment: ParamAlign,
216 /*Ret*/ false, ArgSizeBytes, Out);
217 Arm64ArgTypes.push_back(Elt: Arm64Ty);
218 X64ArgTypes.push_back(Elt: X64Ty);
219 ArgTranslations.push_back(Elt: ArgTranslation);
220 }
221}
222
223void AArch64Arm64ECCallLowering::getThunkRetType(
224 FunctionType *FT, AttributeList AttrList, raw_ostream &Out,
225 Type *&Arm64RetTy, Type *&X64RetTy, SmallVectorImpl<Type *> &Arm64ArgTypes,
226 SmallVectorImpl<Type *> &X64ArgTypes,
227 SmallVector<ThunkArgTranslation> &ArgTranslations, bool &HasSretPtr) {
228 Type *T = FT->getReturnType();
229#if 0
230 // FIXME: Need more information about argument size; see
231 // https://reviews.llvm.org/D132926
232 uint64_t ArgSizeBytes = AttrList.getRetArm64ECArgSizeBytes();
233#else
234 int64_t ArgSizeBytes = 0;
235#endif
236 if (T->isVoidTy()) {
237 if (FT->getNumParams()) {
238 Attribute SRetAttr0 = AttrList.getParamAttr(ArgNo: 0, Kind: Attribute::StructRet);
239 Attribute InRegAttr0 = AttrList.getParamAttr(ArgNo: 0, Kind: Attribute::InReg);
240 Attribute SRetAttr1, InRegAttr1;
241 if (FT->getNumParams() > 1) {
242 // Also check the second parameter (for class methods, the first
243 // parameter is "this", and the second parameter is the sret pointer.)
244 // It doesn't matter which one is sret.
245 SRetAttr1 = AttrList.getParamAttr(ArgNo: 1, Kind: Attribute::StructRet);
246 InRegAttr1 = AttrList.getParamAttr(ArgNo: 1, Kind: Attribute::InReg);
247 }
248 if ((SRetAttr0.isValid() && InRegAttr0.isValid()) ||
249 (SRetAttr1.isValid() && InRegAttr1.isValid())) {
250 // sret+inreg indicates a call that returns a C++ class value. This is
251 // actually equivalent to just passing and returning a void* pointer
252 // as the first or second argument. Translate it that way, instead of
253 // trying to model "inreg" in the thunk's calling convention; this
254 // simplfies the rest of the code, and matches MSVC mangling.
255 Out << "i8";
256 Arm64RetTy = I64Ty;
257 X64RetTy = I64Ty;
258 return;
259 }
260 if (SRetAttr0.isValid()) {
261 // FIXME: Sanity-check the sret type; if it's an integer or pointer,
262 // we'll get screwy mangling/codegen.
263 // FIXME: For large struct types, mangle as an integer argument and
264 // integer return, so we can reuse more thunks, instead of "m" syntax.
265 // (MSVC mangles this case as an integer return with no argument, but
266 // that's a miscompile.)
267 Type *SRetType = SRetAttr0.getValueAsType();
268 Align SRetAlign = AttrList.getParamAlignment(ArgNo: 0).valueOrOne();
269 canonicalizeThunkType(T: SRetType, Alignment: SRetAlign, /*Ret*/ true, ArgSizeBytes,
270 Out);
271 Arm64RetTy = VoidTy;
272 X64RetTy = VoidTy;
273 Arm64ArgTypes.push_back(Elt: FT->getParamType(i: 0));
274 X64ArgTypes.push_back(Elt: FT->getParamType(i: 0));
275 ArgTranslations.push_back(Elt: ThunkArgTranslation::Direct);
276 HasSretPtr = true;
277 return;
278 }
279 }
280
281 Out << "v";
282 Arm64RetTy = VoidTy;
283 X64RetTy = VoidTy;
284 return;
285 }
286
287 auto info =
288 canonicalizeThunkType(T, Alignment: Align(), /*Ret*/ true, ArgSizeBytes, Out);
289 Arm64RetTy = info.Arm64Ty;
290 X64RetTy = info.X64Ty;
291 if (X64RetTy->isPointerTy()) {
292 // If the X64 type is canonicalized to a pointer, that means it's
293 // passed/returned indirectly. For a return value, that means it's an
294 // sret pointer.
295 X64ArgTypes.push_back(Elt: X64RetTy);
296 X64RetTy = VoidTy;
297 }
298}
299
300ThunkArgInfo AArch64Arm64ECCallLowering::canonicalizeThunkType(
301 Type *T, Align Alignment, bool Ret, uint64_t ArgSizeBytes,
302 raw_ostream &Out) {
303
304 auto direct = [](Type *T) {
305 return ThunkArgInfo{.Arm64Ty: T, .X64Ty: T, .Translation: ThunkArgTranslation::Direct};
306 };
307
308 auto bitcast = [this](Type *Arm64Ty, uint64_t SizeInBytes) {
309 return ThunkArgInfo{.Arm64Ty: Arm64Ty,
310 .X64Ty: llvm::Type::getIntNTy(C&: M->getContext(), N: SizeInBytes * 8),
311 .Translation: ThunkArgTranslation::Bitcast};
312 };
313
314 auto pointerIndirection = [this](Type *Arm64Ty) {
315 return ThunkArgInfo{.Arm64Ty: Arm64Ty, .X64Ty: PtrTy,
316 .Translation: ThunkArgTranslation::PointerIndirection};
317 };
318
319 if (T->isHalfTy()) {
320 // Prefix with `llvm` since MSVC doesn't specify `_Float16`
321 Out << "__llvm_h__";
322 return direct(T);
323 }
324
325 if (T->isFloatTy()) {
326 Out << "f";
327 return direct(T);
328 }
329
330 if (T->isDoubleTy()) {
331 Out << "d";
332 return direct(T);
333 }
334
335 if (T->isFloatingPointTy()) {
336 report_fatal_error(reason: "Only 16, 32, and 64 bit floating points are supported "
337 "for ARM64EC thunks");
338 }
339
340 auto &DL = M->getDataLayout();
341
342 if (auto *StructTy = dyn_cast<StructType>(Val: T))
343 if (StructTy->getNumElements() == 1)
344 T = StructTy->getElementType(N: 0);
345
346 if (T->isArrayTy()) {
347 Type *ElementTy = T->getArrayElementType();
348 uint64_t ElementCnt = T->getArrayNumElements();
349 uint64_t ElementSizePerBytes = DL.getTypeSizeInBits(Ty: ElementTy) / 8;
350 uint64_t TotalSizeBytes = ElementCnt * ElementSizePerBytes;
351 if (ElementTy->isHalfTy() || ElementTy->isFloatTy() ||
352 ElementTy->isDoubleTy()) {
353 if (ElementTy->isHalfTy())
354 // Prefix with `llvm` since MSVC doesn't specify `_Float16`
355 Out << "__llvm_H__";
356 else if (ElementTy->isFloatTy())
357 Out << "F";
358 else if (ElementTy->isDoubleTy())
359 Out << "D";
360 Out << TotalSizeBytes;
361 if (Alignment.value() >= 16 && !Ret)
362 Out << "a" << Alignment.value();
363 if (TotalSizeBytes <= 8) {
364 // Arm64 returns small structs of float/double in float registers;
365 // X64 uses RAX.
366 return bitcast(T, TotalSizeBytes);
367 } else {
368 // Struct is passed directly on Arm64, but indirectly on X64.
369 return pointerIndirection(T);
370 }
371 } else if (T->isFloatingPointTy()) {
372 report_fatal_error(
373 reason: "Only 16, 32, and 64 bit floating points are supported "
374 "for ARM64EC thunks");
375 }
376 }
377
378 if ((T->isIntegerTy() || T->isPointerTy()) && DL.getTypeSizeInBits(Ty: T) <= 64) {
379 Out << "i8";
380 return direct(I64Ty);
381 }
382
383 unsigned TypeSize = ArgSizeBytes;
384 if (TypeSize == 0)
385 TypeSize = DL.getTypeSizeInBits(Ty: T) / 8;
386 Out << "m";
387 if (TypeSize != 4)
388 Out << TypeSize;
389 if (Alignment.value() >= 16 && !Ret)
390 Out << "a" << Alignment.value();
391 // FIXME: Try to canonicalize Arm64Ty more thoroughly?
392 if (TypeSize == 1 || TypeSize == 2 || TypeSize == 4 || TypeSize == 8) {
393 // Pass directly in an integer register
394 return bitcast(T, TypeSize);
395 } else {
396 // Passed directly on Arm64, but indirectly on X64.
397 return pointerIndirection(T);
398 }
399}
400
401// This function builds the "exit thunk", a function which translates
402// arguments and return values when calling x64 code from AArch64 code.
403Function *AArch64Arm64ECCallLowering::buildExitThunk(FunctionType *FT,
404 AttributeList Attrs) {
405 SmallString<256> ExitThunkName;
406 llvm::raw_svector_ostream ExitThunkStream(ExitThunkName);
407 FunctionType *Arm64Ty, *X64Ty;
408 SmallVector<ThunkArgTranslation> ArgTranslations;
409 getThunkType(FT, AttrList: Attrs, TT: Arm64ECThunkType::Exit, Out&: ExitThunkStream, Arm64Ty,
410 X64Ty, ArgTranslations);
411 if (Function *F = M->getFunction(Name: ExitThunkName))
412 return F;
413
414 Function *F = Function::Create(Ty: Arm64Ty, Linkage: GlobalValue::LinkOnceODRLinkage, AddrSpace: 0,
415 N: ExitThunkName, M);
416 F->setCallingConv(CallingConv::ARM64EC_Thunk_Native);
417 F->setSection(".wowthk$aa");
418 F->setComdat(M->getOrInsertComdat(Name: ExitThunkName));
419 // Copy MSVC, and always set up a frame pointer. (Maybe this isn't necessary.)
420 F->addFnAttr(Kind: "frame-pointer", Val: "all");
421 // Only copy sret from the first argument. For C++ instance methods, clang can
422 // stick an sret marking on a later argument, but it doesn't actually affect
423 // the ABI, so we can omit it. This avoids triggering a verifier assertion.
424 if (FT->getNumParams()) {
425 auto SRet = Attrs.getParamAttr(ArgNo: 0, Kind: Attribute::StructRet);
426 auto InReg = Attrs.getParamAttr(ArgNo: 0, Kind: Attribute::InReg);
427 if (SRet.isValid() && !InReg.isValid())
428 F->addParamAttr(ArgNo: 1, Attr: SRet);
429 }
430 // FIXME: Copy anything other than sret? Shouldn't be necessary for normal
431 // C ABI, but might show up in other cases.
432 BasicBlock *BB = BasicBlock::Create(Context&: M->getContext(), Name: "", Parent: F);
433 IRBuilder<> IRB(BB);
434 Value *CalleePtr =
435 M->getOrInsertGlobal(Name: "__os_arm64x_dispatch_call_no_redirect", Ty: PtrTy);
436 Value *Callee = IRB.CreateLoad(Ty: PtrTy, Ptr: CalleePtr);
437 auto &DL = M->getDataLayout();
438 SmallVector<Value *> Args;
439
440 // Pass the called function in x9.
441 auto X64TyOffset = 1;
442 Args.push_back(Elt: F->arg_begin());
443
444 Type *RetTy = Arm64Ty->getReturnType();
445 if (RetTy != X64Ty->getReturnType()) {
446 // If the return type is an array or struct, translate it. Values of size
447 // 8 or less go into RAX; bigger values go into memory, and we pass a
448 // pointer.
449 if (DL.getTypeStoreSize(Ty: RetTy) > 8) {
450 Args.push_back(Elt: IRB.CreateAlloca(Ty: RetTy));
451 X64TyOffset++;
452 }
453 }
454
455 for (auto [Arg, X64ArgType, ArgTranslation] : llvm::zip_equal(
456 t: make_range(x: F->arg_begin() + 1, y: F->arg_end()),
457 u: make_range(x: X64Ty->param_begin() + X64TyOffset, y: X64Ty->param_end()),
458 args&: ArgTranslations)) {
459 // Translate arguments from AArch64 calling convention to x86 calling
460 // convention.
461 //
462 // For simple types, we don't need to do any translation: they're
463 // represented the same way. (Implicit sign extension is not part of
464 // either convention.)
465 //
466 // The big thing we have to worry about is struct types... but
467 // fortunately AArch64 clang is pretty friendly here: the cases that need
468 // translation are always passed as a struct or array. (If we run into
469 // some cases where this doesn't work, we can teach clang to mark it up
470 // with an attribute.)
471 //
472 // The first argument is the called function, stored in x9.
473 if (ArgTranslation != ThunkArgTranslation::Direct) {
474 Value *Mem = IRB.CreateAlloca(Ty: Arg.getType());
475 IRB.CreateStore(Val: &Arg, Ptr: Mem);
476 if (ArgTranslation == ThunkArgTranslation::Bitcast) {
477 Type *IntTy = IRB.getIntNTy(N: DL.getTypeStoreSizeInBits(Ty: Arg.getType()));
478 Args.push_back(Elt: IRB.CreateLoad(Ty: IntTy, Ptr: Mem));
479 } else {
480 assert(ArgTranslation == ThunkArgTranslation::PointerIndirection);
481 Args.push_back(Elt: Mem);
482 }
483 } else {
484 Args.push_back(Elt: &Arg);
485 }
486 assert(Args.back()->getType() == X64ArgType);
487 }
488 // FIXME: Transfer necessary attributes? sret? anything else?
489
490 CallInst *Call = IRB.CreateCall(FTy: X64Ty, Callee, Args);
491 Call->setCallingConv(CallingConv::ARM64EC_Thunk_X64);
492
493 Value *RetVal = Call;
494 if (RetTy != X64Ty->getReturnType()) {
495 // If we rewrote the return type earlier, convert the return value to
496 // the proper type.
497 if (DL.getTypeStoreSize(Ty: RetTy) > 8) {
498 RetVal = IRB.CreateLoad(Ty: RetTy, Ptr: Args[1]);
499 } else {
500 Value *CastAlloca = IRB.CreateAlloca(Ty: RetTy);
501 IRB.CreateStore(Val: Call, Ptr: CastAlloca);
502 RetVal = IRB.CreateLoad(Ty: RetTy, Ptr: CastAlloca);
503 }
504 }
505
506 if (RetTy->isVoidTy())
507 IRB.CreateRetVoid();
508 else
509 IRB.CreateRet(V: RetVal);
510 return F;
511}
512
513// This function builds the "entry thunk", a function which translates
514// arguments and return values when calling AArch64 code from x64 code.
515Function *AArch64Arm64ECCallLowering::buildEntryThunk(Function *F) {
516 SmallString<256> EntryThunkName;
517 llvm::raw_svector_ostream EntryThunkStream(EntryThunkName);
518 FunctionType *Arm64Ty, *X64Ty;
519 SmallVector<ThunkArgTranslation> ArgTranslations;
520 getThunkType(FT: F->getFunctionType(), AttrList: F->getAttributes(),
521 TT: Arm64ECThunkType::Entry, Out&: EntryThunkStream, Arm64Ty, X64Ty,
522 ArgTranslations);
523 if (Function *F = M->getFunction(Name: EntryThunkName))
524 return F;
525
526 Function *Thunk = Function::Create(Ty: X64Ty, Linkage: GlobalValue::LinkOnceODRLinkage, AddrSpace: 0,
527 N: EntryThunkName, M);
528 Thunk->setCallingConv(CallingConv::ARM64EC_Thunk_X64);
529 Thunk->setSection(".wowthk$aa");
530 Thunk->setComdat(M->getOrInsertComdat(Name: EntryThunkName));
531 // Copy MSVC, and always set up a frame pointer. (Maybe this isn't necessary.)
532 Thunk->addFnAttr(Kind: "frame-pointer", Val: "all");
533
534 BasicBlock *BB = BasicBlock::Create(Context&: M->getContext(), Name: "", Parent: Thunk);
535 IRBuilder<> IRB(BB);
536
537 Type *RetTy = Arm64Ty->getReturnType();
538 Type *X64RetType = X64Ty->getReturnType();
539
540 bool TransformDirectToSRet = X64RetType->isVoidTy() && !RetTy->isVoidTy();
541 unsigned ThunkArgOffset = TransformDirectToSRet ? 2 : 1;
542 unsigned PassthroughArgSize =
543 (F->isVarArg() ? 5 : Thunk->arg_size()) - ThunkArgOffset;
544 assert(ArgTranslations.size() == (F->isVarArg() ? 5 : PassthroughArgSize));
545
546 // Translate arguments to call.
547 SmallVector<Value *> Args;
548 for (unsigned i = 0; i != PassthroughArgSize; ++i) {
549 Value *Arg = Thunk->getArg(i: i + ThunkArgOffset);
550 Type *ArgTy = Arm64Ty->getParamType(i);
551 ThunkArgTranslation ArgTranslation = ArgTranslations[i];
552 if (ArgTranslation != ThunkArgTranslation::Direct) {
553 // Translate array/struct arguments to the expected type.
554 if (ArgTranslation == ThunkArgTranslation::Bitcast) {
555 Value *CastAlloca = IRB.CreateAlloca(Ty: ArgTy);
556 IRB.CreateStore(Val: Arg, Ptr: CastAlloca);
557 Arg = IRB.CreateLoad(Ty: ArgTy, Ptr: CastAlloca);
558 } else {
559 assert(ArgTranslation == ThunkArgTranslation::PointerIndirection);
560 Arg = IRB.CreateLoad(Ty: ArgTy, Ptr: Arg);
561 }
562 }
563 assert(Arg->getType() == ArgTy);
564 Args.push_back(Elt: Arg);
565 }
566
567 if (F->isVarArg()) {
568 // The 5th argument to variadic entry thunks is used to model the x64 sp
569 // which is passed to the thunk in x4, this can be passed to the callee as
570 // the variadic argument start address after skipping over the 32 byte
571 // shadow store.
572
573 // The EC thunk CC will assign any argument marked as InReg to x4.
574 Thunk->addParamAttr(ArgNo: 5, Kind: Attribute::InReg);
575 Value *Arg = Thunk->getArg(i: 5);
576 Arg = IRB.CreatePtrAdd(Ptr: Arg, Offset: IRB.getInt64(C: 0x20));
577 Args.push_back(Elt: Arg);
578
579 // Pass in a zero variadic argument size (in x5).
580 Args.push_back(Elt: IRB.getInt64(C: 0));
581 }
582
583 // Call the function passed to the thunk.
584 Value *Callee = Thunk->getArg(i: 0);
585 CallInst *Call = IRB.CreateCall(FTy: Arm64Ty, Callee, Args);
586
587 auto SRetAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::StructRet);
588 auto InRegAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::InReg);
589 if (SRetAttr.isValid() && !InRegAttr.isValid()) {
590 Thunk->addParamAttr(ArgNo: 1, Attr: SRetAttr);
591 Call->addParamAttr(ArgNo: 0, Attr: SRetAttr);
592 }
593
594 Value *RetVal = Call;
595 if (TransformDirectToSRet) {
596 // The x64 side returns this value indirectly via a hidden pointer (sret).
597 // Mark the thunk's pointer arg with sret so that ISel saves it and copies
598 // it into x8 (RAX) on return, matching the x64 calling convention.
599 Thunk->addParamAttr(
600 ArgNo: 1, Attr: Attribute::getWithStructRetType(Context&: M->getContext(), Ty: RetTy));
601 IRB.CreateStore(Val: RetVal, Ptr: Thunk->getArg(i: 1));
602 } else if (X64RetType != RetTy) {
603 Value *CastAlloca = IRB.CreateAlloca(Ty: X64RetType);
604 IRB.CreateStore(Val: Call, Ptr: CastAlloca);
605 RetVal = IRB.CreateLoad(Ty: X64RetType, Ptr: CastAlloca);
606 }
607
608 // Return to the caller. Note that the isel has code to translate this
609 // "ret" to a tail call to __os_arm64x_dispatch_ret. (Alternatively, we
610 // could emit a tail call here, but that would require a dedicated calling
611 // convention, which seems more complicated overall.)
612 if (X64RetType->isVoidTy())
613 IRB.CreateRetVoid();
614 else
615 IRB.CreateRet(V: RetVal);
616
617 return Thunk;
618}
619
620std::optional<std::string> getArm64ECMangledFunctionName(GlobalValue &GV) {
621 if (!GV.hasName()) {
622 GV.setName("__unnamed");
623 }
624
625 return llvm::getArm64ECMangledFunctionName(Name: GV.getName());
626}
627
628// Builds the "guest exit thunk", a helper to call a function which may or may
629// not be an exit thunk. (We optimistically assume non-dllimport function
630// declarations refer to functions defined in AArch64 code; if the linker
631// can't prove that, we use this routine instead.)
632Function *AArch64Arm64ECCallLowering::buildGuestExitThunk(Function *F) {
633 llvm::raw_null_ostream NullThunkName;
634 FunctionType *Arm64Ty, *X64Ty;
635 SmallVector<ThunkArgTranslation> ArgTranslations;
636 getThunkType(FT: F->getFunctionType(), AttrList: F->getAttributes(),
637 TT: Arm64ECThunkType::GuestExit, Out&: NullThunkName, Arm64Ty, X64Ty,
638 ArgTranslations);
639 auto MangledName = getArm64ECMangledFunctionName(GV&: *F);
640 assert(MangledName && "Can't guest exit to function that's already native");
641 std::string ThunkName = *MangledName;
642 if (ThunkName[0] == '?' && ThunkName.find(s: "@") != std::string::npos) {
643 ThunkName.insert(pos: ThunkName.find(s: "@"), s: "$exit_thunk");
644 } else {
645 ThunkName.append(s: "$exit_thunk");
646 }
647 Function *GuestExit =
648 Function::Create(Ty: Arm64Ty, Linkage: GlobalValue::WeakODRLinkage, AddrSpace: 0, N: ThunkName, M);
649 GuestExit->setComdat(M->getOrInsertComdat(Name: ThunkName));
650 GuestExit->setSection(".wowthk$aa");
651 GuestExit->addMetadata(
652 Kind: "arm64ec_unmangled_name",
653 MD&: *MDNode::get(Context&: M->getContext(),
654 MDs: MDString::get(Context&: M->getContext(), Str: F->getName())));
655 GuestExit->setMetadata(
656 Kind: "arm64ec_ecmangled_name",
657 Node: MDNode::get(Context&: M->getContext(),
658 MDs: MDString::get(Context&: M->getContext(), Str: *MangledName)));
659 F->setMetadata(Kind: "arm64ec_hasguestexit", Node: MDNode::get(Context&: M->getContext(), MDs: {}));
660 BasicBlock *BB = BasicBlock::Create(Context&: M->getContext(), Name: "", Parent: GuestExit);
661 IRBuilder<> B(BB);
662
663 // Create new call instruction. The call check should always be a call,
664 // even if the original CallBase is an Invoke or CallBr instructio.
665 // This is treated as a direct call, so do not use GuardFnCFGlobal.
666 LoadInst *GuardCheckLoad = B.CreateLoad(Ty: PtrTy, Ptr: GuardFnGlobal);
667 Function *Thunk = buildExitThunk(FT: F->getFunctionType(), Attrs: F->getAttributes());
668 CallInst *GuardCheck = B.CreateCall(
669 FTy: GuardFnType, Callee: GuardCheckLoad, Args: {F, Thunk});
670 Value *GuardCheckDest = B.CreateExtractValue(Agg: GuardCheck, Idxs: 0);
671 Value *GuardFinalDest = B.CreateExtractValue(Agg: GuardCheck, Idxs: 1);
672
673 // Ensure that the first argument is passed in the correct register.
674 GuardCheck->setCallingConv(CallingConv::CFGuard_Check);
675
676 SmallVector<Value *> Args(llvm::make_pointer_range(Range: GuestExit->args()));
677 OperandBundleDef OB("cfguardtarget", GuardFinalDest);
678 CallInst *Call = B.CreateCall(FTy: Arm64Ty, Callee: GuardCheckDest, Args, OpBundles: OB);
679 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
680
681 if (Call->getType()->isVoidTy())
682 B.CreateRetVoid();
683 else
684 B.CreateRet(V: Call);
685
686 auto SRetAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::StructRet);
687 auto InRegAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::InReg);
688 if (SRetAttr.isValid() && !InRegAttr.isValid()) {
689 GuestExit->addParamAttr(ArgNo: 0, Attr: SRetAttr);
690 Call->addParamAttr(ArgNo: 0, Attr: SRetAttr);
691 }
692
693 return GuestExit;
694}
695
696Function *
697AArch64Arm64ECCallLowering::buildPatchableThunk(GlobalAlias *UnmangledAlias,
698 GlobalAlias *MangledAlias) {
699 llvm::raw_null_ostream NullThunkName;
700 FunctionType *Arm64Ty, *X64Ty;
701 Function *F = cast<Function>(Val: MangledAlias->getAliasee());
702 SmallVector<ThunkArgTranslation> ArgTranslations;
703 getThunkType(FT: F->getFunctionType(), AttrList: F->getAttributes(),
704 TT: Arm64ECThunkType::GuestExit, Out&: NullThunkName, Arm64Ty, X64Ty,
705 ArgTranslations);
706 std::string ThunkName(MangledAlias->getName());
707 if (ThunkName[0] == '?' && ThunkName.find(s: "@") != std::string::npos) {
708 ThunkName.insert(pos: ThunkName.find(s: "@"), s: "$hybpatch_thunk");
709 } else {
710 ThunkName.append(s: "$hybpatch_thunk");
711 }
712
713 Function *GuestExit =
714 Function::Create(Ty: Arm64Ty, Linkage: GlobalValue::WeakODRLinkage, AddrSpace: 0, N: ThunkName, M);
715 GuestExit->setComdat(M->getOrInsertComdat(Name: ThunkName));
716 GuestExit->setSection(".wowthk$aa");
717 BasicBlock *BB = BasicBlock::Create(Context&: M->getContext(), Name: "", Parent: GuestExit);
718 IRBuilder<> B(BB);
719
720 // Load the global symbol as a pointer to the check function.
721 LoadInst *DispatchLoad = B.CreateLoad(Ty: PtrTy, Ptr: DispatchFnGlobal);
722
723 // Create new dispatch call instruction.
724 Function *ExitThunk =
725 buildExitThunk(FT: F->getFunctionType(), Attrs: F->getAttributes());
726 CallInst *Dispatch =
727 B.CreateCall(FTy: DispatchFnType, Callee: DispatchLoad,
728 Args: {UnmangledAlias, ExitThunk, UnmangledAlias->getAliasee()});
729
730 // Ensure that the first arguments are passed in the correct registers.
731 Dispatch->setCallingConv(CallingConv::CFGuard_Check);
732
733 SmallVector<Value *> Args(llvm::make_pointer_range(Range: GuestExit->args()));
734 CallInst *Call = B.CreateCall(FTy: Arm64Ty, Callee: Dispatch, Args);
735 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
736
737 if (Call->getType()->isVoidTy())
738 B.CreateRetVoid();
739 else
740 B.CreateRet(V: Call);
741
742 auto SRetAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::StructRet);
743 auto InRegAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::InReg);
744 if (SRetAttr.isValid() && !InRegAttr.isValid()) {
745 GuestExit->addParamAttr(ArgNo: 0, Attr: SRetAttr);
746 Call->addParamAttr(ArgNo: 0, Attr: SRetAttr);
747 }
748
749 MangledAlias->setAliasee(GuestExit);
750 return GuestExit;
751}
752
753// Lower an indirect call with inline code.
754void AArch64Arm64ECCallLowering::lowerCall(CallBase *CB) {
755 IRBuilder<> B(CB);
756 Value *CalledOperand = CB->getCalledOperand();
757
758 // If the indirect call is called within catchpad or cleanuppad,
759 // we need to copy "funclet" bundle of the call.
760 SmallVector<llvm::OperandBundleDef, 1> Bundles;
761 if (auto Bundle = CB->getOperandBundle(ID: LLVMContext::OB_funclet))
762 Bundles.push_back(Elt: OperandBundleDef(*Bundle));
763
764 // Load the global symbol as a pointer to the check function.
765 Value *GuardFn;
766 if ((CFGuardModuleFlag == ControlFlowGuardMode::Enabled) &&
767 !CB->hasFnAttr(Kind: "guard_nocf"))
768 GuardFn = GuardFnCFGlobal;
769 else
770 GuardFn = GuardFnGlobal;
771 LoadInst *GuardCheckLoad = B.CreateLoad(Ty: PtrTy, Ptr: GuardFn);
772
773 // Create new call instruction. The CFGuard check should always be a call,
774 // even if the original CallBase is an Invoke or CallBr instruction.
775 Function *Thunk = buildExitThunk(FT: CB->getFunctionType(), Attrs: CB->getAttributes());
776 CallInst *GuardCheck =
777 B.CreateCall(FTy: GuardFnType, Callee: GuardCheckLoad, Args: {CalledOperand, Thunk},
778 OpBundles: Bundles);
779 Value *GuardCheckDest = B.CreateExtractValue(Agg: GuardCheck, Idxs: 0);
780 Value *GuardFinalDest = B.CreateExtractValue(Agg: GuardCheck, Idxs: 1);
781
782 // Ensure that the first argument is passed in the correct register.
783 GuardCheck->setCallingConv(CallingConv::CFGuard_Check);
784
785 // Update the call: set the callee, and add a bundle with the final
786 // destination,
787 CB->setCalledOperand(GuardCheckDest);
788 OperandBundleDef OB("cfguardtarget", GuardFinalDest);
789 auto *NewCall = CallBase::addOperandBundle(CB, ID: LLVMContext::OB_cfguardtarget,
790 OB, InsertPt: CB->getIterator());
791 NewCall->copyMetadata(SrcInst: *CB);
792 CB->replaceAllUsesWith(V: NewCall);
793 CB->eraseFromParent();
794}
795
796bool AArch64Arm64ECCallLowering::runOnModule(Module &Mod) {
797 if (!GenerateThunks)
798 return false;
799
800 M = &Mod;
801
802 // Check if this module has the cfguard flag and read its value.
803 CFGuardModuleFlag = M->getControlFlowGuardMode();
804
805 PtrTy = PointerType::getUnqual(C&: M->getContext());
806 I64Ty = Type::getInt64Ty(C&: M->getContext());
807 VoidTy = Type::getVoidTy(C&: M->getContext());
808
809 GuardFnType =
810 FunctionType::get(Result: StructType::get(elt1: PtrTy, elts: PtrTy), Params: {PtrTy, PtrTy}, isVarArg: false);
811 DispatchFnType = FunctionType::get(Result: PtrTy, Params: {PtrTy, PtrTy, PtrTy}, isVarArg: false);
812 GuardFnCFGlobal = M->getOrInsertGlobal(Name: "__os_arm64x_check_icall_cfg", Ty: PtrTy);
813 GuardFnGlobal = M->getOrInsertGlobal(Name: "__os_arm64x_check_icall", Ty: PtrTy);
814 DispatchFnGlobal = M->getOrInsertGlobal(Name: "__os_arm64x_dispatch_call", Ty: PtrTy);
815
816 // Mangle names of function aliases and add the alias name to
817 // arm64ec_unmangled_name metadata to ensure a weak anti-dependency symbol is
818 // emitted for the alias as well. Do this early, before handling
819 // hybrid_patchable functions, to avoid mangling their aliases.
820 for (GlobalAlias &A : Mod.aliases()) {
821 auto F = dyn_cast_or_null<Function>(Val: A.getAliaseeObject());
822 if (!F)
823 continue;
824 if (std::optional<std::string> MangledName =
825 getArm64ECMangledFunctionName(GV&: A)) {
826 F->addMetadata(Kind: "arm64ec_unmangled_name",
827 MD&: *MDNode::get(Context&: M->getContext(),
828 MDs: MDString::get(Context&: M->getContext(), Str: A.getName())));
829 A.setName(MangledName.value());
830 }
831 }
832
833 DenseMap<GlobalAlias *, GlobalAlias *> FnsMap;
834 SetVector<GlobalAlias *> PatchableFns;
835
836 for (Function &F : Mod) {
837 if (F.hasPersonalityFn()) {
838 GlobalValue *PersFn =
839 cast<GlobalValue>(Val: F.getPersonalityFn()->stripPointerCasts());
840 if (PersFn->getValueType() && PersFn->getValueType()->isFunctionTy()) {
841 if (std::optional<std::string> MangledName =
842 getArm64ECMangledFunctionName(GV&: *PersFn)) {
843 PersFn->setName(MangledName.value());
844 }
845 }
846 }
847
848 if (!F.hasFnAttribute(Kind: Attribute::HybridPatchable) ||
849 F.isDeclarationForLinker() || F.hasLocalLinkage() ||
850 F.getName().ends_with(Suffix: HybridPatchableTargetSuffix))
851 continue;
852
853 // Rename hybrid patchable functions and change callers to use a global
854 // alias instead.
855 if (std::optional<std::string> MangledName =
856 getArm64ECMangledFunctionName(GV&: F)) {
857 std::string OrigName(F.getName());
858 F.setName(MangledName.value() + HybridPatchableTargetSuffix);
859
860 // The unmangled symbol is a weak alias to an undefined symbol with the
861 // "EXP+" prefix. This undefined symbol is resolved by the linker by
862 // creating an x86 thunk that jumps back to the actual EC target. Since we
863 // can't represent that in IR, we create an alias to the target instead.
864 // The "EXP+" symbol is set as metadata, which is then used by
865 // emitGlobalAlias to emit the right alias.
866 auto *A =
867 GlobalAlias::create(Linkage: GlobalValue::LinkOnceODRLinkage, Name: OrigName, Aliasee: &F);
868 auto *AM = GlobalAlias::create(Linkage: GlobalValue::LinkOnceODRLinkage,
869 Name: MangledName.value(), Aliasee: &F);
870 F.replaceUsesWithIf(New: AM,
871 ShouldReplace: [](Use &U) { return isa<GlobalAlias>(Val: U.getUser()); });
872 F.replaceAllUsesWith(V: A);
873 F.setMetadata(Kind: "arm64ec_exp_name",
874 Node: MDNode::get(Context&: M->getContext(),
875 MDs: MDString::get(Context&: M->getContext(),
876 Str: "EXP+" + MangledName.value())));
877 A->setAliasee(&F);
878 AM->setAliasee(&F);
879
880 if (F.hasDLLExportStorageClass()) {
881 A->setDLLStorageClass(GlobalValue::DLLExportStorageClass);
882 F.setDLLStorageClass(GlobalValue::DefaultStorageClass);
883 }
884
885 FnsMap[A] = AM;
886 PatchableFns.insert(X: A);
887 }
888 }
889
890 SetVector<GlobalValue *> DirectCalledFns;
891 for (Function &F : Mod)
892 if (!F.isDeclarationForLinker() &&
893 F.getCallingConv() != CallingConv::ARM64EC_Thunk_Native &&
894 F.getCallingConv() != CallingConv::ARM64EC_Thunk_X64)
895 processFunction(F, DirectCalledFns, FnsMap);
896
897 struct ThunkInfo {
898 Constant *Src;
899 Constant *Dst;
900 Arm64ECThunkType Kind;
901 };
902 SmallVector<ThunkInfo> ThunkMapping;
903 for (Function &F : Mod) {
904 if (!F.isDeclarationForLinker() &&
905 (!F.hasLocalLinkage() || F.hasAddressTaken()) &&
906 F.getCallingConv() != CallingConv::ARM64EC_Thunk_Native &&
907 F.getCallingConv() != CallingConv::ARM64EC_Thunk_X64) {
908 if (!F.hasComdat())
909 F.setComdat(Mod.getOrInsertComdat(Name: F.getName()));
910 ThunkMapping.push_back(
911 Elt: {.Src: &F, .Dst: buildEntryThunk(F: &F), .Kind: Arm64ECThunkType::Entry});
912 }
913 }
914 for (GlobalValue *O : DirectCalledFns) {
915 auto GA = dyn_cast<GlobalAlias>(Val: O);
916 auto F = dyn_cast<Function>(Val: GA ? GA->getAliasee() : O);
917 ThunkMapping.push_back(
918 Elt: {.Src: O, .Dst: buildExitThunk(FT: F->getFunctionType(), Attrs: F->getAttributes()),
919 .Kind: Arm64ECThunkType::Exit});
920 if (!GA && !F->hasDLLImportStorageClass())
921 ThunkMapping.push_back(
922 Elt: {.Src: buildGuestExitThunk(F), .Dst: F, .Kind: Arm64ECThunkType::GuestExit});
923 }
924 for (GlobalAlias *A : PatchableFns) {
925 Function *Thunk = buildPatchableThunk(UnmangledAlias: A, MangledAlias: FnsMap[A]);
926 ThunkMapping.push_back(Elt: {.Src: Thunk, .Dst: A, .Kind: Arm64ECThunkType::GuestExit});
927 }
928
929 if (!ThunkMapping.empty()) {
930 SmallVector<Constant *> ThunkMappingArrayElems;
931 for (ThunkInfo &Thunk : ThunkMapping) {
932 ThunkMappingArrayElems.push_back(Elt: ConstantStruct::getAnon(
933 V: {Thunk.Src, Thunk.Dst,
934 ConstantInt::get(Context&: M->getContext(), V: APInt(32, uint8_t(Thunk.Kind)))}));
935 }
936 Constant *ThunkMappingArray = ConstantArray::get(
937 T: llvm::ArrayType::get(ElementType: ThunkMappingArrayElems[0]->getType(),
938 NumElements: ThunkMappingArrayElems.size()),
939 V: ThunkMappingArrayElems);
940 new GlobalVariable(Mod, ThunkMappingArray->getType(), /*isConstant*/ false,
941 GlobalValue::ExternalLinkage, ThunkMappingArray,
942 "llvm.arm64ec.symbolmap");
943 }
944
945 return true;
946}
947
948bool AArch64Arm64ECCallLowering::processFunction(
949 Function &F, SetVector<GlobalValue *> &DirectCalledFns,
950 DenseMap<GlobalAlias *, GlobalAlias *> &FnsMap) {
951 SmallVector<CallBase *, 8> IndirectCalls;
952
953 // For ARM64EC targets, a function definition's name is mangled differently
954 // from the normal symbol. We currently have no representation of this sort
955 // of symbol in IR, so we change the name to the mangled name, then store
956 // the unmangled name as metadata. Later passes that need the unmangled
957 // name (emitting the definition) can grab it from the metadata.
958 //
959 // FIXME: Handle functions with weak linkage?
960 if (!F.hasLocalLinkage() || F.hasAddressTaken()) {
961 if (std::optional<std::string> MangledName =
962 getArm64ECMangledFunctionName(GV&: F)) {
963 F.addMetadata(Kind: "arm64ec_unmangled_name",
964 MD&: *MDNode::get(Context&: M->getContext(),
965 MDs: MDString::get(Context&: M->getContext(), Str: F.getName())));
966 if (F.hasComdat() && F.getComdat()->getName() == F.getName()) {
967 Comdat *MangledComdat = M->getOrInsertComdat(Name: MangledName.value());
968 SmallVector<GlobalObject *> ComdatUsers =
969 to_vector(Range: F.getComdat()->getUsers());
970 for (GlobalObject *User : ComdatUsers)
971 User->setComdat(MangledComdat);
972 }
973 F.setName(MangledName.value());
974 }
975 }
976
977 // Iterate over the instructions to find all indirect call/invoke/callbr
978 // instructions. Make a separate list of pointers to indirect
979 // call/invoke/callbr instructions because the original instructions will be
980 // deleted as the checks are added.
981 for (BasicBlock &BB : F) {
982 for (Instruction &I : BB) {
983 auto *CB = dyn_cast<CallBase>(Val: &I);
984 if (!CB || CB->getCallingConv() == CallingConv::ARM64EC_Thunk_X64 ||
985 CB->isInlineAsm())
986 continue;
987
988 // We need to instrument any call that isn't directly calling an
989 // ARM64 function.
990 //
991 // FIXME: getCalledFunction() fails if there's a bitcast (e.g.
992 // unprototyped functions in C)
993 if (Function *F = CB->getCalledFunction()) {
994 if (!LowerDirectToIndirect || F->hasLocalLinkage() ||
995 F->isIntrinsic() || !F->isDeclarationForLinker())
996 continue;
997
998 DirectCalledFns.insert(X: F);
999 continue;
1000 }
1001
1002 // Use mangled global alias for direct calls to patchable functions.
1003 if (GlobalAlias *A = dyn_cast<GlobalAlias>(Val: CB->getCalledOperand())) {
1004 auto I = FnsMap.find(Val: A);
1005 if (I != FnsMap.end()) {
1006 CB->setCalledOperand(I->second);
1007 DirectCalledFns.insert(X: I->first);
1008 continue;
1009 }
1010 }
1011
1012 IndirectCalls.push_back(Elt: CB);
1013 ++Arm64ECCallsLowered;
1014 }
1015 }
1016
1017 if (IndirectCalls.empty())
1018 return false;
1019
1020 for (CallBase *CB : IndirectCalls)
1021 lowerCall(CB);
1022
1023 return true;
1024}
1025
1026char AArch64Arm64ECCallLowering::ID = 0;
1027INITIALIZE_PASS(AArch64Arm64ECCallLowering, "Arm64ECCallLowering",
1028 "AArch64Arm64ECCallLowering", false, false)
1029
1030ModulePass *llvm::createAArch64Arm64ECCallLoweringPass() {
1031 return new AArch64Arm64ECCallLowering;
1032}
1033