1//===-- AArch64Arm64ECCallLowering.cpp - Lower Arm64EC calls ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file contains the IR transform to lower external or indirect calls for
11/// the ARM64EC calling convention. Such calls must go through the runtime, so
12/// we can translate the calling convention for calls into the emulator.
13///
14/// This subsumes Control Flow Guard handling.
15///
16//===----------------------------------------------------------------------===//
17
18#include "AArch64.h"
19#include "AArch64Subtarget.h"
20#include "llvm/ADT/SetVector.h"
21#include "llvm/ADT/SmallString.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/ADT/Statistic.h"
24#include "llvm/IR/CallingConv.h"
25#include "llvm/IR/DiagnosticInfo.h"
26#include "llvm/IR/GlobalAlias.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/Instruction.h"
29#include "llvm/IR/Mangler.h"
30#include "llvm/IR/Module.h"
31#include "llvm/Object/COFF.h"
32#include "llvm/Pass.h"
33#include "llvm/TargetParser/Triple.h"
34
35using namespace llvm;
36using namespace llvm::COFF;
37
38using OperandBundleDef = OperandBundleDefT<Value *>;
39
40#define DEBUG_TYPE "arm64eccalllowering"
41
42STATISTIC(Arm64ECCallsLowered, "Number of Arm64EC calls lowered");
43
44namespace {
45
46enum ThunkArgTranslation : uint8_t {
47 Direct,
48 Bitcast,
49 PointerIndirection,
50};
51
52struct ThunkArgInfo {
53 Type *Arm64Ty;
54 Type *X64Ty;
55 ThunkArgTranslation Translation;
56};
57
58class AArch64Arm64ECCallLowering : public ModulePass {
59public:
60 static char ID;
61 AArch64Arm64ECCallLowering() : ModulePass(ID) {}
62
63 Function *buildExitThunk(FunctionType *FnTy, AttributeList Attrs);
64 Function *buildEntryThunk(Function *F);
65 void lowerCall(CallBase *CB);
66 Function *buildGuestExitThunk(Function *F);
67 Function *buildPatchableThunk(GlobalAlias *UnmangledAlias,
68 GlobalAlias *MangledAlias);
69 bool processFunction(Function &F, SetVector<GlobalValue *> &DirectCalledFns,
70 DenseMap<GlobalAlias *, GlobalAlias *> &FnsMap);
71 bool runOnModule(Module &M) override;
72
73private:
74 const AArch64Options *CLOpts = nullptr;
75 ControlFlowGuardMode CFGuardModuleFlag = ControlFlowGuardMode::Disabled;
76 FunctionType *GuardFnType = nullptr;
77 FunctionType *DispatchFnType = nullptr;
78 Constant *GuardFnCFGlobal = nullptr;
79 Constant *GuardFnGlobal = nullptr;
80 Constant *DispatchFnGlobal = nullptr;
81 Module *M = nullptr;
82
83 Type *PtrTy;
84 Type *I64Ty;
85 Type *VoidTy;
86
87 void getThunkType(FunctionType *FT, AttributeList AttrList,
88 Arm64ECThunkType TT, raw_ostream &Out,
89 FunctionType *&Arm64Ty, FunctionType *&X64Ty,
90 SmallVector<ThunkArgTranslation> &ArgTranslations);
91 void getThunkRetType(FunctionType *FT, AttributeList AttrList,
92 raw_ostream &Out, Type *&Arm64RetTy, Type *&X64RetTy,
93 SmallVectorImpl<Type *> &Arm64ArgTypes,
94 SmallVectorImpl<Type *> &X64ArgTypes,
95 SmallVector<ThunkArgTranslation> &ArgTranslations,
96 bool &HasSretPtr);
97 void getThunkArgTypes(FunctionType *FT, AttributeList AttrList,
98 Arm64ECThunkType TT, raw_ostream &Out,
99 SmallVectorImpl<Type *> &Arm64ArgTypes,
100 SmallVectorImpl<Type *> &X64ArgTypes,
101 SmallVectorImpl<ThunkArgTranslation> &ArgTranslations,
102 bool HasSretPtr);
103 ThunkArgInfo canonicalizeThunkType(Type *T, Align Alignment, bool Ret,
104 uint64_t ArgSizeBytes, raw_ostream &Out);
105};
106
107} // end anonymous namespace
108
109void AArch64Arm64ECCallLowering::getThunkType(
110 FunctionType *FT, AttributeList AttrList, Arm64ECThunkType TT,
111 raw_ostream &Out, FunctionType *&Arm64Ty, FunctionType *&X64Ty,
112 SmallVector<ThunkArgTranslation> &ArgTranslations) {
113 Out << (TT == Arm64ECThunkType::Entry ? "$ientry_thunk$cdecl$"
114 : "$iexit_thunk$cdecl$");
115
116 Type *Arm64RetTy;
117 Type *X64RetTy;
118
119 SmallVector<Type *> Arm64ArgTypes;
120 SmallVector<Type *> X64ArgTypes;
121
122 // The first argument to a thunk is the called function, stored in x9.
123 // For exit thunks, we pass the called function down to the emulator;
124 // for entry/guest exit thunks, we just call the Arm64 function directly.
125 if (TT == Arm64ECThunkType::Exit)
126 Arm64ArgTypes.push_back(Elt: PtrTy);
127 X64ArgTypes.push_back(Elt: PtrTy);
128
129 bool HasSretPtr = false;
130 getThunkRetType(FT, AttrList, Out, Arm64RetTy, X64RetTy, Arm64ArgTypes,
131 X64ArgTypes, ArgTranslations, HasSretPtr);
132
133 getThunkArgTypes(FT, AttrList, TT, Out, Arm64ArgTypes, X64ArgTypes,
134 ArgTranslations, HasSretPtr);
135
136 Arm64Ty = FunctionType::get(Result: Arm64RetTy, Params: Arm64ArgTypes, isVarArg: false);
137
138 X64Ty = FunctionType::get(Result: X64RetTy, Params: X64ArgTypes, isVarArg: false);
139}
140
141void AArch64Arm64ECCallLowering::getThunkArgTypes(
142 FunctionType *FT, AttributeList AttrList, Arm64ECThunkType TT,
143 raw_ostream &Out, SmallVectorImpl<Type *> &Arm64ArgTypes,
144 SmallVectorImpl<Type *> &X64ArgTypes,
145 SmallVectorImpl<ThunkArgTranslation> &ArgTranslations, bool HasSretPtr) {
146
147 Out << "$";
148 if (FT->isVarArg()) {
149 // We treat the variadic function's thunk as a normal function
150 // with the following type on the ARM side:
151 // rettype exitthunk(
152 // ptr x9, ptr x0, i64 x1, i64 x2, i64 x3, ptr x4, i64 x5)
153 //
154 // that can coverage all types of variadic function.
155 // x9 is similar to normal exit thunk, store the called function.
156 // x0-x3 is the arguments be stored in registers.
157 // x4 is the address of the arguments on the stack.
158 // x5 is the size of the arguments on the stack.
159 //
160 // On the x64 side, it's the same except that x5 isn't set.
161 //
162 // If both the ARM and X64 sides are sret, there are only three
163 // arguments in registers.
164 //
165 // If the X64 side is sret, but the ARM side isn't, we pass an extra value
166 // to/from the X64 side, and let SelectionDAG transform it into a memory
167 // location.
168 Out << "varargs";
169
170 // x0-x3
171 for (int i = HasSretPtr ? 1 : 0; i < 4; i++) {
172 Arm64ArgTypes.push_back(Elt: I64Ty);
173 X64ArgTypes.push_back(Elt: I64Ty);
174 ArgTranslations.push_back(Elt: ThunkArgTranslation::Direct);
175 }
176
177 // x4
178 Arm64ArgTypes.push_back(Elt: PtrTy);
179 X64ArgTypes.push_back(Elt: PtrTy);
180 ArgTranslations.push_back(Elt: ThunkArgTranslation::Direct);
181 // x5
182 Arm64ArgTypes.push_back(Elt: I64Ty);
183 if (TT != Arm64ECThunkType::Entry) {
184 // FIXME: x5 isn't actually used by the x64 side; revisit once we
185 // have proper isel for varargs
186 X64ArgTypes.push_back(Elt: I64Ty);
187 ArgTranslations.push_back(Elt: ThunkArgTranslation::Direct);
188 }
189 return;
190 }
191
192 unsigned I = 0;
193 if (HasSretPtr)
194 I++;
195
196 if (I == FT->getNumParams()) {
197 Out << "v";
198 return;
199 }
200
201 for (unsigned E = FT->getNumParams(); I != E; ++I) {
202#if 0
203 // FIXME: Need more information about argument size; see
204 // https://reviews.llvm.org/D132926
205 uint64_t ArgSizeBytes = AttrList.getParamArm64ECArgSizeBytes(I);
206 Align ParamAlign = AttrList.getParamAlignment(I).valueOrOne();
207#else
208 uint64_t ArgSizeBytes = 0;
209 Align ParamAlign = Align();
210#endif
211 auto [Arm64Ty, X64Ty, ArgTranslation] =
212 canonicalizeThunkType(T: FT->getParamType(i: I), Alignment: ParamAlign,
213 /*Ret*/ false, ArgSizeBytes, Out);
214 Arm64ArgTypes.push_back(Elt: Arm64Ty);
215 X64ArgTypes.push_back(Elt: X64Ty);
216 ArgTranslations.push_back(Elt: ArgTranslation);
217 }
218}
219
220void AArch64Arm64ECCallLowering::getThunkRetType(
221 FunctionType *FT, AttributeList AttrList, raw_ostream &Out,
222 Type *&Arm64RetTy, Type *&X64RetTy, SmallVectorImpl<Type *> &Arm64ArgTypes,
223 SmallVectorImpl<Type *> &X64ArgTypes,
224 SmallVector<ThunkArgTranslation> &ArgTranslations, bool &HasSretPtr) {
225 Type *T = FT->getReturnType();
226#if 0
227 // FIXME: Need more information about argument size; see
228 // https://reviews.llvm.org/D132926
229 uint64_t ArgSizeBytes = AttrList.getRetArm64ECArgSizeBytes();
230#else
231 int64_t ArgSizeBytes = 0;
232#endif
233 if (T->isVoidTy()) {
234 if (FT->getNumParams()) {
235 Attribute SRetAttr0 = AttrList.getParamAttr(ArgNo: 0, Kind: Attribute::StructRet);
236 Attribute InRegAttr0 = AttrList.getParamAttr(ArgNo: 0, Kind: Attribute::InReg);
237 Attribute SRetAttr1, InRegAttr1;
238 if (FT->getNumParams() > 1) {
239 // Also check the second parameter (for class methods, the first
240 // parameter is "this", and the second parameter is the sret pointer.)
241 // It doesn't matter which one is sret.
242 SRetAttr1 = AttrList.getParamAttr(ArgNo: 1, Kind: Attribute::StructRet);
243 InRegAttr1 = AttrList.getParamAttr(ArgNo: 1, Kind: Attribute::InReg);
244 }
245 if ((SRetAttr0.isValid() && InRegAttr0.isValid()) ||
246 (SRetAttr1.isValid() && InRegAttr1.isValid())) {
247 // sret+inreg indicates a call that returns a C++ class value. This is
248 // actually equivalent to just passing and returning a void* pointer
249 // as the first or second argument. Translate it that way, instead of
250 // trying to model "inreg" in the thunk's calling convention; this
251 // simplfies the rest of the code, and matches MSVC mangling.
252 Out << "i8";
253 Arm64RetTy = I64Ty;
254 X64RetTy = I64Ty;
255 return;
256 }
257 if (SRetAttr0.isValid()) {
258 // FIXME: Sanity-check the sret type; if it's an integer or pointer,
259 // we'll get screwy mangling/codegen.
260 // FIXME: For large struct types, mangle as an integer argument and
261 // integer return, so we can reuse more thunks, instead of "m" syntax.
262 // (MSVC mangles this case as an integer return with no argument, but
263 // that's a miscompile.)
264 Type *SRetType = SRetAttr0.getValueAsType();
265 Align SRetAlign = AttrList.getParamAlignment(ArgNo: 0).valueOrOne();
266 canonicalizeThunkType(T: SRetType, Alignment: SRetAlign, /*Ret*/ true, ArgSizeBytes,
267 Out);
268 Arm64RetTy = VoidTy;
269 X64RetTy = VoidTy;
270 Arm64ArgTypes.push_back(Elt: FT->getParamType(i: 0));
271 X64ArgTypes.push_back(Elt: FT->getParamType(i: 0));
272 ArgTranslations.push_back(Elt: ThunkArgTranslation::Direct);
273 HasSretPtr = true;
274 return;
275 }
276 }
277
278 Out << "v";
279 Arm64RetTy = VoidTy;
280 X64RetTy = VoidTy;
281 return;
282 }
283
284 auto info =
285 canonicalizeThunkType(T, Alignment: Align(), /*Ret*/ true, ArgSizeBytes, Out);
286 Arm64RetTy = info.Arm64Ty;
287 X64RetTy = info.X64Ty;
288 if (X64RetTy->isPointerTy()) {
289 // If the X64 type is canonicalized to a pointer, that means it's
290 // passed/returned indirectly. For a return value, that means it's an
291 // sret pointer.
292 X64ArgTypes.push_back(Elt: X64RetTy);
293 X64RetTy = VoidTy;
294 }
295}
296
297ThunkArgInfo AArch64Arm64ECCallLowering::canonicalizeThunkType(
298 Type *T, Align Alignment, bool Ret, uint64_t ArgSizeBytes,
299 raw_ostream &Out) {
300
301 auto direct = [](Type *T) {
302 return ThunkArgInfo{.Arm64Ty: T, .X64Ty: T, .Translation: ThunkArgTranslation::Direct};
303 };
304
305 auto bitcast = [this](Type *Arm64Ty, uint64_t SizeInBytes) {
306 return ThunkArgInfo{.Arm64Ty: Arm64Ty,
307 .X64Ty: llvm::Type::getIntNTy(C&: M->getContext(), N: SizeInBytes * 8),
308 .Translation: ThunkArgTranslation::Bitcast};
309 };
310
311 auto pointerIndirection = [this](Type *Arm64Ty) {
312 return ThunkArgInfo{.Arm64Ty: Arm64Ty, .X64Ty: PtrTy,
313 .Translation: ThunkArgTranslation::PointerIndirection};
314 };
315
316 if (T->isHalfTy()) {
317 // Prefix with `llvm` since MSVC doesn't specify `_Float16`
318 Out << "__llvm_h__";
319 return direct(T);
320 }
321
322 if (T->isBFloatTy()) {
323 // Prefix with `llvm` since MSVC doesn't specify `__bf16`
324 Out << "__llvm_bf16__";
325 return direct(T);
326 }
327
328 if (T->isFloatTy()) {
329 Out << "f";
330 return direct(T);
331 }
332
333 if (T->isDoubleTy()) {
334 Out << "d";
335 return direct(T);
336 }
337
338 if (T->isFP128Ty()) {
339 // Prefix with `llvm` since MSVC doesn't specify `_Float128`
340 Out << "__llvm_q__";
341 // On windows f128 is passed indirectly, and Clang/LLVM
342 // returns using sret for compatibility with GCC.
343 return pointerIndirection(T);
344 }
345
346 if (T->isFloatingPointTy()) {
347 report_fatal_error(
348 reason: "Only half, bfloat16, float, double, and fp128 are supported "
349 "for ARM64EC thunks");
350 }
351
352 auto &DL = M->getDataLayout();
353
354 if (auto *StructTy = dyn_cast<StructType>(Val: T))
355 if (StructTy->getNumElements() == 1)
356 T = StructTy->getElementType(N: 0);
357
358 if (T->isArrayTy()) {
359 Type *ElementTy = T->getArrayElementType();
360 uint64_t ElementCnt = T->getArrayNumElements();
361 uint64_t ElementSizePerBytes = DL.getTypeSizeInBits(Ty: ElementTy) / 8;
362 uint64_t TotalSizeBytes = ElementCnt * ElementSizePerBytes;
363 if (ElementTy->isHalfTy() || ElementTy->isBFloatTy() ||
364 ElementTy->isFloatTy() || ElementTy->isDoubleTy() ||
365 ElementTy->isFP128Ty()) {
366 if (ElementTy->isHalfTy())
367 // Prefix with `llvm` since MSVC doesn't specify `_Float16`
368 Out << "__llvm_H__";
369 else if (ElementTy->isBFloatTy())
370 // Prefix with `llvm` since MSVC doesn't specify `__bf16`
371 Out << "__llvm_BF16__";
372 else if (ElementTy->isFloatTy())
373 Out << "F";
374 else if (ElementTy->isDoubleTy())
375 Out << "D";
376 else if (ElementTy->isFP128Ty())
377 // Prefix with `llvm` since MSVC doesn't specify `_Float128`
378 Out << "__llvm_Q__";
379 Out << TotalSizeBytes;
380 if (Alignment.value() >= 16 && !Ret)
381 Out << "a" << Alignment.value();
382 if (TotalSizeBytes <= 8) {
383 // Arm64 returns small structs of float/double in float registers;
384 // X64 uses RAX.
385 return bitcast(T, TotalSizeBytes);
386 } else {
387 // Struct is passed directly on Arm64, but indirectly on X64.
388 return pointerIndirection(T);
389 }
390 } else if (ElementTy->isFloatingPointTy()) {
391 report_fatal_error(
392 reason: "Only half, bfloat16, float, double, and fp128 are supported "
393 "for ARM64EC thunks");
394 }
395 }
396
397 if ((T->isIntegerTy() || T->isPointerTy()) && DL.getTypeSizeInBits(Ty: T) <= 64) {
398 Out << "i8";
399 return direct(I64Ty);
400 }
401
402 unsigned TypeSize = ArgSizeBytes;
403 if (TypeSize == 0)
404 TypeSize = DL.getTypeSizeInBits(Ty: T) / 8;
405 Out << "m";
406 if (TypeSize != 4)
407 Out << TypeSize;
408 if (Alignment.value() >= 16 && !Ret)
409 Out << "a" << Alignment.value();
410 // FIXME: Try to canonicalize Arm64Ty more thoroughly?
411 if (TypeSize == 1 || TypeSize == 2 || TypeSize == 4 || TypeSize == 8) {
412 // Pass directly in an integer register
413 return bitcast(T, TypeSize);
414 } else {
415 // Passed directly on Arm64, but indirectly on X64.
416 return pointerIndirection(T);
417 }
418}
419
420// This function builds the "exit thunk", a function which translates
421// arguments and return values when calling x64 code from AArch64 code.
422Function *AArch64Arm64ECCallLowering::buildExitThunk(FunctionType *FT,
423 AttributeList Attrs) {
424 SmallString<256> ExitThunkName;
425 llvm::raw_svector_ostream ExitThunkStream(ExitThunkName);
426 FunctionType *Arm64Ty, *X64Ty;
427 SmallVector<ThunkArgTranslation> ArgTranslations;
428 getThunkType(FT, AttrList: Attrs, TT: Arm64ECThunkType::Exit, Out&: ExitThunkStream, Arm64Ty,
429 X64Ty, ArgTranslations);
430 if (Function *F = M->getFunction(Name: ExitThunkName))
431 return F;
432
433 Function *F = Function::Create(Ty: Arm64Ty, Linkage: GlobalValue::LinkOnceODRLinkage, AddrSpace: 0,
434 N: ExitThunkName, M);
435 F->setCallingConv(CallingConv::ARM64EC_Thunk_Native);
436 F->setSection(".wowthk$aa");
437 F->setComdat(M->getOrInsertComdat(Name: ExitThunkName));
438 // Copy MSVC, and always set up a frame pointer. (Maybe this isn't necessary.)
439 F->addFnAttr(Kind: "frame-pointer", Val: "all");
440 // Only copy sret from the first argument. For C++ instance methods, clang can
441 // stick an sret marking on a later argument, but it doesn't actually affect
442 // the ABI, so we can omit it. This avoids triggering a verifier assertion.
443 if (FT->getNumParams()) {
444 auto SRet = Attrs.getParamAttr(ArgNo: 0, Kind: Attribute::StructRet);
445 auto InReg = Attrs.getParamAttr(ArgNo: 0, Kind: Attribute::InReg);
446 if (SRet.isValid() && !InReg.isValid())
447 F->addParamAttr(ArgNo: 1, Attr: SRet);
448 }
449 // FIXME: Copy anything other than sret? Shouldn't be necessary for normal
450 // C ABI, but might show up in other cases.
451 BasicBlock *BB = BasicBlock::Create(Context&: M->getContext(), Name: "", Parent: F);
452 IRBuilder<> IRB(BB);
453 Value *CalleePtr =
454 M->getOrInsertGlobal(Name: "__os_arm64x_dispatch_call_no_redirect", Ty: PtrTy);
455 Value *Callee = IRB.CreateLoad(Ty: PtrTy, Ptr: CalleePtr);
456 auto &DL = M->getDataLayout();
457 SmallVector<Value *> Args;
458 FunctionType *DispatcherCallTy = X64Ty;
459 // If we have a vararg function, the SelectionDAG lowering will need to
460 // recognize this so it can copy the arguments described by x4 (pointer) and
461 // x5 (length) to set up the x86-64 context correctly.
462 if (FT->isVarArg())
463 DispatcherCallTy =
464 FunctionType::get(Result: X64Ty->getReturnType(), Params: X64Ty->params(),
465 /*isVarArg=*/true);
466
467 // Pass the called function in x9.
468 auto X64TyOffset = 1;
469 Args.push_back(Elt: F->arg_begin());
470
471 Type *RetTy = Arm64Ty->getReturnType();
472 if (RetTy != X64Ty->getReturnType()) {
473 // If the return type is an array or struct, translate it. Values of size
474 // 8 or less go into RAX; bigger values go into memory, and we pass a
475 // pointer.
476 if (DL.getTypeStoreSize(Ty: RetTy) > 8) {
477 Args.push_back(Elt: IRB.CreateAlloca(Ty: RetTy));
478 X64TyOffset++;
479 }
480 }
481
482 for (auto [Arg, X64ArgType, ArgTranslation] : llvm::zip_equal(
483 t: make_range(x: F->arg_begin() + 1, y: F->arg_end()),
484 u: make_range(x: X64Ty->param_begin() + X64TyOffset, y: X64Ty->param_end()),
485 args&: ArgTranslations)) {
486 // Translate arguments from AArch64 calling convention to x86 calling
487 // convention.
488 //
489 // For simple types, we don't need to do any translation: they're
490 // represented the same way. (Implicit sign extension is not part of
491 // either convention.)
492 //
493 // The big thing we have to worry about is struct types... but
494 // fortunately AArch64 clang is pretty friendly here: the cases that need
495 // translation are always passed as a struct or array. (If we run into
496 // some cases where this doesn't work, we can teach clang to mark it up
497 // with an attribute.)
498 //
499 // The first argument is the called function, stored in x9.
500 if (ArgTranslation != ThunkArgTranslation::Direct) {
501 Value *Mem = IRB.CreateAlloca(Ty: Arg.getType());
502 IRB.CreateStore(Val: &Arg, Ptr: Mem);
503 if (ArgTranslation == ThunkArgTranslation::Bitcast) {
504 Type *IntTy = IRB.getIntNTy(N: DL.getTypeStoreSizeInBits(Ty: Arg.getType()));
505 Args.push_back(Elt: IRB.CreateLoad(Ty: IntTy, Ptr: Mem));
506 } else {
507 assert(ArgTranslation == ThunkArgTranslation::PointerIndirection);
508 Args.push_back(Elt: Mem);
509 }
510 } else {
511 Args.push_back(Elt: &Arg);
512 }
513 assert(Args.back()->getType() == X64ArgType);
514 }
515 // FIXME: Transfer necessary attributes? sret? anything else?
516
517 CallInst *Call = IRB.CreateCall(FTy: DispatcherCallTy, Callee, Args);
518 Call->setCallingConv(CallingConv::ARM64EC_Thunk_X64);
519
520 Value *RetVal = Call;
521 if (RetTy != X64Ty->getReturnType()) {
522 // If we rewrote the return type earlier, convert the return value to
523 // the proper type.
524 if (DL.getTypeStoreSize(Ty: RetTy) > 8) {
525 RetVal = IRB.CreateLoad(Ty: RetTy, Ptr: Args[1]);
526 } else {
527 Value *CastAlloca = IRB.CreateAlloca(Ty: RetTy);
528 IRB.CreateStore(Val: Call, Ptr: CastAlloca);
529 RetVal = IRB.CreateLoad(Ty: RetTy, Ptr: CastAlloca);
530 }
531 }
532
533 if (RetTy->isVoidTy())
534 IRB.CreateRetVoid();
535 else
536 IRB.CreateRet(V: RetVal);
537 return F;
538}
539
540// This function builds the "entry thunk", a function which translates
541// arguments and return values when calling AArch64 code from x64 code.
542Function *AArch64Arm64ECCallLowering::buildEntryThunk(Function *F) {
543 SmallString<256> EntryThunkName;
544 llvm::raw_svector_ostream EntryThunkStream(EntryThunkName);
545 FunctionType *Arm64Ty, *X64Ty;
546 SmallVector<ThunkArgTranslation> ArgTranslations;
547 getThunkType(FT: F->getFunctionType(), AttrList: F->getAttributes(),
548 TT: Arm64ECThunkType::Entry, Out&: EntryThunkStream, Arm64Ty, X64Ty,
549 ArgTranslations);
550 if (Function *F = M->getFunction(Name: EntryThunkName))
551 return F;
552
553 Function *Thunk = Function::Create(Ty: X64Ty, Linkage: GlobalValue::LinkOnceODRLinkage, AddrSpace: 0,
554 N: EntryThunkName, M);
555 Thunk->setCallingConv(CallingConv::ARM64EC_Thunk_X64);
556 Thunk->setSection(".wowthk$aa");
557 Thunk->setComdat(M->getOrInsertComdat(Name: EntryThunkName));
558 // Copy MSVC, and always set up a frame pointer. (Maybe this isn't necessary.)
559 Thunk->addFnAttr(Kind: "frame-pointer", Val: "all");
560
561 BasicBlock *BB = BasicBlock::Create(Context&: M->getContext(), Name: "", Parent: Thunk);
562 IRBuilder<> IRB(BB);
563
564 Type *RetTy = Arm64Ty->getReturnType();
565 Type *X64RetType = X64Ty->getReturnType();
566
567 bool TransformDirectToSRet = X64RetType->isVoidTy() && !RetTy->isVoidTy();
568 unsigned ThunkArgOffset = TransformDirectToSRet ? 2 : 1;
569 unsigned PassthroughArgSize =
570 (F->isVarArg() ? 5 : Thunk->arg_size()) - ThunkArgOffset;
571 assert(ArgTranslations.size() == (F->isVarArg() ? 5 : PassthroughArgSize));
572
573 // Translate arguments to call.
574 SmallVector<Value *> Args;
575 for (unsigned i = 0; i != PassthroughArgSize; ++i) {
576 Value *Arg = Thunk->getArg(i: i + ThunkArgOffset);
577 Type *ArgTy = Arm64Ty->getParamType(i);
578 ThunkArgTranslation ArgTranslation = ArgTranslations[i];
579 if (ArgTranslation != ThunkArgTranslation::Direct) {
580 // Translate array/struct arguments to the expected type.
581 if (ArgTranslation == ThunkArgTranslation::Bitcast) {
582 Value *CastAlloca = IRB.CreateAlloca(Ty: ArgTy);
583 IRB.CreateStore(Val: Arg, Ptr: CastAlloca);
584 Arg = IRB.CreateLoad(Ty: ArgTy, Ptr: CastAlloca);
585 } else {
586 assert(ArgTranslation == ThunkArgTranslation::PointerIndirection);
587 Arg = IRB.CreateLoad(Ty: ArgTy, Ptr: Arg);
588 }
589 }
590 assert(Arg->getType() == ArgTy);
591 Args.push_back(Elt: Arg);
592 }
593
594 if (F->isVarArg()) {
595 // The 5th argument to variadic entry thunks is used to model the x64 sp
596 // which is passed to the thunk in x4, this can be passed to the callee as
597 // the variadic argument start address after skipping over the 32 byte
598 // shadow store.
599
600 // The EC thunk CC will assign any argument marked as InReg to x4.
601 Thunk->addParamAttr(ArgNo: 5, Kind: Attribute::InReg);
602 Value *Arg = Thunk->getArg(i: 5);
603 Arg = IRB.CreatePtrAdd(Ptr: Arg, Offset: IRB.getInt64(C: 0x20));
604 Args.push_back(Elt: Arg);
605
606 // Pass in a zero variadic argument size (in x5).
607 Args.push_back(Elt: IRB.getInt64(C: 0));
608 }
609
610 // Call the function passed to the thunk.
611 Value *Callee = Thunk->getArg(i: 0);
612 CallInst *Call = IRB.CreateCall(FTy: Arm64Ty, Callee, Args);
613
614 auto SRetAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::StructRet);
615 auto InRegAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::InReg);
616 if (SRetAttr.isValid() && !InRegAttr.isValid()) {
617 Thunk->addParamAttr(ArgNo: 1, Attr: SRetAttr);
618 Call->addParamAttr(ArgNo: 0, Attr: SRetAttr);
619 }
620
621 Value *RetVal = Call;
622 if (TransformDirectToSRet) {
623 // The x64 side returns this value indirectly via a hidden pointer (sret).
624 // Mark the thunk's pointer arg with sret so that ISel saves it and copies
625 // it into x8 (RAX) on return, matching the x64 calling convention.
626 Thunk->addParamAttr(
627 ArgNo: 1, Attr: Attribute::getWithStructRetType(Context&: M->getContext(), Ty: RetTy));
628 IRB.CreateStore(Val: RetVal, Ptr: Thunk->getArg(i: 1));
629 } else if (X64RetType != RetTy) {
630 Value *CastAlloca = IRB.CreateAlloca(Ty: X64RetType);
631 IRB.CreateStore(Val: Call, Ptr: CastAlloca);
632 RetVal = IRB.CreateLoad(Ty: X64RetType, Ptr: CastAlloca);
633 }
634
635 // Return to the caller. Note that the isel has code to translate this
636 // "ret" to a tail call to __os_arm64x_dispatch_ret. (Alternatively, we
637 // could emit a tail call here, but that would require a dedicated calling
638 // convention, which seems more complicated overall.)
639 if (X64RetType->isVoidTy())
640 IRB.CreateRetVoid();
641 else
642 IRB.CreateRet(V: RetVal);
643
644 return Thunk;
645}
646
647std::optional<std::string> getArm64ECMangledFunctionName(GlobalValue &GV) {
648 if (!GV.hasName()) {
649 GV.setName("__unnamed");
650 }
651
652 return llvm::getArm64ECMangledFunctionName(Name: GV.getName());
653}
654
655// Builds the "guest exit thunk", a helper to call a function which may or may
656// not be an exit thunk. (We optimistically assume non-dllimport function
657// declarations refer to functions defined in AArch64 code; if the linker
658// can't prove that, we use this routine instead.)
659Function *AArch64Arm64ECCallLowering::buildGuestExitThunk(Function *F) {
660 llvm::raw_null_ostream NullThunkName;
661 FunctionType *Arm64Ty, *X64Ty;
662 SmallVector<ThunkArgTranslation> ArgTranslations;
663 getThunkType(FT: F->getFunctionType(), AttrList: F->getAttributes(),
664 TT: Arm64ECThunkType::GuestExit, Out&: NullThunkName, Arm64Ty, X64Ty,
665 ArgTranslations);
666 auto MangledName = getArm64ECMangledFunctionName(GV&: *F);
667 assert(MangledName && "Can't guest exit to function that's already native");
668 std::string ThunkName = *MangledName;
669 if (ThunkName[0] == '?' && ThunkName.find(s: "@") != std::string::npos) {
670 ThunkName.insert(pos: ThunkName.find(s: "@"), s: "$exit_thunk");
671 } else {
672 ThunkName.append(s: "$exit_thunk");
673 }
674 Function *GuestExit =
675 Function::Create(Ty: Arm64Ty, Linkage: GlobalValue::WeakODRLinkage, AddrSpace: 0, N: ThunkName, M);
676 GuestExit->setComdat(M->getOrInsertComdat(Name: ThunkName));
677 GuestExit->setSection(".wowthk$aa");
678 GuestExit->addMetadata(
679 Kind: "arm64ec_unmangled_name",
680 MD&: *MDNode::get(Context&: M->getContext(),
681 MDs: MDString::get(Context&: M->getContext(), Str: F->getName())));
682 GuestExit->setMetadata(
683 Kind: "arm64ec_ecmangled_name",
684 Node: MDNode::get(Context&: M->getContext(),
685 MDs: MDString::get(Context&: M->getContext(), Str: *MangledName)));
686 F->setMetadata(Kind: "arm64ec_hasguestexit", Node: MDNode::get(Context&: M->getContext(), MDs: {}));
687 BasicBlock *BB = BasicBlock::Create(Context&: M->getContext(), Name: "", Parent: GuestExit);
688 IRBuilder<> B(BB);
689
690 // Create new call instruction. The call check should always be a call,
691 // even if the original CallBase is an Invoke or CallBr instructio.
692 // This is treated as a direct call, so do not use GuardFnCFGlobal.
693 LoadInst *GuardCheckLoad = B.CreateLoad(Ty: PtrTy, Ptr: GuardFnGlobal);
694 Function *Thunk = buildExitThunk(FT: F->getFunctionType(), Attrs: F->getAttributes());
695 CallInst *GuardCheck = B.CreateCall(
696 FTy: GuardFnType, Callee: GuardCheckLoad, Args: {F, Thunk});
697 Value *GuardCheckDest = B.CreateExtractValue(Agg: GuardCheck, Idxs: 0);
698 Value *GuardFinalDest = B.CreateExtractValue(Agg: GuardCheck, Idxs: 1);
699
700 // Ensure that the first argument is passed in the correct register.
701 GuardCheck->setCallingConv(CallingConv::CFGuard_Check);
702
703 SmallVector<Value *> Args(llvm::make_pointer_range(Range: GuestExit->args()));
704 OperandBundleDef OB("cfguardtarget", GuardFinalDest);
705 CallInst *Call = B.CreateCall(FTy: Arm64Ty, Callee: GuardCheckDest, Args, OpBundles: OB);
706 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
707
708 if (Call->getType()->isVoidTy())
709 B.CreateRetVoid();
710 else
711 B.CreateRet(V: Call);
712
713 auto SRetAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::StructRet);
714 auto InRegAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::InReg);
715 if (SRetAttr.isValid() && !InRegAttr.isValid()) {
716 GuestExit->addParamAttr(ArgNo: 0, Attr: SRetAttr);
717 Call->addParamAttr(ArgNo: 0, Attr: SRetAttr);
718 }
719
720 return GuestExit;
721}
722
723Function *
724AArch64Arm64ECCallLowering::buildPatchableThunk(GlobalAlias *UnmangledAlias,
725 GlobalAlias *MangledAlias) {
726 llvm::raw_null_ostream NullThunkName;
727 FunctionType *Arm64Ty, *X64Ty;
728 Function *F = cast<Function>(Val: MangledAlias->getAliasee());
729 SmallVector<ThunkArgTranslation> ArgTranslations;
730 getThunkType(FT: F->getFunctionType(), AttrList: F->getAttributes(),
731 TT: Arm64ECThunkType::GuestExit, Out&: NullThunkName, Arm64Ty, X64Ty,
732 ArgTranslations);
733 std::string ThunkName(MangledAlias->getName());
734 if (ThunkName[0] == '?' && ThunkName.find(s: "@") != std::string::npos) {
735 ThunkName.insert(pos: ThunkName.find(s: "@"), s: "$hybpatch_thunk");
736 } else {
737 ThunkName.append(s: "$hybpatch_thunk");
738 }
739
740 Function *GuestExit =
741 Function::Create(Ty: Arm64Ty, Linkage: GlobalValue::WeakODRLinkage, AddrSpace: 0, N: ThunkName, M);
742 GuestExit->setComdat(M->getOrInsertComdat(Name: ThunkName));
743 GuestExit->setSection(".wowthk$aa");
744 BasicBlock *BB = BasicBlock::Create(Context&: M->getContext(), Name: "", Parent: GuestExit);
745 IRBuilder<> B(BB);
746
747 // Load the global symbol as a pointer to the check function.
748 LoadInst *DispatchLoad = B.CreateLoad(Ty: PtrTy, Ptr: DispatchFnGlobal);
749
750 // Create new dispatch call instruction.
751 Function *ExitThunk =
752 buildExitThunk(FT: F->getFunctionType(), Attrs: F->getAttributes());
753 CallInst *Dispatch =
754 B.CreateCall(FTy: DispatchFnType, Callee: DispatchLoad,
755 Args: {UnmangledAlias, ExitThunk, UnmangledAlias->getAliasee()});
756
757 // Ensure that the first arguments are passed in the correct registers.
758 Dispatch->setCallingConv(CallingConv::CFGuard_Check);
759
760 SmallVector<Value *> Args(llvm::make_pointer_range(Range: GuestExit->args()));
761 CallInst *Call = B.CreateCall(FTy: Arm64Ty, Callee: Dispatch, Args);
762 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
763
764 if (Call->getType()->isVoidTy())
765 B.CreateRetVoid();
766 else
767 B.CreateRet(V: Call);
768
769 auto SRetAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::StructRet);
770 auto InRegAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::InReg);
771 if (SRetAttr.isValid() && !InRegAttr.isValid()) {
772 GuestExit->addParamAttr(ArgNo: 0, Attr: SRetAttr);
773 Call->addParamAttr(ArgNo: 0, Attr: SRetAttr);
774 }
775
776 MangledAlias->setAliasee(GuestExit);
777 return GuestExit;
778}
779
780// Lower an indirect call with inline code.
781void AArch64Arm64ECCallLowering::lowerCall(CallBase *CB) {
782 IRBuilder<> B(CB);
783 Value *CalledOperand = CB->getCalledOperand();
784
785 // If the indirect call is called within catchpad or cleanuppad,
786 // we need to copy "funclet" bundle of the call.
787 SmallVector<llvm::OperandBundleDef, 1> Bundles;
788 if (auto Bundle = CB->getOperandBundle(ID: LLVMContext::OB_funclet))
789 Bundles.push_back(Elt: OperandBundleDef(*Bundle));
790
791 // Load the global symbol as a pointer to the check function.
792 Value *GuardFn;
793 if ((CFGuardModuleFlag == ControlFlowGuardMode::Enabled) &&
794 !CB->hasFnAttr(Kind: "guard_nocf"))
795 GuardFn = GuardFnCFGlobal;
796 else
797 GuardFn = GuardFnGlobal;
798 LoadInst *GuardCheckLoad = B.CreateLoad(Ty: PtrTy, Ptr: GuardFn);
799
800 // Create new call instruction. The CFGuard check should always be a call,
801 // even if the original CallBase is an Invoke or CallBr instruction.
802 Function *Thunk = buildExitThunk(FT: CB->getFunctionType(), Attrs: CB->getAttributes());
803 CallInst *GuardCheck =
804 B.CreateCall(FTy: GuardFnType, Callee: GuardCheckLoad, Args: {CalledOperand, Thunk},
805 OpBundles: Bundles);
806 Value *GuardCheckDest = B.CreateExtractValue(Agg: GuardCheck, Idxs: 0);
807 Value *GuardFinalDest = B.CreateExtractValue(Agg: GuardCheck, Idxs: 1);
808
809 // Ensure that the first argument is passed in the correct register.
810 GuardCheck->setCallingConv(CallingConv::CFGuard_Check);
811
812 // Update the call: set the callee, and add a bundle with the final
813 // destination,
814 CB->setCalledOperand(GuardCheckDest);
815 OperandBundleDef OB("cfguardtarget", GuardFinalDest);
816 auto *NewCall = CallBase::addOperandBundle(CB, ID: LLVMContext::OB_cfguardtarget,
817 OB, InsertPt: CB->getIterator());
818 NewCall->copyMetadata(SrcInst: *CB);
819 CB->replaceAllUsesWith(V: NewCall);
820 CB->eraseFromParent();
821}
822
823bool AArch64Arm64ECCallLowering::runOnModule(Module &Mod) {
824 CLOpts = &Mod.getContext().getLibraryOptions().get<AArch64Options>();
825 if (!CLOpts->arm64ec_generate_thunks)
826 return false;
827
828 M = &Mod;
829
830 // Check if this module has the cfguard flag and read its value.
831 CFGuardModuleFlag = M->getControlFlowGuardMode();
832
833 // Warn if the module flag requests an unsupported CFGuard mechanism.
834 if (CFGuardModuleFlag == ControlFlowGuardMode::Enabled) {
835 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(
836 MD: Mod.getModuleFlag(Key: "cfguard-mechanism"))) {
837 auto MechanismOverride =
838 static_cast<ControlFlowGuardMechanism>(CI->getZExtValue());
839 if (MechanismOverride != ControlFlowGuardMechanism::Automatic &&
840 MechanismOverride != ControlFlowGuardMechanism::Check)
841 Mod.getContext().diagnose(
842 DI: DiagnosticInfoGeneric("only the Check Control Flow Guard mechanism "
843 "is supported for Arm64EC",
844 DS_Warning));
845 }
846 }
847
848 PtrTy = PointerType::getUnqual(C&: M->getContext());
849 I64Ty = Type::getInt64Ty(C&: M->getContext());
850 VoidTy = Type::getVoidTy(C&: M->getContext());
851
852 GuardFnType =
853 FunctionType::get(Result: StructType::get(elt1: PtrTy, elts: PtrTy), Params: {PtrTy, PtrTy}, isVarArg: false);
854 DispatchFnType = FunctionType::get(Result: PtrTy, Params: {PtrTy, PtrTy, PtrTy}, isVarArg: false);
855 GuardFnCFGlobal = M->getOrInsertGlobal(Name: "__os_arm64x_check_icall_cfg", Ty: PtrTy);
856 GuardFnGlobal = M->getOrInsertGlobal(Name: "__os_arm64x_check_icall", Ty: PtrTy);
857 DispatchFnGlobal = M->getOrInsertGlobal(Name: "__os_arm64x_dispatch_call", Ty: PtrTy);
858
859 // Mangle names of function aliases and add the alias name to
860 // arm64ec_unmangled_name metadata to ensure a weak anti-dependency symbol is
861 // emitted for the alias as well. Do this early, before handling
862 // hybrid_patchable functions, to avoid mangling their aliases.
863 for (GlobalAlias &A : Mod.aliases()) {
864 auto F = dyn_cast_or_null<Function>(Val: A.getAliaseeObject());
865 if (!F)
866 continue;
867 if (std::optional<std::string> MangledName =
868 getArm64ECMangledFunctionName(GV&: A)) {
869 F->addMetadata(Kind: "arm64ec_unmangled_name",
870 MD&: *MDNode::get(Context&: M->getContext(),
871 MDs: MDString::get(Context&: M->getContext(), Str: A.getName())));
872 A.setName(MangledName.value());
873 }
874 }
875
876 DenseMap<GlobalAlias *, GlobalAlias *> FnsMap;
877 SetVector<GlobalAlias *> PatchableFns;
878
879 for (Function &F : Mod) {
880 if (F.hasPersonalityFn()) {
881 GlobalValue *PersFn =
882 cast<GlobalValue>(Val: F.getPersonalityFn()->stripPointerCasts());
883 if (PersFn->getValueType() && PersFn->getValueType()->isFunctionTy()) {
884 if (std::optional<std::string> MangledName =
885 getArm64ECMangledFunctionName(GV&: *PersFn)) {
886 PersFn->setName(MangledName.value());
887 }
888 }
889 }
890
891 if (!F.hasFnAttribute(Kind: Attribute::HybridPatchable) ||
892 F.isDeclarationForLinker() || F.hasLocalLinkage() ||
893 F.getName().ends_with(Suffix: HybridPatchableTargetSuffix))
894 continue;
895
896 // Rename hybrid patchable functions and change callers to use a global
897 // alias instead.
898 if (std::optional<std::string> MangledName =
899 getArm64ECMangledFunctionName(GV&: F)) {
900 std::string OrigName(F.getName());
901 F.setName(MangledName.value() + HybridPatchableTargetSuffix);
902
903 // The unmangled symbol is a weak alias to an undefined symbol with the
904 // "EXP+" prefix. This undefined symbol is resolved by the linker by
905 // creating an x86 thunk that jumps back to the actual EC target. Since we
906 // can't represent that in IR, we create an alias to the target instead.
907 // The "EXP+" symbol is set as metadata, which is then used by
908 // emitGlobalAlias to emit the right alias.
909 auto *A =
910 GlobalAlias::create(Linkage: GlobalValue::LinkOnceODRLinkage, Name: OrigName, Aliasee: &F);
911 auto *AM = GlobalAlias::create(Linkage: GlobalValue::LinkOnceODRLinkage,
912 Name: MangledName.value(), Aliasee: &F);
913 F.replaceUsesWithIf(New: AM,
914 ShouldReplace: [](Use &U) { return isa<GlobalAlias>(Val: U.getUser()); });
915 F.replaceAllUsesWith(V: A);
916 F.setMetadata(Kind: "arm64ec_exp_name",
917 Node: MDNode::get(Context&: M->getContext(),
918 MDs: MDString::get(Context&: M->getContext(),
919 Str: "EXP+" + MangledName.value())));
920 A->setAliasee(&F);
921 AM->setAliasee(&F);
922
923 if (F.hasDLLExportStorageClass()) {
924 A->setDLLStorageClass(GlobalValue::DLLExportStorageClass);
925 F.setDLLStorageClass(GlobalValue::DefaultStorageClass);
926 }
927
928 FnsMap[A] = AM;
929 PatchableFns.insert(X: A);
930 }
931 }
932
933 SetVector<GlobalValue *> DirectCalledFns;
934 for (Function &F : Mod)
935 if (!F.isDeclarationForLinker() &&
936 F.getCallingConv() != CallingConv::ARM64EC_Thunk_Native &&
937 F.getCallingConv() != CallingConv::ARM64EC_Thunk_X64)
938 processFunction(F, DirectCalledFns, FnsMap);
939
940 struct ThunkInfo {
941 Constant *Src;
942 Constant *Dst;
943 Arm64ECThunkType Kind;
944 };
945 SmallVector<ThunkInfo> ThunkMapping;
946 for (Function &F : Mod) {
947 if (!F.isDeclarationForLinker() &&
948 (!F.hasLocalLinkage() || F.hasAddressTaken()) &&
949 F.getCallingConv() != CallingConv::ARM64EC_Thunk_Native &&
950 F.getCallingConv() != CallingConv::ARM64EC_Thunk_X64) {
951 if (!F.hasComdat())
952 F.setComdat(Mod.getOrInsertComdat(Name: F.getName()));
953 ThunkMapping.push_back(
954 Elt: {.Src: &F, .Dst: buildEntryThunk(F: &F), .Kind: Arm64ECThunkType::Entry});
955 }
956 }
957 for (GlobalValue *O : DirectCalledFns) {
958 auto GA = dyn_cast<GlobalAlias>(Val: O);
959 auto F = dyn_cast<Function>(Val: GA ? GA->getAliasee() : O);
960 ThunkMapping.push_back(
961 Elt: {.Src: O, .Dst: buildExitThunk(FT: F->getFunctionType(), Attrs: F->getAttributes()),
962 .Kind: Arm64ECThunkType::Exit});
963 if (!GA && !F->hasDLLImportStorageClass())
964 ThunkMapping.push_back(
965 Elt: {.Src: buildGuestExitThunk(F), .Dst: F, .Kind: Arm64ECThunkType::GuestExit});
966 }
967 for (GlobalAlias *A : PatchableFns) {
968 Function *Thunk = buildPatchableThunk(UnmangledAlias: A, MangledAlias: FnsMap[A]);
969 ThunkMapping.push_back(Elt: {.Src: Thunk, .Dst: A, .Kind: Arm64ECThunkType::GuestExit});
970 }
971
972 if (!ThunkMapping.empty()) {
973 SmallVector<Constant *> ThunkMappingArrayElems;
974 for (ThunkInfo &Thunk : ThunkMapping) {
975 ThunkMappingArrayElems.push_back(Elt: ConstantStruct::getAnon(
976 V: {Thunk.Src, Thunk.Dst,
977 ConstantInt::get(Context&: M->getContext(), V: APInt(32, uint8_t(Thunk.Kind)))}));
978 }
979 Constant *ThunkMappingArray = ConstantArray::get(
980 T: llvm::ArrayType::get(ElementType: ThunkMappingArrayElems[0]->getType(),
981 NumElements: ThunkMappingArrayElems.size()),
982 V: ThunkMappingArrayElems);
983 new GlobalVariable(Mod, ThunkMappingArray->getType(), /*isConstant*/ false,
984 GlobalValue::ExternalLinkage, ThunkMappingArray,
985 "llvm.arm64ec.symbolmap");
986 }
987
988 return true;
989}
990
991bool AArch64Arm64ECCallLowering::processFunction(
992 Function &F, SetVector<GlobalValue *> &DirectCalledFns,
993 DenseMap<GlobalAlias *, GlobalAlias *> &FnsMap) {
994 SmallVector<CallBase *, 8> IndirectCalls;
995
996 // For ARM64EC targets, a function definition's name is mangled differently
997 // from the normal symbol. We currently have no representation of this sort
998 // of symbol in IR, so we change the name to the mangled name, then store
999 // the unmangled name as metadata. Later passes that need the unmangled
1000 // name (emitting the definition) can grab it from the metadata.
1001 //
1002 // FIXME: Handle functions with weak linkage?
1003 if (!F.hasLocalLinkage() || F.hasAddressTaken()) {
1004 if (std::optional<std::string> MangledName =
1005 getArm64ECMangledFunctionName(GV&: F)) {
1006 F.addMetadata(Kind: "arm64ec_unmangled_name",
1007 MD&: *MDNode::get(Context&: M->getContext(),
1008 MDs: MDString::get(Context&: M->getContext(), Str: F.getName())));
1009 if (F.hasComdat() && F.getComdat()->getName() == F.getName()) {
1010 Comdat *MangledComdat = M->getOrInsertComdat(Name: MangledName.value());
1011 SmallVector<GlobalObject *> ComdatUsers =
1012 to_vector(Range: F.getComdat()->getUsers());
1013 for (GlobalObject *User : ComdatUsers)
1014 User->setComdat(MangledComdat);
1015 }
1016 F.setName(MangledName.value());
1017 }
1018 }
1019
1020 // Iterate over the instructions to find all indirect call/invoke/callbr
1021 // instructions. Make a separate list of pointers to indirect
1022 // call/invoke/callbr instructions because the original instructions will be
1023 // deleted as the checks are added.
1024 for (BasicBlock &BB : F) {
1025 for (Instruction &I : BB) {
1026 auto *CB = dyn_cast<CallBase>(Val: &I);
1027 if (!CB || CB->getCallingConv() == CallingConv::ARM64EC_Thunk_X64 ||
1028 CB->isInlineAsm())
1029 continue;
1030
1031 // We need to instrument any call that isn't directly calling an
1032 // ARM64 function.
1033 //
1034 // FIXME: getCalledFunction() fails if there's a bitcast (e.g.
1035 // unprototyped functions in C)
1036 if (Function *F = CB->getCalledFunction()) {
1037 if (!CLOpts->arm64ec_lower_direct_to_indirect || F->hasLocalLinkage() ||
1038 F->isIntrinsic() || !F->isDeclarationForLinker())
1039 continue;
1040
1041 DirectCalledFns.insert(X: F);
1042 continue;
1043 }
1044
1045 // Use mangled global alias for direct calls to patchable functions.
1046 if (GlobalAlias *A = dyn_cast<GlobalAlias>(Val: CB->getCalledOperand())) {
1047 auto I = FnsMap.find(Val: A);
1048 if (I != FnsMap.end()) {
1049 CB->setCalledOperand(I->second);
1050 DirectCalledFns.insert(X: I->first);
1051 continue;
1052 }
1053 }
1054
1055 IndirectCalls.push_back(Elt: CB);
1056 ++Arm64ECCallsLowered;
1057 }
1058 }
1059
1060 if (IndirectCalls.empty())
1061 return false;
1062
1063 for (CallBase *CB : IndirectCalls)
1064 lowerCall(CB);
1065
1066 return true;
1067}
1068
1069char AArch64Arm64ECCallLowering::ID = 0;
1070INITIALIZE_PASS(AArch64Arm64ECCallLowering, "Arm64ECCallLowering",
1071 "AArch64Arm64ECCallLowering", false, false)
1072
1073ModulePass *llvm::createAArch64Arm64ECCallLoweringPass() {
1074 return new AArch64Arm64ECCallLowering;
1075}
1076