1//===-- AArch64Arm64ECCallLowering.cpp - Lower Arm64EC calls ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file contains the IR transform to lower external or indirect calls for
11/// the ARM64EC calling convention. Such calls must go through the runtime, so
12/// we can translate the calling convention for calls into the emulator.
13///
14/// This subsumes Control Flow Guard handling.
15///
16//===----------------------------------------------------------------------===//
17
18#include "AArch64.h"
19#include "llvm/ADT/SetVector.h"
20#include "llvm/ADT/SmallString.h"
21#include "llvm/ADT/SmallVector.h"
22#include "llvm/ADT/Statistic.h"
23#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/DiagnosticInfo.h"
25#include "llvm/IR/GlobalAlias.h"
26#include "llvm/IR/IRBuilder.h"
27#include "llvm/IR/Instruction.h"
28#include "llvm/IR/Mangler.h"
29#include "llvm/IR/Module.h"
30#include "llvm/Object/COFF.h"
31#include "llvm/Pass.h"
32#include "llvm/Support/CommandLine.h"
33#include "llvm/TargetParser/Triple.h"
34
35using namespace llvm;
36using namespace llvm::COFF;
37
38using OperandBundleDef = OperandBundleDefT<Value *>;
39
40#define DEBUG_TYPE "arm64eccalllowering"
41
42STATISTIC(Arm64ECCallsLowered, "Number of Arm64EC calls lowered");
43
44static cl::opt<bool> LowerDirectToIndirect("arm64ec-lower-direct-to-indirect",
45 cl::Hidden, cl::init(Val: true));
46static cl::opt<bool> GenerateThunks("arm64ec-generate-thunks", cl::Hidden,
47 cl::init(Val: true));
48
49namespace {
50
51enum ThunkArgTranslation : uint8_t {
52 Direct,
53 Bitcast,
54 PointerIndirection,
55};
56
57struct ThunkArgInfo {
58 Type *Arm64Ty;
59 Type *X64Ty;
60 ThunkArgTranslation Translation;
61};
62
63class AArch64Arm64ECCallLowering : public ModulePass {
64public:
65 static char ID;
66 AArch64Arm64ECCallLowering() : ModulePass(ID) {}
67
68 Function *buildExitThunk(FunctionType *FnTy, AttributeList Attrs);
69 Function *buildEntryThunk(Function *F);
70 void lowerCall(CallBase *CB);
71 Function *buildGuestExitThunk(Function *F);
72 Function *buildPatchableThunk(GlobalAlias *UnmangledAlias,
73 GlobalAlias *MangledAlias);
74 bool processFunction(Function &F, SetVector<GlobalValue *> &DirectCalledFns,
75 DenseMap<GlobalAlias *, GlobalAlias *> &FnsMap);
76 bool runOnModule(Module &M) override;
77
78private:
79 ControlFlowGuardMode CFGuardModuleFlag = ControlFlowGuardMode::Disabled;
80 FunctionType *GuardFnType = nullptr;
81 FunctionType *DispatchFnType = nullptr;
82 Constant *GuardFnCFGlobal = nullptr;
83 Constant *GuardFnGlobal = nullptr;
84 Constant *DispatchFnGlobal = nullptr;
85 Module *M = nullptr;
86
87 Type *PtrTy;
88 Type *I64Ty;
89 Type *VoidTy;
90
91 void getThunkType(FunctionType *FT, AttributeList AttrList,
92 Arm64ECThunkType TT, raw_ostream &Out,
93 FunctionType *&Arm64Ty, FunctionType *&X64Ty,
94 SmallVector<ThunkArgTranslation> &ArgTranslations);
95 void getThunkRetType(FunctionType *FT, AttributeList AttrList,
96 raw_ostream &Out, Type *&Arm64RetTy, Type *&X64RetTy,
97 SmallVectorImpl<Type *> &Arm64ArgTypes,
98 SmallVectorImpl<Type *> &X64ArgTypes,
99 SmallVector<ThunkArgTranslation> &ArgTranslations,
100 bool &HasSretPtr);
101 void getThunkArgTypes(FunctionType *FT, AttributeList AttrList,
102 Arm64ECThunkType TT, raw_ostream &Out,
103 SmallVectorImpl<Type *> &Arm64ArgTypes,
104 SmallVectorImpl<Type *> &X64ArgTypes,
105 SmallVectorImpl<ThunkArgTranslation> &ArgTranslations,
106 bool HasSretPtr);
107 ThunkArgInfo canonicalizeThunkType(Type *T, Align Alignment, bool Ret,
108 uint64_t ArgSizeBytes, raw_ostream &Out);
109};
110
111} // end anonymous namespace
112
113void AArch64Arm64ECCallLowering::getThunkType(
114 FunctionType *FT, AttributeList AttrList, Arm64ECThunkType TT,
115 raw_ostream &Out, FunctionType *&Arm64Ty, FunctionType *&X64Ty,
116 SmallVector<ThunkArgTranslation> &ArgTranslations) {
117 Out << (TT == Arm64ECThunkType::Entry ? "$ientry_thunk$cdecl$"
118 : "$iexit_thunk$cdecl$");
119
120 Type *Arm64RetTy;
121 Type *X64RetTy;
122
123 SmallVector<Type *> Arm64ArgTypes;
124 SmallVector<Type *> X64ArgTypes;
125
126 // The first argument to a thunk is the called function, stored in x9.
127 // For exit thunks, we pass the called function down to the emulator;
128 // for entry/guest exit thunks, we just call the Arm64 function directly.
129 if (TT == Arm64ECThunkType::Exit)
130 Arm64ArgTypes.push_back(Elt: PtrTy);
131 X64ArgTypes.push_back(Elt: PtrTy);
132
133 bool HasSretPtr = false;
134 getThunkRetType(FT, AttrList, Out, Arm64RetTy, X64RetTy, Arm64ArgTypes,
135 X64ArgTypes, ArgTranslations, HasSretPtr);
136
137 getThunkArgTypes(FT, AttrList, TT, Out, Arm64ArgTypes, X64ArgTypes,
138 ArgTranslations, HasSretPtr);
139
140 Arm64Ty = FunctionType::get(Result: Arm64RetTy, Params: Arm64ArgTypes, isVarArg: false);
141
142 X64Ty = FunctionType::get(Result: X64RetTy, Params: X64ArgTypes, isVarArg: false);
143}
144
145void AArch64Arm64ECCallLowering::getThunkArgTypes(
146 FunctionType *FT, AttributeList AttrList, Arm64ECThunkType TT,
147 raw_ostream &Out, SmallVectorImpl<Type *> &Arm64ArgTypes,
148 SmallVectorImpl<Type *> &X64ArgTypes,
149 SmallVectorImpl<ThunkArgTranslation> &ArgTranslations, bool HasSretPtr) {
150
151 Out << "$";
152 if (FT->isVarArg()) {
153 // We treat the variadic function's thunk as a normal function
154 // with the following type on the ARM side:
155 // rettype exitthunk(
156 // ptr x9, ptr x0, i64 x1, i64 x2, i64 x3, ptr x4, i64 x5)
157 //
158 // that can coverage all types of variadic function.
159 // x9 is similar to normal exit thunk, store the called function.
160 // x0-x3 is the arguments be stored in registers.
161 // x4 is the address of the arguments on the stack.
162 // x5 is the size of the arguments on the stack.
163 //
164 // On the x64 side, it's the same except that x5 isn't set.
165 //
166 // If both the ARM and X64 sides are sret, there are only three
167 // arguments in registers.
168 //
169 // If the X64 side is sret, but the ARM side isn't, we pass an extra value
170 // to/from the X64 side, and let SelectionDAG transform it into a memory
171 // location.
172 Out << "varargs";
173
174 // x0-x3
175 for (int i = HasSretPtr ? 1 : 0; i < 4; i++) {
176 Arm64ArgTypes.push_back(Elt: I64Ty);
177 X64ArgTypes.push_back(Elt: I64Ty);
178 ArgTranslations.push_back(Elt: ThunkArgTranslation::Direct);
179 }
180
181 // x4
182 Arm64ArgTypes.push_back(Elt: PtrTy);
183 X64ArgTypes.push_back(Elt: PtrTy);
184 ArgTranslations.push_back(Elt: ThunkArgTranslation::Direct);
185 // x5
186 Arm64ArgTypes.push_back(Elt: I64Ty);
187 if (TT != Arm64ECThunkType::Entry) {
188 // FIXME: x5 isn't actually used by the x64 side; revisit once we
189 // have proper isel for varargs
190 X64ArgTypes.push_back(Elt: I64Ty);
191 ArgTranslations.push_back(Elt: ThunkArgTranslation::Direct);
192 }
193 return;
194 }
195
196 unsigned I = 0;
197 if (HasSretPtr)
198 I++;
199
200 if (I == FT->getNumParams()) {
201 Out << "v";
202 return;
203 }
204
205 for (unsigned E = FT->getNumParams(); I != E; ++I) {
206#if 0
207 // FIXME: Need more information about argument size; see
208 // https://reviews.llvm.org/D132926
209 uint64_t ArgSizeBytes = AttrList.getParamArm64ECArgSizeBytes(I);
210 Align ParamAlign = AttrList.getParamAlignment(I).valueOrOne();
211#else
212 uint64_t ArgSizeBytes = 0;
213 Align ParamAlign = Align();
214#endif
215 auto [Arm64Ty, X64Ty, ArgTranslation] =
216 canonicalizeThunkType(T: FT->getParamType(i: I), Alignment: ParamAlign,
217 /*Ret*/ false, ArgSizeBytes, Out);
218 Arm64ArgTypes.push_back(Elt: Arm64Ty);
219 X64ArgTypes.push_back(Elt: X64Ty);
220 ArgTranslations.push_back(Elt: ArgTranslation);
221 }
222}
223
224void AArch64Arm64ECCallLowering::getThunkRetType(
225 FunctionType *FT, AttributeList AttrList, raw_ostream &Out,
226 Type *&Arm64RetTy, Type *&X64RetTy, SmallVectorImpl<Type *> &Arm64ArgTypes,
227 SmallVectorImpl<Type *> &X64ArgTypes,
228 SmallVector<ThunkArgTranslation> &ArgTranslations, bool &HasSretPtr) {
229 Type *T = FT->getReturnType();
230#if 0
231 // FIXME: Need more information about argument size; see
232 // https://reviews.llvm.org/D132926
233 uint64_t ArgSizeBytes = AttrList.getRetArm64ECArgSizeBytes();
234#else
235 int64_t ArgSizeBytes = 0;
236#endif
237 if (T->isVoidTy()) {
238 if (FT->getNumParams()) {
239 Attribute SRetAttr0 = AttrList.getParamAttr(ArgNo: 0, Kind: Attribute::StructRet);
240 Attribute InRegAttr0 = AttrList.getParamAttr(ArgNo: 0, Kind: Attribute::InReg);
241 Attribute SRetAttr1, InRegAttr1;
242 if (FT->getNumParams() > 1) {
243 // Also check the second parameter (for class methods, the first
244 // parameter is "this", and the second parameter is the sret pointer.)
245 // It doesn't matter which one is sret.
246 SRetAttr1 = AttrList.getParamAttr(ArgNo: 1, Kind: Attribute::StructRet);
247 InRegAttr1 = AttrList.getParamAttr(ArgNo: 1, Kind: Attribute::InReg);
248 }
249 if ((SRetAttr0.isValid() && InRegAttr0.isValid()) ||
250 (SRetAttr1.isValid() && InRegAttr1.isValid())) {
251 // sret+inreg indicates a call that returns a C++ class value. This is
252 // actually equivalent to just passing and returning a void* pointer
253 // as the first or second argument. Translate it that way, instead of
254 // trying to model "inreg" in the thunk's calling convention; this
255 // simplfies the rest of the code, and matches MSVC mangling.
256 Out << "i8";
257 Arm64RetTy = I64Ty;
258 X64RetTy = I64Ty;
259 return;
260 }
261 if (SRetAttr0.isValid()) {
262 // FIXME: Sanity-check the sret type; if it's an integer or pointer,
263 // we'll get screwy mangling/codegen.
264 // FIXME: For large struct types, mangle as an integer argument and
265 // integer return, so we can reuse more thunks, instead of "m" syntax.
266 // (MSVC mangles this case as an integer return with no argument, but
267 // that's a miscompile.)
268 Type *SRetType = SRetAttr0.getValueAsType();
269 Align SRetAlign = AttrList.getParamAlignment(ArgNo: 0).valueOrOne();
270 canonicalizeThunkType(T: SRetType, Alignment: SRetAlign, /*Ret*/ true, ArgSizeBytes,
271 Out);
272 Arm64RetTy = VoidTy;
273 X64RetTy = VoidTy;
274 Arm64ArgTypes.push_back(Elt: FT->getParamType(i: 0));
275 X64ArgTypes.push_back(Elt: FT->getParamType(i: 0));
276 ArgTranslations.push_back(Elt: ThunkArgTranslation::Direct);
277 HasSretPtr = true;
278 return;
279 }
280 }
281
282 Out << "v";
283 Arm64RetTy = VoidTy;
284 X64RetTy = VoidTy;
285 return;
286 }
287
288 auto info =
289 canonicalizeThunkType(T, Alignment: Align(), /*Ret*/ true, ArgSizeBytes, Out);
290 Arm64RetTy = info.Arm64Ty;
291 X64RetTy = info.X64Ty;
292 if (X64RetTy->isPointerTy()) {
293 // If the X64 type is canonicalized to a pointer, that means it's
294 // passed/returned indirectly. For a return value, that means it's an
295 // sret pointer.
296 X64ArgTypes.push_back(Elt: X64RetTy);
297 X64RetTy = VoidTy;
298 }
299}
300
301ThunkArgInfo AArch64Arm64ECCallLowering::canonicalizeThunkType(
302 Type *T, Align Alignment, bool Ret, uint64_t ArgSizeBytes,
303 raw_ostream &Out) {
304
305 auto direct = [](Type *T) {
306 return ThunkArgInfo{.Arm64Ty: T, .X64Ty: T, .Translation: ThunkArgTranslation::Direct};
307 };
308
309 auto bitcast = [this](Type *Arm64Ty, uint64_t SizeInBytes) {
310 return ThunkArgInfo{.Arm64Ty: Arm64Ty,
311 .X64Ty: llvm::Type::getIntNTy(C&: M->getContext(), N: SizeInBytes * 8),
312 .Translation: ThunkArgTranslation::Bitcast};
313 };
314
315 auto pointerIndirection = [this](Type *Arm64Ty) {
316 return ThunkArgInfo{.Arm64Ty: Arm64Ty, .X64Ty: PtrTy,
317 .Translation: ThunkArgTranslation::PointerIndirection};
318 };
319
320 if (T->isHalfTy()) {
321 // Prefix with `llvm` since MSVC doesn't specify `_Float16`
322 Out << "__llvm_h__";
323 return direct(T);
324 }
325
326 if (T->isBFloatTy()) {
327 // Prefix with `llvm` since MSVC doesn't specify `__bf16`
328 Out << "__llvm_bf16__";
329 return direct(T);
330 }
331
332 if (T->isFloatTy()) {
333 Out << "f";
334 return direct(T);
335 }
336
337 if (T->isDoubleTy()) {
338 Out << "d";
339 return direct(T);
340 }
341
342 if (T->isFP128Ty()) {
343 // Prefix with `llvm` since MSVC doesn't specify `_Float128`
344 Out << "__llvm_q__";
345 // On windows f128 is passed indirectly, and Clang/LLVM
346 // returns using sret for compatibility with GCC.
347 return pointerIndirection(T);
348 }
349
350 if (T->isFloatingPointTy()) {
351 report_fatal_error(
352 reason: "Only half, bfloat16, float, double, and fp128 are supported "
353 "for ARM64EC thunks");
354 }
355
356 auto &DL = M->getDataLayout();
357
358 if (auto *StructTy = dyn_cast<StructType>(Val: T))
359 if (StructTy->getNumElements() == 1)
360 T = StructTy->getElementType(N: 0);
361
362 if (T->isArrayTy()) {
363 Type *ElementTy = T->getArrayElementType();
364 uint64_t ElementCnt = T->getArrayNumElements();
365 uint64_t ElementSizePerBytes = DL.getTypeSizeInBits(Ty: ElementTy) / 8;
366 uint64_t TotalSizeBytes = ElementCnt * ElementSizePerBytes;
367 if (ElementTy->isHalfTy() || ElementTy->isBFloatTy() ||
368 ElementTy->isFloatTy() || ElementTy->isDoubleTy() ||
369 ElementTy->isFP128Ty()) {
370 if (ElementTy->isHalfTy())
371 // Prefix with `llvm` since MSVC doesn't specify `_Float16`
372 Out << "__llvm_H__";
373 else if (ElementTy->isBFloatTy())
374 // Prefix with `llvm` since MSVC doesn't specify `__bf16`
375 Out << "__llvm_BF16__";
376 else if (ElementTy->isFloatTy())
377 Out << "F";
378 else if (ElementTy->isDoubleTy())
379 Out << "D";
380 else if (ElementTy->isFP128Ty())
381 // Prefix with `llvm` since MSVC doesn't specify `_Float128`
382 Out << "__llvm_Q__";
383 Out << TotalSizeBytes;
384 if (Alignment.value() >= 16 && !Ret)
385 Out << "a" << Alignment.value();
386 if (TotalSizeBytes <= 8) {
387 // Arm64 returns small structs of float/double in float registers;
388 // X64 uses RAX.
389 return bitcast(T, TotalSizeBytes);
390 } else {
391 // Struct is passed directly on Arm64, but indirectly on X64.
392 return pointerIndirection(T);
393 }
394 } else if (ElementTy->isFloatingPointTy()) {
395 report_fatal_error(
396 reason: "Only half, bfloat16, float, double, and fp128 are supported "
397 "for ARM64EC thunks");
398 }
399 }
400
401 if ((T->isIntegerTy() || T->isPointerTy()) && DL.getTypeSizeInBits(Ty: T) <= 64) {
402 Out << "i8";
403 return direct(I64Ty);
404 }
405
406 unsigned TypeSize = ArgSizeBytes;
407 if (TypeSize == 0)
408 TypeSize = DL.getTypeSizeInBits(Ty: T) / 8;
409 Out << "m";
410 if (TypeSize != 4)
411 Out << TypeSize;
412 if (Alignment.value() >= 16 && !Ret)
413 Out << "a" << Alignment.value();
414 // FIXME: Try to canonicalize Arm64Ty more thoroughly?
415 if (TypeSize == 1 || TypeSize == 2 || TypeSize == 4 || TypeSize == 8) {
416 // Pass directly in an integer register
417 return bitcast(T, TypeSize);
418 } else {
419 // Passed directly on Arm64, but indirectly on X64.
420 return pointerIndirection(T);
421 }
422}
423
424// This function builds the "exit thunk", a function which translates
425// arguments and return values when calling x64 code from AArch64 code.
426Function *AArch64Arm64ECCallLowering::buildExitThunk(FunctionType *FT,
427 AttributeList Attrs) {
428 SmallString<256> ExitThunkName;
429 llvm::raw_svector_ostream ExitThunkStream(ExitThunkName);
430 FunctionType *Arm64Ty, *X64Ty;
431 SmallVector<ThunkArgTranslation> ArgTranslations;
432 getThunkType(FT, AttrList: Attrs, TT: Arm64ECThunkType::Exit, Out&: ExitThunkStream, Arm64Ty,
433 X64Ty, ArgTranslations);
434 if (Function *F = M->getFunction(Name: ExitThunkName))
435 return F;
436
437 Function *F = Function::Create(Ty: Arm64Ty, Linkage: GlobalValue::LinkOnceODRLinkage, AddrSpace: 0,
438 N: ExitThunkName, M);
439 F->setCallingConv(CallingConv::ARM64EC_Thunk_Native);
440 F->setSection(".wowthk$aa");
441 F->setComdat(M->getOrInsertComdat(Name: ExitThunkName));
442 // Copy MSVC, and always set up a frame pointer. (Maybe this isn't necessary.)
443 F->addFnAttr(Kind: "frame-pointer", Val: "all");
444 // Only copy sret from the first argument. For C++ instance methods, clang can
445 // stick an sret marking on a later argument, but it doesn't actually affect
446 // the ABI, so we can omit it. This avoids triggering a verifier assertion.
447 if (FT->getNumParams()) {
448 auto SRet = Attrs.getParamAttr(ArgNo: 0, Kind: Attribute::StructRet);
449 auto InReg = Attrs.getParamAttr(ArgNo: 0, Kind: Attribute::InReg);
450 if (SRet.isValid() && !InReg.isValid())
451 F->addParamAttr(ArgNo: 1, Attr: SRet);
452 }
453 // FIXME: Copy anything other than sret? Shouldn't be necessary for normal
454 // C ABI, but might show up in other cases.
455 BasicBlock *BB = BasicBlock::Create(Context&: M->getContext(), Name: "", Parent: F);
456 IRBuilder<> IRB(BB);
457 Value *CalleePtr =
458 M->getOrInsertGlobal(Name: "__os_arm64x_dispatch_call_no_redirect", Ty: PtrTy);
459 Value *Callee = IRB.CreateLoad(Ty: PtrTy, Ptr: CalleePtr);
460 auto &DL = M->getDataLayout();
461 SmallVector<Value *> Args;
462 FunctionType *DispatcherCallTy = X64Ty;
463 // If we have a vararg function, the SelectionDAG lowering will need to
464 // recognize this so it can copy the arguments described by x4 (pointer) and
465 // x5 (length) to set up the x86-64 context correctly.
466 if (FT->isVarArg())
467 DispatcherCallTy =
468 FunctionType::get(Result: X64Ty->getReturnType(), Params: X64Ty->params(),
469 /*isVarArg=*/true);
470
471 // Pass the called function in x9.
472 auto X64TyOffset = 1;
473 Args.push_back(Elt: F->arg_begin());
474
475 Type *RetTy = Arm64Ty->getReturnType();
476 if (RetTy != X64Ty->getReturnType()) {
477 // If the return type is an array or struct, translate it. Values of size
478 // 8 or less go into RAX; bigger values go into memory, and we pass a
479 // pointer.
480 if (DL.getTypeStoreSize(Ty: RetTy) > 8) {
481 Args.push_back(Elt: IRB.CreateAlloca(Ty: RetTy));
482 X64TyOffset++;
483 }
484 }
485
486 for (auto [Arg, X64ArgType, ArgTranslation] : llvm::zip_equal(
487 t: make_range(x: F->arg_begin() + 1, y: F->arg_end()),
488 u: make_range(x: X64Ty->param_begin() + X64TyOffset, y: X64Ty->param_end()),
489 args&: ArgTranslations)) {
490 // Translate arguments from AArch64 calling convention to x86 calling
491 // convention.
492 //
493 // For simple types, we don't need to do any translation: they're
494 // represented the same way. (Implicit sign extension is not part of
495 // either convention.)
496 //
497 // The big thing we have to worry about is struct types... but
498 // fortunately AArch64 clang is pretty friendly here: the cases that need
499 // translation are always passed as a struct or array. (If we run into
500 // some cases where this doesn't work, we can teach clang to mark it up
501 // with an attribute.)
502 //
503 // The first argument is the called function, stored in x9.
504 if (ArgTranslation != ThunkArgTranslation::Direct) {
505 Value *Mem = IRB.CreateAlloca(Ty: Arg.getType());
506 IRB.CreateStore(Val: &Arg, Ptr: Mem);
507 if (ArgTranslation == ThunkArgTranslation::Bitcast) {
508 Type *IntTy = IRB.getIntNTy(N: DL.getTypeStoreSizeInBits(Ty: Arg.getType()));
509 Args.push_back(Elt: IRB.CreateLoad(Ty: IntTy, Ptr: Mem));
510 } else {
511 assert(ArgTranslation == ThunkArgTranslation::PointerIndirection);
512 Args.push_back(Elt: Mem);
513 }
514 } else {
515 Args.push_back(Elt: &Arg);
516 }
517 assert(Args.back()->getType() == X64ArgType);
518 }
519 // FIXME: Transfer necessary attributes? sret? anything else?
520
521 CallInst *Call = IRB.CreateCall(FTy: DispatcherCallTy, Callee, Args);
522 Call->setCallingConv(CallingConv::ARM64EC_Thunk_X64);
523
524 Value *RetVal = Call;
525 if (RetTy != X64Ty->getReturnType()) {
526 // If we rewrote the return type earlier, convert the return value to
527 // the proper type.
528 if (DL.getTypeStoreSize(Ty: RetTy) > 8) {
529 RetVal = IRB.CreateLoad(Ty: RetTy, Ptr: Args[1]);
530 } else {
531 Value *CastAlloca = IRB.CreateAlloca(Ty: RetTy);
532 IRB.CreateStore(Val: Call, Ptr: CastAlloca);
533 RetVal = IRB.CreateLoad(Ty: RetTy, Ptr: CastAlloca);
534 }
535 }
536
537 if (RetTy->isVoidTy())
538 IRB.CreateRetVoid();
539 else
540 IRB.CreateRet(V: RetVal);
541 return F;
542}
543
544// This function builds the "entry thunk", a function which translates
545// arguments and return values when calling AArch64 code from x64 code.
546Function *AArch64Arm64ECCallLowering::buildEntryThunk(Function *F) {
547 SmallString<256> EntryThunkName;
548 llvm::raw_svector_ostream EntryThunkStream(EntryThunkName);
549 FunctionType *Arm64Ty, *X64Ty;
550 SmallVector<ThunkArgTranslation> ArgTranslations;
551 getThunkType(FT: F->getFunctionType(), AttrList: F->getAttributes(),
552 TT: Arm64ECThunkType::Entry, Out&: EntryThunkStream, Arm64Ty, X64Ty,
553 ArgTranslations);
554 if (Function *F = M->getFunction(Name: EntryThunkName))
555 return F;
556
557 Function *Thunk = Function::Create(Ty: X64Ty, Linkage: GlobalValue::LinkOnceODRLinkage, AddrSpace: 0,
558 N: EntryThunkName, M);
559 Thunk->setCallingConv(CallingConv::ARM64EC_Thunk_X64);
560 Thunk->setSection(".wowthk$aa");
561 Thunk->setComdat(M->getOrInsertComdat(Name: EntryThunkName));
562 // Copy MSVC, and always set up a frame pointer. (Maybe this isn't necessary.)
563 Thunk->addFnAttr(Kind: "frame-pointer", Val: "all");
564
565 BasicBlock *BB = BasicBlock::Create(Context&: M->getContext(), Name: "", Parent: Thunk);
566 IRBuilder<> IRB(BB);
567
568 Type *RetTy = Arm64Ty->getReturnType();
569 Type *X64RetType = X64Ty->getReturnType();
570
571 bool TransformDirectToSRet = X64RetType->isVoidTy() && !RetTy->isVoidTy();
572 unsigned ThunkArgOffset = TransformDirectToSRet ? 2 : 1;
573 unsigned PassthroughArgSize =
574 (F->isVarArg() ? 5 : Thunk->arg_size()) - ThunkArgOffset;
575 assert(ArgTranslations.size() == (F->isVarArg() ? 5 : PassthroughArgSize));
576
577 // Translate arguments to call.
578 SmallVector<Value *> Args;
579 for (unsigned i = 0; i != PassthroughArgSize; ++i) {
580 Value *Arg = Thunk->getArg(i: i + ThunkArgOffset);
581 Type *ArgTy = Arm64Ty->getParamType(i);
582 ThunkArgTranslation ArgTranslation = ArgTranslations[i];
583 if (ArgTranslation != ThunkArgTranslation::Direct) {
584 // Translate array/struct arguments to the expected type.
585 if (ArgTranslation == ThunkArgTranslation::Bitcast) {
586 Value *CastAlloca = IRB.CreateAlloca(Ty: ArgTy);
587 IRB.CreateStore(Val: Arg, Ptr: CastAlloca);
588 Arg = IRB.CreateLoad(Ty: ArgTy, Ptr: CastAlloca);
589 } else {
590 assert(ArgTranslation == ThunkArgTranslation::PointerIndirection);
591 Arg = IRB.CreateLoad(Ty: ArgTy, Ptr: Arg);
592 }
593 }
594 assert(Arg->getType() == ArgTy);
595 Args.push_back(Elt: Arg);
596 }
597
598 if (F->isVarArg()) {
599 // The 5th argument to variadic entry thunks is used to model the x64 sp
600 // which is passed to the thunk in x4, this can be passed to the callee as
601 // the variadic argument start address after skipping over the 32 byte
602 // shadow store.
603
604 // The EC thunk CC will assign any argument marked as InReg to x4.
605 Thunk->addParamAttr(ArgNo: 5, Kind: Attribute::InReg);
606 Value *Arg = Thunk->getArg(i: 5);
607 Arg = IRB.CreatePtrAdd(Ptr: Arg, Offset: IRB.getInt64(C: 0x20));
608 Args.push_back(Elt: Arg);
609
610 // Pass in a zero variadic argument size (in x5).
611 Args.push_back(Elt: IRB.getInt64(C: 0));
612 }
613
614 // Call the function passed to the thunk.
615 Value *Callee = Thunk->getArg(i: 0);
616 CallInst *Call = IRB.CreateCall(FTy: Arm64Ty, Callee, Args);
617
618 auto SRetAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::StructRet);
619 auto InRegAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::InReg);
620 if (SRetAttr.isValid() && !InRegAttr.isValid()) {
621 Thunk->addParamAttr(ArgNo: 1, Attr: SRetAttr);
622 Call->addParamAttr(ArgNo: 0, Attr: SRetAttr);
623 }
624
625 Value *RetVal = Call;
626 if (TransformDirectToSRet) {
627 // The x64 side returns this value indirectly via a hidden pointer (sret).
628 // Mark the thunk's pointer arg with sret so that ISel saves it and copies
629 // it into x8 (RAX) on return, matching the x64 calling convention.
630 Thunk->addParamAttr(
631 ArgNo: 1, Attr: Attribute::getWithStructRetType(Context&: M->getContext(), Ty: RetTy));
632 IRB.CreateStore(Val: RetVal, Ptr: Thunk->getArg(i: 1));
633 } else if (X64RetType != RetTy) {
634 Value *CastAlloca = IRB.CreateAlloca(Ty: X64RetType);
635 IRB.CreateStore(Val: Call, Ptr: CastAlloca);
636 RetVal = IRB.CreateLoad(Ty: X64RetType, Ptr: CastAlloca);
637 }
638
639 // Return to the caller. Note that the isel has code to translate this
640 // "ret" to a tail call to __os_arm64x_dispatch_ret. (Alternatively, we
641 // could emit a tail call here, but that would require a dedicated calling
642 // convention, which seems more complicated overall.)
643 if (X64RetType->isVoidTy())
644 IRB.CreateRetVoid();
645 else
646 IRB.CreateRet(V: RetVal);
647
648 return Thunk;
649}
650
651std::optional<std::string> getArm64ECMangledFunctionName(GlobalValue &GV) {
652 if (!GV.hasName()) {
653 GV.setName("__unnamed");
654 }
655
656 return llvm::getArm64ECMangledFunctionName(Name: GV.getName());
657}
658
659// Builds the "guest exit thunk", a helper to call a function which may or may
660// not be an exit thunk. (We optimistically assume non-dllimport function
661// declarations refer to functions defined in AArch64 code; if the linker
662// can't prove that, we use this routine instead.)
663Function *AArch64Arm64ECCallLowering::buildGuestExitThunk(Function *F) {
664 llvm::raw_null_ostream NullThunkName;
665 FunctionType *Arm64Ty, *X64Ty;
666 SmallVector<ThunkArgTranslation> ArgTranslations;
667 getThunkType(FT: F->getFunctionType(), AttrList: F->getAttributes(),
668 TT: Arm64ECThunkType::GuestExit, Out&: NullThunkName, Arm64Ty, X64Ty,
669 ArgTranslations);
670 auto MangledName = getArm64ECMangledFunctionName(GV&: *F);
671 assert(MangledName && "Can't guest exit to function that's already native");
672 std::string ThunkName = *MangledName;
673 if (ThunkName[0] == '?' && ThunkName.find(s: "@") != std::string::npos) {
674 ThunkName.insert(pos: ThunkName.find(s: "@"), s: "$exit_thunk");
675 } else {
676 ThunkName.append(s: "$exit_thunk");
677 }
678 Function *GuestExit =
679 Function::Create(Ty: Arm64Ty, Linkage: GlobalValue::WeakODRLinkage, AddrSpace: 0, N: ThunkName, M);
680 GuestExit->setComdat(M->getOrInsertComdat(Name: ThunkName));
681 GuestExit->setSection(".wowthk$aa");
682 GuestExit->addMetadata(
683 Kind: "arm64ec_unmangled_name",
684 MD&: *MDNode::get(Context&: M->getContext(),
685 MDs: MDString::get(Context&: M->getContext(), Str: F->getName())));
686 GuestExit->setMetadata(
687 Kind: "arm64ec_ecmangled_name",
688 Node: MDNode::get(Context&: M->getContext(),
689 MDs: MDString::get(Context&: M->getContext(), Str: *MangledName)));
690 F->setMetadata(Kind: "arm64ec_hasguestexit", Node: MDNode::get(Context&: M->getContext(), MDs: {}));
691 BasicBlock *BB = BasicBlock::Create(Context&: M->getContext(), Name: "", Parent: GuestExit);
692 IRBuilder<> B(BB);
693
694 // Create new call instruction. The call check should always be a call,
695 // even if the original CallBase is an Invoke or CallBr instructio.
696 // This is treated as a direct call, so do not use GuardFnCFGlobal.
697 LoadInst *GuardCheckLoad = B.CreateLoad(Ty: PtrTy, Ptr: GuardFnGlobal);
698 Function *Thunk = buildExitThunk(FT: F->getFunctionType(), Attrs: F->getAttributes());
699 CallInst *GuardCheck = B.CreateCall(
700 FTy: GuardFnType, Callee: GuardCheckLoad, Args: {F, Thunk});
701 Value *GuardCheckDest = B.CreateExtractValue(Agg: GuardCheck, Idxs: 0);
702 Value *GuardFinalDest = B.CreateExtractValue(Agg: GuardCheck, Idxs: 1);
703
704 // Ensure that the first argument is passed in the correct register.
705 GuardCheck->setCallingConv(CallingConv::CFGuard_Check);
706
707 SmallVector<Value *> Args(llvm::make_pointer_range(Range: GuestExit->args()));
708 OperandBundleDef OB("cfguardtarget", GuardFinalDest);
709 CallInst *Call = B.CreateCall(FTy: Arm64Ty, Callee: GuardCheckDest, Args, OpBundles: OB);
710 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
711
712 if (Call->getType()->isVoidTy())
713 B.CreateRetVoid();
714 else
715 B.CreateRet(V: Call);
716
717 auto SRetAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::StructRet);
718 auto InRegAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::InReg);
719 if (SRetAttr.isValid() && !InRegAttr.isValid()) {
720 GuestExit->addParamAttr(ArgNo: 0, Attr: SRetAttr);
721 Call->addParamAttr(ArgNo: 0, Attr: SRetAttr);
722 }
723
724 return GuestExit;
725}
726
727Function *
728AArch64Arm64ECCallLowering::buildPatchableThunk(GlobalAlias *UnmangledAlias,
729 GlobalAlias *MangledAlias) {
730 llvm::raw_null_ostream NullThunkName;
731 FunctionType *Arm64Ty, *X64Ty;
732 Function *F = cast<Function>(Val: MangledAlias->getAliasee());
733 SmallVector<ThunkArgTranslation> ArgTranslations;
734 getThunkType(FT: F->getFunctionType(), AttrList: F->getAttributes(),
735 TT: Arm64ECThunkType::GuestExit, Out&: NullThunkName, Arm64Ty, X64Ty,
736 ArgTranslations);
737 std::string ThunkName(MangledAlias->getName());
738 if (ThunkName[0] == '?' && ThunkName.find(s: "@") != std::string::npos) {
739 ThunkName.insert(pos: ThunkName.find(s: "@"), s: "$hybpatch_thunk");
740 } else {
741 ThunkName.append(s: "$hybpatch_thunk");
742 }
743
744 Function *GuestExit =
745 Function::Create(Ty: Arm64Ty, Linkage: GlobalValue::WeakODRLinkage, AddrSpace: 0, N: ThunkName, M);
746 GuestExit->setComdat(M->getOrInsertComdat(Name: ThunkName));
747 GuestExit->setSection(".wowthk$aa");
748 BasicBlock *BB = BasicBlock::Create(Context&: M->getContext(), Name: "", Parent: GuestExit);
749 IRBuilder<> B(BB);
750
751 // Load the global symbol as a pointer to the check function.
752 LoadInst *DispatchLoad = B.CreateLoad(Ty: PtrTy, Ptr: DispatchFnGlobal);
753
754 // Create new dispatch call instruction.
755 Function *ExitThunk =
756 buildExitThunk(FT: F->getFunctionType(), Attrs: F->getAttributes());
757 CallInst *Dispatch =
758 B.CreateCall(FTy: DispatchFnType, Callee: DispatchLoad,
759 Args: {UnmangledAlias, ExitThunk, UnmangledAlias->getAliasee()});
760
761 // Ensure that the first arguments are passed in the correct registers.
762 Dispatch->setCallingConv(CallingConv::CFGuard_Check);
763
764 SmallVector<Value *> Args(llvm::make_pointer_range(Range: GuestExit->args()));
765 CallInst *Call = B.CreateCall(FTy: Arm64Ty, Callee: Dispatch, Args);
766 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
767
768 if (Call->getType()->isVoidTy())
769 B.CreateRetVoid();
770 else
771 B.CreateRet(V: Call);
772
773 auto SRetAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::StructRet);
774 auto InRegAttr = F->getAttributes().getParamAttr(ArgNo: 0, Kind: Attribute::InReg);
775 if (SRetAttr.isValid() && !InRegAttr.isValid()) {
776 GuestExit->addParamAttr(ArgNo: 0, Attr: SRetAttr);
777 Call->addParamAttr(ArgNo: 0, Attr: SRetAttr);
778 }
779
780 MangledAlias->setAliasee(GuestExit);
781 return GuestExit;
782}
783
784// Lower an indirect call with inline code.
785void AArch64Arm64ECCallLowering::lowerCall(CallBase *CB) {
786 IRBuilder<> B(CB);
787 Value *CalledOperand = CB->getCalledOperand();
788
789 // If the indirect call is called within catchpad or cleanuppad,
790 // we need to copy "funclet" bundle of the call.
791 SmallVector<llvm::OperandBundleDef, 1> Bundles;
792 if (auto Bundle = CB->getOperandBundle(ID: LLVMContext::OB_funclet))
793 Bundles.push_back(Elt: OperandBundleDef(*Bundle));
794
795 // Load the global symbol as a pointer to the check function.
796 Value *GuardFn;
797 if ((CFGuardModuleFlag == ControlFlowGuardMode::Enabled) &&
798 !CB->hasFnAttr(Kind: "guard_nocf"))
799 GuardFn = GuardFnCFGlobal;
800 else
801 GuardFn = GuardFnGlobal;
802 LoadInst *GuardCheckLoad = B.CreateLoad(Ty: PtrTy, Ptr: GuardFn);
803
804 // Create new call instruction. The CFGuard check should always be a call,
805 // even if the original CallBase is an Invoke or CallBr instruction.
806 Function *Thunk = buildExitThunk(FT: CB->getFunctionType(), Attrs: CB->getAttributes());
807 CallInst *GuardCheck =
808 B.CreateCall(FTy: GuardFnType, Callee: GuardCheckLoad, Args: {CalledOperand, Thunk},
809 OpBundles: Bundles);
810 Value *GuardCheckDest = B.CreateExtractValue(Agg: GuardCheck, Idxs: 0);
811 Value *GuardFinalDest = B.CreateExtractValue(Agg: GuardCheck, Idxs: 1);
812
813 // Ensure that the first argument is passed in the correct register.
814 GuardCheck->setCallingConv(CallingConv::CFGuard_Check);
815
816 // Update the call: set the callee, and add a bundle with the final
817 // destination,
818 CB->setCalledOperand(GuardCheckDest);
819 OperandBundleDef OB("cfguardtarget", GuardFinalDest);
820 auto *NewCall = CallBase::addOperandBundle(CB, ID: LLVMContext::OB_cfguardtarget,
821 OB, InsertPt: CB->getIterator());
822 NewCall->copyMetadata(SrcInst: *CB);
823 CB->replaceAllUsesWith(V: NewCall);
824 CB->eraseFromParent();
825}
826
827bool AArch64Arm64ECCallLowering::runOnModule(Module &Mod) {
828 if (!GenerateThunks)
829 return false;
830
831 M = &Mod;
832
833 // Check if this module has the cfguard flag and read its value.
834 CFGuardModuleFlag = M->getControlFlowGuardMode();
835
836 // Warn if the module flag requests an unsupported CFGuard mechanism.
837 if (CFGuardModuleFlag == ControlFlowGuardMode::Enabled) {
838 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(
839 MD: Mod.getModuleFlag(Key: "cfguard-mechanism"))) {
840 auto MechanismOverride =
841 static_cast<ControlFlowGuardMechanism>(CI->getZExtValue());
842 if (MechanismOverride != ControlFlowGuardMechanism::Automatic &&
843 MechanismOverride != ControlFlowGuardMechanism::Check)
844 Mod.getContext().diagnose(
845 DI: DiagnosticInfoGeneric("only the Check Control Flow Guard mechanism "
846 "is supported for Arm64EC",
847 DS_Warning));
848 }
849 }
850
851 PtrTy = PointerType::getUnqual(C&: M->getContext());
852 I64Ty = Type::getInt64Ty(C&: M->getContext());
853 VoidTy = Type::getVoidTy(C&: M->getContext());
854
855 GuardFnType =
856 FunctionType::get(Result: StructType::get(elt1: PtrTy, elts: PtrTy), Params: {PtrTy, PtrTy}, isVarArg: false);
857 DispatchFnType = FunctionType::get(Result: PtrTy, Params: {PtrTy, PtrTy, PtrTy}, isVarArg: false);
858 GuardFnCFGlobal = M->getOrInsertGlobal(Name: "__os_arm64x_check_icall_cfg", Ty: PtrTy);
859 GuardFnGlobal = M->getOrInsertGlobal(Name: "__os_arm64x_check_icall", Ty: PtrTy);
860 DispatchFnGlobal = M->getOrInsertGlobal(Name: "__os_arm64x_dispatch_call", Ty: PtrTy);
861
862 // Mangle names of function aliases and add the alias name to
863 // arm64ec_unmangled_name metadata to ensure a weak anti-dependency symbol is
864 // emitted for the alias as well. Do this early, before handling
865 // hybrid_patchable functions, to avoid mangling their aliases.
866 for (GlobalAlias &A : Mod.aliases()) {
867 auto F = dyn_cast_or_null<Function>(Val: A.getAliaseeObject());
868 if (!F)
869 continue;
870 if (std::optional<std::string> MangledName =
871 getArm64ECMangledFunctionName(GV&: A)) {
872 F->addMetadata(Kind: "arm64ec_unmangled_name",
873 MD&: *MDNode::get(Context&: M->getContext(),
874 MDs: MDString::get(Context&: M->getContext(), Str: A.getName())));
875 A.setName(MangledName.value());
876 }
877 }
878
879 DenseMap<GlobalAlias *, GlobalAlias *> FnsMap;
880 SetVector<GlobalAlias *> PatchableFns;
881
882 for (Function &F : Mod) {
883 if (F.hasPersonalityFn()) {
884 GlobalValue *PersFn =
885 cast<GlobalValue>(Val: F.getPersonalityFn()->stripPointerCasts());
886 if (PersFn->getValueType() && PersFn->getValueType()->isFunctionTy()) {
887 if (std::optional<std::string> MangledName =
888 getArm64ECMangledFunctionName(GV&: *PersFn)) {
889 PersFn->setName(MangledName.value());
890 }
891 }
892 }
893
894 if (!F.hasFnAttribute(Kind: Attribute::HybridPatchable) ||
895 F.isDeclarationForLinker() || F.hasLocalLinkage() ||
896 F.getName().ends_with(Suffix: HybridPatchableTargetSuffix))
897 continue;
898
899 // Rename hybrid patchable functions and change callers to use a global
900 // alias instead.
901 if (std::optional<std::string> MangledName =
902 getArm64ECMangledFunctionName(GV&: F)) {
903 std::string OrigName(F.getName());
904 F.setName(MangledName.value() + HybridPatchableTargetSuffix);
905
906 // The unmangled symbol is a weak alias to an undefined symbol with the
907 // "EXP+" prefix. This undefined symbol is resolved by the linker by
908 // creating an x86 thunk that jumps back to the actual EC target. Since we
909 // can't represent that in IR, we create an alias to the target instead.
910 // The "EXP+" symbol is set as metadata, which is then used by
911 // emitGlobalAlias to emit the right alias.
912 auto *A =
913 GlobalAlias::create(Linkage: GlobalValue::LinkOnceODRLinkage, Name: OrigName, Aliasee: &F);
914 auto *AM = GlobalAlias::create(Linkage: GlobalValue::LinkOnceODRLinkage,
915 Name: MangledName.value(), Aliasee: &F);
916 F.replaceUsesWithIf(New: AM,
917 ShouldReplace: [](Use &U) { return isa<GlobalAlias>(Val: U.getUser()); });
918 F.replaceAllUsesWith(V: A);
919 F.setMetadata(Kind: "arm64ec_exp_name",
920 Node: MDNode::get(Context&: M->getContext(),
921 MDs: MDString::get(Context&: M->getContext(),
922 Str: "EXP+" + MangledName.value())));
923 A->setAliasee(&F);
924 AM->setAliasee(&F);
925
926 if (F.hasDLLExportStorageClass()) {
927 A->setDLLStorageClass(GlobalValue::DLLExportStorageClass);
928 F.setDLLStorageClass(GlobalValue::DefaultStorageClass);
929 }
930
931 FnsMap[A] = AM;
932 PatchableFns.insert(X: A);
933 }
934 }
935
936 SetVector<GlobalValue *> DirectCalledFns;
937 for (Function &F : Mod)
938 if (!F.isDeclarationForLinker() &&
939 F.getCallingConv() != CallingConv::ARM64EC_Thunk_Native &&
940 F.getCallingConv() != CallingConv::ARM64EC_Thunk_X64)
941 processFunction(F, DirectCalledFns, FnsMap);
942
943 struct ThunkInfo {
944 Constant *Src;
945 Constant *Dst;
946 Arm64ECThunkType Kind;
947 };
948 SmallVector<ThunkInfo> ThunkMapping;
949 for (Function &F : Mod) {
950 if (!F.isDeclarationForLinker() &&
951 (!F.hasLocalLinkage() || F.hasAddressTaken()) &&
952 F.getCallingConv() != CallingConv::ARM64EC_Thunk_Native &&
953 F.getCallingConv() != CallingConv::ARM64EC_Thunk_X64) {
954 if (!F.hasComdat())
955 F.setComdat(Mod.getOrInsertComdat(Name: F.getName()));
956 ThunkMapping.push_back(
957 Elt: {.Src: &F, .Dst: buildEntryThunk(F: &F), .Kind: Arm64ECThunkType::Entry});
958 }
959 }
960 for (GlobalValue *O : DirectCalledFns) {
961 auto GA = dyn_cast<GlobalAlias>(Val: O);
962 auto F = dyn_cast<Function>(Val: GA ? GA->getAliasee() : O);
963 ThunkMapping.push_back(
964 Elt: {.Src: O, .Dst: buildExitThunk(FT: F->getFunctionType(), Attrs: F->getAttributes()),
965 .Kind: Arm64ECThunkType::Exit});
966 if (!GA && !F->hasDLLImportStorageClass())
967 ThunkMapping.push_back(
968 Elt: {.Src: buildGuestExitThunk(F), .Dst: F, .Kind: Arm64ECThunkType::GuestExit});
969 }
970 for (GlobalAlias *A : PatchableFns) {
971 Function *Thunk = buildPatchableThunk(UnmangledAlias: A, MangledAlias: FnsMap[A]);
972 ThunkMapping.push_back(Elt: {.Src: Thunk, .Dst: A, .Kind: Arm64ECThunkType::GuestExit});
973 }
974
975 if (!ThunkMapping.empty()) {
976 SmallVector<Constant *> ThunkMappingArrayElems;
977 for (ThunkInfo &Thunk : ThunkMapping) {
978 ThunkMappingArrayElems.push_back(Elt: ConstantStruct::getAnon(
979 V: {Thunk.Src, Thunk.Dst,
980 ConstantInt::get(Context&: M->getContext(), V: APInt(32, uint8_t(Thunk.Kind)))}));
981 }
982 Constant *ThunkMappingArray = ConstantArray::get(
983 T: llvm::ArrayType::get(ElementType: ThunkMappingArrayElems[0]->getType(),
984 NumElements: ThunkMappingArrayElems.size()),
985 V: ThunkMappingArrayElems);
986 new GlobalVariable(Mod, ThunkMappingArray->getType(), /*isConstant*/ false,
987 GlobalValue::ExternalLinkage, ThunkMappingArray,
988 "llvm.arm64ec.symbolmap");
989 }
990
991 return true;
992}
993
994bool AArch64Arm64ECCallLowering::processFunction(
995 Function &F, SetVector<GlobalValue *> &DirectCalledFns,
996 DenseMap<GlobalAlias *, GlobalAlias *> &FnsMap) {
997 SmallVector<CallBase *, 8> IndirectCalls;
998
999 // For ARM64EC targets, a function definition's name is mangled differently
1000 // from the normal symbol. We currently have no representation of this sort
1001 // of symbol in IR, so we change the name to the mangled name, then store
1002 // the unmangled name as metadata. Later passes that need the unmangled
1003 // name (emitting the definition) can grab it from the metadata.
1004 //
1005 // FIXME: Handle functions with weak linkage?
1006 if (!F.hasLocalLinkage() || F.hasAddressTaken()) {
1007 if (std::optional<std::string> MangledName =
1008 getArm64ECMangledFunctionName(GV&: F)) {
1009 F.addMetadata(Kind: "arm64ec_unmangled_name",
1010 MD&: *MDNode::get(Context&: M->getContext(),
1011 MDs: MDString::get(Context&: M->getContext(), Str: F.getName())));
1012 if (F.hasComdat() && F.getComdat()->getName() == F.getName()) {
1013 Comdat *MangledComdat = M->getOrInsertComdat(Name: MangledName.value());
1014 SmallVector<GlobalObject *> ComdatUsers =
1015 to_vector(Range: F.getComdat()->getUsers());
1016 for (GlobalObject *User : ComdatUsers)
1017 User->setComdat(MangledComdat);
1018 }
1019 F.setName(MangledName.value());
1020 }
1021 }
1022
1023 // Iterate over the instructions to find all indirect call/invoke/callbr
1024 // instructions. Make a separate list of pointers to indirect
1025 // call/invoke/callbr instructions because the original instructions will be
1026 // deleted as the checks are added.
1027 for (BasicBlock &BB : F) {
1028 for (Instruction &I : BB) {
1029 auto *CB = dyn_cast<CallBase>(Val: &I);
1030 if (!CB || CB->getCallingConv() == CallingConv::ARM64EC_Thunk_X64 ||
1031 CB->isInlineAsm())
1032 continue;
1033
1034 // We need to instrument any call that isn't directly calling an
1035 // ARM64 function.
1036 //
1037 // FIXME: getCalledFunction() fails if there's a bitcast (e.g.
1038 // unprototyped functions in C)
1039 if (Function *F = CB->getCalledFunction()) {
1040 if (!LowerDirectToIndirect || F->hasLocalLinkage() ||
1041 F->isIntrinsic() || !F->isDeclarationForLinker())
1042 continue;
1043
1044 DirectCalledFns.insert(X: F);
1045 continue;
1046 }
1047
1048 // Use mangled global alias for direct calls to patchable functions.
1049 if (GlobalAlias *A = dyn_cast<GlobalAlias>(Val: CB->getCalledOperand())) {
1050 auto I = FnsMap.find(Val: A);
1051 if (I != FnsMap.end()) {
1052 CB->setCalledOperand(I->second);
1053 DirectCalledFns.insert(X: I->first);
1054 continue;
1055 }
1056 }
1057
1058 IndirectCalls.push_back(Elt: CB);
1059 ++Arm64ECCallsLowered;
1060 }
1061 }
1062
1063 if (IndirectCalls.empty())
1064 return false;
1065
1066 for (CallBase *CB : IndirectCalls)
1067 lowerCall(CB);
1068
1069 return true;
1070}
1071
1072char AArch64Arm64ECCallLowering::ID = 0;
1073INITIALIZE_PASS(AArch64Arm64ECCallLowering, "Arm64ECCallLowering",
1074 "AArch64Arm64ECCallLowering", false, false)
1075
1076ModulePass *llvm::createAArch64Arm64ECCallLoweringPass() {
1077 return new AArch64Arm64ECCallLowering;
1078}
1079