1//===--------- SMEABI - SME ABI-------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements parts of the the SME ABI, such as:
10// * Using the lazy-save mechanism before enabling the use of ZA.
11// * Setting up the lazy-save mechanism around invokes.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AArch64.h"
16#include "Utils/AArch64BaseInfo.h"
17#include "Utils/AArch64SMEAttributes.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/IR/Constants.h"
20#include "llvm/IR/IRBuilder.h"
21#include "llvm/IR/Instructions.h"
22#include "llvm/IR/IntrinsicInst.h"
23#include "llvm/IR/IntrinsicsAArch64.h"
24#include "llvm/IR/LLVMContext.h"
25#include "llvm/IR/Module.h"
26#include "llvm/InitializePasses.h"
27#include "llvm/Support/Debug.h"
28#include "llvm/Transforms/Utils/Cloning.h"
29
30using namespace llvm;
31
32#define DEBUG_TYPE "aarch64-sme-abi"
33
34namespace {
35struct SMEABI : public FunctionPass {
36 static char ID; // Pass identification, replacement for typeid
37 SMEABI() : FunctionPass(ID) {
38 initializeSMEABIPass(*PassRegistry::getPassRegistry());
39 }
40
41 bool runOnFunction(Function &F) override;
42
43private:
44 bool updateNewStateFunctions(Module *M, Function *F, IRBuilder<> &Builder,
45 SMEAttrs FnAttrs);
46};
47} // end anonymous namespace
48
49char SMEABI::ID = 0;
50static const char *name = "SME ABI Pass";
51INITIALIZE_PASS_BEGIN(SMEABI, DEBUG_TYPE, name, false, false)
52INITIALIZE_PASS_END(SMEABI, DEBUG_TYPE, name, false, false)
53
54FunctionPass *llvm::createSMEABIPass() { return new SMEABI(); }
55
56//===----------------------------------------------------------------------===//
57// Utility functions
58//===----------------------------------------------------------------------===//
59
60// Utility function to emit a call to __arm_tpidr2_save and clear TPIDR2_EL0.
61void emitTPIDR2Save(Module *M, IRBuilder<> &Builder) {
62 auto *TPIDR2SaveTy =
63 FunctionType::get(Result: Builder.getVoidTy(), Params: {}, /*IsVarArgs=*/isVarArg: false);
64 auto Attrs = AttributeList().addFnAttribute(C&: M->getContext(),
65 Kind: "aarch64_pstate_sm_compatible");
66 FunctionCallee Callee =
67 M->getOrInsertFunction(Name: "__arm_tpidr2_save", T: TPIDR2SaveTy, AttributeList: Attrs);
68 CallInst *Call = Builder.CreateCall(Callee);
69 Call->setCallingConv(
70 CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0);
71
72 // A save to TPIDR2 should be followed by clearing TPIDR2_EL0.
73 Function *WriteIntr =
74 Intrinsic::getDeclaration(M, id: Intrinsic::aarch64_sme_set_tpidr2);
75 Builder.CreateCall(FTy: WriteIntr->getFunctionType(), Callee: WriteIntr,
76 Args: Builder.getInt64(C: 0));
77}
78
79/// This function generates code at the beginning and end of a function marked
80/// with either `aarch64_new_za` or `aarch64_new_zt0`.
81/// At the beginning of the function, the following code is generated:
82/// - Commit lazy-save if active [Private-ZA Interface*]
83/// - Enable PSTATE.ZA [Private-ZA Interface]
84/// - Zero ZA [Has New ZA State]
85/// - Zero ZT0 [Has New ZT0 State]
86///
87/// * A function with new ZT0 state will not change ZA, so committing the
88/// lazy-save is not strictly necessary. However, the lazy-save mechanism
89/// may be active on entry to the function, with PSTATE.ZA set to 1. If
90/// the new ZT0 function calls a function that does not share ZT0, we will
91/// need to conditionally SMSTOP ZA before the call, setting PSTATE.ZA to 0.
92/// For this reason, it's easier to always commit the lazy-save at the
93/// beginning of the function regardless of whether it has ZA state.
94///
95/// At the end of the function, PSTATE.ZA is disabled if the function has a
96/// Private-ZA Interface. A function is considered to have a Private-ZA
97/// interface if it does not share ZA or ZT0.
98///
99bool SMEABI::updateNewStateFunctions(Module *M, Function *F,
100 IRBuilder<> &Builder, SMEAttrs FnAttrs) {
101 LLVMContext &Context = F->getContext();
102 BasicBlock *OrigBB = &F->getEntryBlock();
103 Builder.SetInsertPoint(&OrigBB->front());
104
105 // Commit any active lazy-saves if this is a Private-ZA function. If the
106 // value read from TPIDR2_EL0 is not null on entry to the function then
107 // the lazy-saving scheme is active and we should call __arm_tpidr2_save
108 // to commit the lazy save.
109 if (FnAttrs.hasPrivateZAInterface()) {
110 // Create the new blocks for reading TPIDR2_EL0 & enabling ZA state.
111 auto *SaveBB = OrigBB->splitBasicBlock(I: OrigBB->begin(), BBName: "save.za", Before: true);
112 auto *PreludeBB = BasicBlock::Create(Context, Name: "prelude", Parent: F, InsertBefore: SaveBB);
113
114 // Read TPIDR2_EL0 in PreludeBB & branch to SaveBB if not 0.
115 Builder.SetInsertPoint(PreludeBB);
116 Function *TPIDR2Intr =
117 Intrinsic::getDeclaration(M, id: Intrinsic::aarch64_sme_get_tpidr2);
118 auto *TPIDR2 = Builder.CreateCall(FTy: TPIDR2Intr->getFunctionType(), Callee: TPIDR2Intr,
119 Args: {}, Name: "tpidr2");
120 auto *Cmp = Builder.CreateCmp(Pred: ICmpInst::ICMP_NE, LHS: TPIDR2,
121 RHS: Builder.getInt64(C: 0), Name: "cmp");
122 Builder.CreateCondBr(Cond: Cmp, True: SaveBB, False: OrigBB);
123
124 // Create a call __arm_tpidr2_save, which commits the lazy save.
125 Builder.SetInsertPoint(&SaveBB->back());
126 emitTPIDR2Save(M, Builder);
127
128 // Enable pstate.za at the start of the function.
129 Builder.SetInsertPoint(&OrigBB->front());
130 Function *EnableZAIntr =
131 Intrinsic::getDeclaration(M, id: Intrinsic::aarch64_sme_za_enable);
132 Builder.CreateCall(FTy: EnableZAIntr->getFunctionType(), Callee: EnableZAIntr);
133 }
134
135 if (FnAttrs.isNewZA()) {
136 Function *ZeroIntr =
137 Intrinsic::getDeclaration(M, id: Intrinsic::aarch64_sme_zero);
138 Builder.CreateCall(FTy: ZeroIntr->getFunctionType(), Callee: ZeroIntr,
139 Args: Builder.getInt32(C: 0xff));
140 }
141
142 if (FnAttrs.isNewZT0()) {
143 Function *ClearZT0Intr =
144 Intrinsic::getDeclaration(M, id: Intrinsic::aarch64_sme_zero_zt);
145 Builder.CreateCall(FTy: ClearZT0Intr->getFunctionType(), Callee: ClearZT0Intr,
146 Args: {Builder.getInt32(C: 0)});
147 }
148
149 if (FnAttrs.hasPrivateZAInterface()) {
150 // Before returning, disable pstate.za
151 for (BasicBlock &BB : *F) {
152 Instruction *T = BB.getTerminator();
153 if (!T || !isa<ReturnInst>(Val: T))
154 continue;
155 Builder.SetInsertPoint(T);
156 Function *DisableZAIntr =
157 Intrinsic::getDeclaration(M, id: Intrinsic::aarch64_sme_za_disable);
158 Builder.CreateCall(FTy: DisableZAIntr->getFunctionType(), Callee: DisableZAIntr);
159 }
160 }
161
162 F->addFnAttr(Kind: "aarch64_expanded_pstate_za");
163 return true;
164}
165
166bool SMEABI::runOnFunction(Function &F) {
167 Module *M = F.getParent();
168 LLVMContext &Context = F.getContext();
169 IRBuilder<> Builder(Context);
170
171 if (F.isDeclaration() || F.hasFnAttribute(Kind: "aarch64_expanded_pstate_za"))
172 return false;
173
174 bool Changed = false;
175 SMEAttrs FnAttrs(F);
176 if (FnAttrs.isNewZA() || FnAttrs.isNewZT0())
177 Changed |= updateNewStateFunctions(M, F: &F, Builder, FnAttrs);
178
179 return Changed;
180}
181