1//===- MachineSMEABIPass.cpp ----------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements the SME ABI requirements for ZA state. This includes
10// implementing the lazy (and agnostic) ZA state save schemes around calls.
11//
12//===----------------------------------------------------------------------===//
13//
14// This pass works by collecting instructions that require ZA to be in a
15// specific state (e.g., "ACTIVE" or "SAVED") and inserting the necessary state
16// transitions to ensure ZA is in the required state before instructions. State
17// transitions represent actions such as setting up or restoring a lazy save.
18// Certain points within a function may also have predefined states independent
19// of any instructions, for example, a "shared_za" function is always entered
20// and exited in the "ACTIVE" state.
21//
22// To handle ZA state across control flow, we make use of edge bundling. This
23// assigns each block an "incoming" and "outgoing" edge bundle (representing
24// incoming and outgoing edges). Initially, these are unique to each block;
25// then, in the process of forming bundles, the outgoing bundle of a block is
26// joined with the incoming bundle of all successors. The result is that each
27// bundle can be assigned a single ZA state, which ensures the state required by
28// all a blocks' successors is the same, and that each basic block will always
29// be entered with the same ZA state. This eliminates the need for splitting
30// edges to insert state transitions or "phi" nodes for ZA states.
31//
32// See below for a simple example of edge bundling.
33//
34// The following shows a conditionally executed basic block (BB1):
35//
36// if (cond)
37// BB1
38// BB2
39//
40// Initial Bundles Joined Bundles
41//
42// ┌──0──┐ ┌──0──┐
43// │ BB0 │ │ BB0 │
44// └──1──┘ └──1──┘
45// ├───────┐ ├───────┐
46// ▼ │ ▼ │
47// ┌──2──┐ │ ─────► ┌──1──┐ │
48// │ BB1 │ ▼ │ BB1 │ ▼
49// └──3──┘ ┌──4──┐ └──1──┘ ┌──1──┐
50// └───►4 BB2 │ └───►1 BB2 │
51// └──5──┘ └──2──┘
52//
53// On the left are the initial per-block bundles, and on the right are the
54// joined bundles (which are the result of the EdgeBundles analysis).
55
56#include "AArch64InstrInfo.h"
57#include "AArch64MachineFunctionInfo.h"
58#include "AArch64Subtarget.h"
59#include "MCTargetDesc/AArch64AddressingModes.h"
60#include "llvm/ADT/BitmaskEnum.h"
61#include "llvm/ADT/SmallVector.h"
62#include "llvm/CodeGen/EdgeBundles.h"
63#include "llvm/CodeGen/LivePhysRegs.h"
64#include "llvm/CodeGen/MachineBasicBlock.h"
65#include "llvm/CodeGen/MachineFunctionPass.h"
66#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
67#include "llvm/CodeGen/MachineRegisterInfo.h"
68#include "llvm/CodeGen/TargetRegisterInfo.h"
69
70using namespace llvm;
71
72#define DEBUG_TYPE "aarch64-machine-sme-abi"
73
74namespace {
75
76// Note: For agnostic ZA, we assume the function is always entered/exited in the
77// "ACTIVE" state -- this _may_ not be the case (since OFF is also a
78// possibility, but for the purpose of placing ZA saves/restores, that does not
79// matter).
80enum ZAState : uint8_t {
81 // Any/unknown state (not valid)
82 ANY = 0,
83
84 // ZA is in use and active (i.e. within the accumulator)
85 ACTIVE,
86
87 // ZA is active, but ZT0 has been saved.
88 // This handles the edge case of sharedZA && !sharesZT0.
89 ACTIVE_ZT0_SAVED,
90
91 // A ZA save has been set up or committed (i.e. ZA is dormant or off)
92 // If the function uses ZT0 it must also be saved.
93 LOCAL_SAVED,
94
95 // ZA has been committed to the lazy save buffer of the current function.
96 // If the function uses ZT0 it must also be saved.
97 // ZA is off.
98 LOCAL_COMMITTED,
99
100 // The ZA/ZT0 state on entry to the function.
101 ENTRY,
102
103 // ZA is off.
104 OFF,
105
106 // The number of ZA states (not a valid state)
107 NUM_ZA_STATE
108};
109
110/// A bitmask enum to record live physical registers that the "emit*" routines
111/// may need to preserve. Note: This only tracks registers we may clobber.
112enum LiveRegs : uint8_t {
113 None = 0,
114 NZCV = 1 << 0,
115 W0 = 1 << 1,
116 W0_HI = 1 << 2,
117 X0 = W0 | W0_HI,
118 LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ W0_HI)
119};
120
121/// Holds the virtual registers live physical registers have been saved to.
122struct PhysRegSave {
123 LiveRegs PhysLiveRegs;
124 Register StatusFlags = Register();
125 Register X0Save = Register();
126};
127
128/// Contains the needed ZA state (and live registers) at an instruction. That is
129/// the state ZA must be in _before_ "InsertPt".
130struct InstInfo {
131 ZAState NeededState{ZAState::ANY};
132 MachineBasicBlock::iterator InsertPt;
133 LiveRegs PhysLiveRegs = LiveRegs::None;
134};
135
136/// Contains the needed ZA state for each instruction in a block. Instructions
137/// that do not require a ZA state are not recorded.
138struct BlockInfo {
139 SmallVector<InstInfo> Insts;
140 ZAState FixedEntryState{ZAState::ANY};
141 ZAState DesiredIncomingState{ZAState::ANY};
142 ZAState DesiredOutgoingState{ZAState::ANY};
143 LiveRegs PhysLiveRegsAtEntry = LiveRegs::None;
144 LiveRegs PhysLiveRegsAtExit = LiveRegs::None;
145};
146
147/// Contains the needed ZA state information for all blocks within a function.
148struct FunctionInfo {
149 SmallVector<BlockInfo> Blocks;
150 std::optional<MachineBasicBlock::iterator> AfterSMEProloguePt;
151 LiveRegs PhysLiveRegsAfterSMEPrologue = LiveRegs::None;
152};
153
154/// State/helpers that is only needed when emitting code to handle
155/// saving/restoring ZA.
156class EmitContext {
157public:
158 EmitContext() = default;
159
160 /// Get or create a TPIDR2 block in \p MF.
161 int getTPIDR2Block(MachineFunction &MF) {
162 if (TPIDR2BlockFI)
163 return *TPIDR2BlockFI;
164 MachineFrameInfo &MFI = MF.getFrameInfo();
165 TPIDR2BlockFI = MFI.CreateStackObject(Size: 16, Alignment: Align(16), isSpillSlot: false);
166 return *TPIDR2BlockFI;
167 }
168
169 /// Get or create agnostic ZA buffer pointer in \p MF.
170 Register getAgnosticZABufferPtr(MachineFunction &MF) {
171 if (AgnosticZABufferPtr.isValid())
172 return AgnosticZABufferPtr;
173 Register BufferPtr =
174 MF.getInfo<AArch64FunctionInfo>()->getEarlyAllocSMESaveBuffer();
175 AgnosticZABufferPtr =
176 BufferPtr.isValid()
177 ? BufferPtr
178 : MF.getRegInfo().createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
179 return AgnosticZABufferPtr;
180 }
181
182 int getZT0SaveSlot(MachineFunction &MF) {
183 if (ZT0SaveFI)
184 return *ZT0SaveFI;
185 MachineFrameInfo &MFI = MF.getFrameInfo();
186 ZT0SaveFI = MFI.CreateSpillStackObject(Size: 64, Alignment: Align(16));
187 return *ZT0SaveFI;
188 }
189
190 /// Returns true if the function must allocate a ZA save buffer on entry. This
191 /// will be the case if, at any point in the function, a ZA save was emitted.
192 bool needsSaveBuffer() const {
193 assert(!(TPIDR2BlockFI && AgnosticZABufferPtr) &&
194 "Cannot have both a TPIDR2 block and agnostic ZA buffer");
195 return TPIDR2BlockFI || AgnosticZABufferPtr.isValid();
196 }
197
198private:
199 std::optional<int> ZT0SaveFI;
200 std::optional<int> TPIDR2BlockFI;
201 Register AgnosticZABufferPtr = Register();
202};
203
204StringRef getZAStateString(ZAState State) {
205#define MAKE_CASE(V) \
206 case V: \
207 return #V;
208 switch (State) {
209 MAKE_CASE(ZAState::ANY)
210 MAKE_CASE(ZAState::ACTIVE)
211 MAKE_CASE(ZAState::ACTIVE_ZT0_SAVED)
212 MAKE_CASE(ZAState::LOCAL_SAVED)
213 MAKE_CASE(ZAState::LOCAL_COMMITTED)
214 MAKE_CASE(ZAState::ENTRY)
215 MAKE_CASE(ZAState::OFF)
216 default:
217 llvm_unreachable("Unexpected ZAState");
218 }
219#undef MAKE_CASE
220}
221
222static bool isZAorZTRegOp(const TargetRegisterInfo &TRI,
223 const MachineOperand &MO) {
224 if (!MO.isReg() || !MO.getReg().isPhysical())
225 return false;
226 return any_of(Range: TRI.subregs_inclusive(Reg: MO.getReg()), P: [](const MCPhysReg &SR) {
227 return AArch64::MPR128RegClass.contains(Reg: SR) ||
228 AArch64::ZTRRegClass.contains(Reg: SR);
229 });
230}
231
232/// Returns the required ZA state needed before \p MI and an iterator pointing
233/// to where any code required to change the ZA state should be inserted.
234static std::pair<ZAState, MachineBasicBlock::iterator>
235getInstNeededZAState(const TargetRegisterInfo &TRI, MachineInstr &MI,
236 SMEAttrs SMEFnAttrs) {
237 MachineBasicBlock::iterator InsertPt(MI);
238
239 // Note: InOutZAUsePseudo, RequiresZASavePseudo, and RequiresZT0SavePseudo are
240 // intended to mark the position immediately before a call. Due to
241 // SelectionDAG constraints, these markers occur after the ADJCALLSTACKDOWN,
242 // so we use std::prev(InsertPt) to get the position before the call.
243
244 if (MI.getOpcode() == AArch64::InOutZAUsePseudo)
245 return {ZAState::ACTIVE, std::prev(x: InsertPt)};
246
247 // Note: If we need to save both ZA and ZT0 we use RequiresZASavePseudo.
248 if (MI.getOpcode() == AArch64::RequiresZASavePseudo)
249 return {ZAState::LOCAL_SAVED, std::prev(x: InsertPt)};
250
251 // If we only need to save ZT0 there's two cases to consider:
252 // 1. The function has ZA state (that we don't need to save).
253 // - In this case we switch to the "ACTIVE_ZT0_SAVED" state.
254 // This only saves ZT0.
255 // 2. The function does not have ZA state
256 // - In this case we switch to "LOCAL_COMMITTED" state.
257 // This saves ZT0 and turns ZA off.
258 if (MI.getOpcode() == AArch64::RequiresZT0SavePseudo) {
259 return {SMEFnAttrs.hasZAState() ? ZAState::ACTIVE_ZT0_SAVED
260 : ZAState::LOCAL_COMMITTED,
261 std::prev(x: InsertPt)};
262 }
263
264 if (MI.isReturn()) {
265 bool ZAOffAtReturn = SMEFnAttrs.hasPrivateZAInterface();
266 return {ZAOffAtReturn ? ZAState::OFF : ZAState::ACTIVE, InsertPt};
267 }
268
269 for (auto &MO : MI.operands()) {
270 if (isZAorZTRegOp(TRI, MO))
271 return {ZAState::ACTIVE, InsertPt};
272 }
273
274 return {ZAState::ANY, InsertPt};
275}
276
277struct MachineSMEABI : public MachineFunctionPass {
278 inline static char ID = 0;
279
280 MachineSMEABI() : MachineFunctionPass(ID) {}
281
282 bool runOnMachineFunction(MachineFunction &MF) override;
283
284 StringRef getPassName() const override { return "Machine SME ABI pass"; }
285
286 void getAnalysisUsage(AnalysisUsage &AU) const override {
287 AU.setPreservesCFG();
288 AU.addRequired<EdgeBundlesWrapperLegacy>();
289 AU.addRequired<MachineOptimizationRemarkEmitterPass>();
290 AU.addRequired<LibcallLoweringInfoWrapper>();
291 AU.addPreservedID(ID&: MachineLoopInfoID);
292 AU.addPreservedID(ID&: MachineDominatorsID);
293 MachineFunctionPass::getAnalysisUsage(AU);
294 }
295
296 /// Collects the needed ZA state (and live registers) before each instruction
297 /// within the machine function.
298 FunctionInfo collectNeededZAStates(SMEAttrs SMEFnAttrs);
299
300 /// Assigns each edge bundle a ZA state based on the desired states of
301 /// incoming and outgoing blocks in the bundle.
302 SmallVector<ZAState> assignBundleZAStates(const EdgeBundles &Bundles,
303 const FunctionInfo &FnInfo);
304
305 /// Inserts code to handle changes between ZA states within the function.
306 /// E.g., ACTIVE -> LOCAL_SAVED will insert code required to save ZA.
307 void insertStateChanges(EmitContext &, const FunctionInfo &FnInfo,
308 const EdgeBundles &Bundles,
309 ArrayRef<ZAState> BundleStates);
310
311 void addSMELibCall(MachineInstrBuilder &MIB, RTLIB::Libcall LC,
312 CallingConv::ID ExpectedCC);
313
314 void emitZT0SaveRestore(EmitContext &, MachineBasicBlock &MBB,
315 MachineBasicBlock::iterator MBBI, bool IsSave);
316
317 // Emission routines for private and shared ZA functions (using lazy saves).
318 void emitSMEPrologue(MachineBasicBlock &MBB,
319 MachineBasicBlock::iterator MBBI);
320 void emitRestoreLazySave(EmitContext &, MachineBasicBlock &MBB,
321 MachineBasicBlock::iterator MBBI,
322 LiveRegs PhysLiveRegs);
323 void emitSetupLazySave(EmitContext &, MachineBasicBlock &MBB,
324 MachineBasicBlock::iterator MBBI);
325 void emitAllocateLazySaveBuffer(EmitContext &, MachineBasicBlock &MBB,
326 MachineBasicBlock::iterator MBBI);
327 void emitZAMode(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
328 bool ClearTPIDR2, bool On);
329
330 // Emission routines for agnostic ZA functions.
331 void emitSetupFullZASave(MachineBasicBlock &MBB,
332 MachineBasicBlock::iterator MBBI,
333 LiveRegs PhysLiveRegs);
334 // Emit a "full" ZA save or restore. It is "full" in the sense that this
335 // function will emit a call to __arm_sme_save or __arm_sme_restore, which
336 // handles saving and restoring both ZA and ZT0.
337 void emitFullZASaveRestore(EmitContext &, MachineBasicBlock &MBB,
338 MachineBasicBlock::iterator MBBI,
339 LiveRegs PhysLiveRegs, bool IsSave);
340 void emitAllocateFullZASaveBuffer(EmitContext &, MachineBasicBlock &MBB,
341 MachineBasicBlock::iterator MBBI,
342 LiveRegs PhysLiveRegs);
343
344 /// Attempts to find an insertion point before \p Inst where the status flags
345 /// are not live. If \p Inst is `Block.Insts.end()` a point before the end of
346 /// the block is found.
347 std::pair<MachineBasicBlock::iterator, LiveRegs>
348 findStateChangeInsertionPoint(MachineBasicBlock &MBB, const BlockInfo &Block,
349 SmallVectorImpl<InstInfo>::const_iterator Inst);
350 void emitStateChange(EmitContext &, MachineBasicBlock &MBB,
351 MachineBasicBlock::iterator MBBI, ZAState From,
352 ZAState To, LiveRegs PhysLiveRegs);
353
354 // Helpers for switching between lazy/full ZA save/restore routines.
355 void emitZASave(EmitContext &Context, MachineBasicBlock &MBB,
356 MachineBasicBlock::iterator MBBI, LiveRegs PhysLiveRegs) {
357 if (AFI->getSMEFnAttrs().hasAgnosticZAInterface())
358 return emitFullZASaveRestore(Context, MBB, MBBI, PhysLiveRegs,
359 /*IsSave=*/true);
360 return emitSetupLazySave(Context, MBB, MBBI);
361 }
362 void emitZARestore(EmitContext &Context, MachineBasicBlock &MBB,
363 MachineBasicBlock::iterator MBBI, LiveRegs PhysLiveRegs) {
364 if (AFI->getSMEFnAttrs().hasAgnosticZAInterface())
365 return emitFullZASaveRestore(Context, MBB, MBBI, PhysLiveRegs,
366 /*IsSave=*/false);
367 return emitRestoreLazySave(Context, MBB, MBBI, PhysLiveRegs);
368 }
369 void emitAllocateZASaveBuffer(EmitContext &Context, MachineBasicBlock &MBB,
370 MachineBasicBlock::iterator MBBI,
371 LiveRegs PhysLiveRegs) {
372 if (AFI->getSMEFnAttrs().hasAgnosticZAInterface())
373 return emitAllocateFullZASaveBuffer(Context, MBB, MBBI, PhysLiveRegs);
374 return emitAllocateLazySaveBuffer(Context, MBB, MBBI);
375 }
376
377 /// Collects the reachable calls from \p MBBI marked with \p Marker. This is
378 /// intended to be used to emit lazy save remarks. Note: This stops at the
379 /// first marked call along any path.
380 void collectReachableMarkedCalls(const MachineBasicBlock &MBB,
381 MachineBasicBlock::const_iterator MBBI,
382 SmallVectorImpl<const MachineInstr *> &Calls,
383 unsigned Marker) const;
384
385 void emitCallSaveRemarks(const MachineBasicBlock &MBB,
386 MachineBasicBlock::const_iterator MBBI, DebugLoc DL,
387 unsigned Marker, StringRef RemarkName,
388 StringRef SaveName) const;
389
390 void emitError(const Twine &Message) {
391 LLVMContext &Context = MF->getFunction().getContext();
392 Context.emitError(ErrorStr: MF->getName() + ": " + Message);
393 }
394
395 /// Save live physical registers to virtual registers.
396 PhysRegSave createPhysRegSave(LiveRegs PhysLiveRegs, MachineBasicBlock &MBB,
397 MachineBasicBlock::iterator MBBI, DebugLoc DL);
398 /// Restore physical registers from a save of their previous values.
399 void restorePhyRegSave(const PhysRegSave &RegSave, MachineBasicBlock &MBB,
400 MachineBasicBlock::iterator MBBI, DebugLoc DL);
401
402private:
403 MachineFunction *MF = nullptr;
404 const AArch64Subtarget *Subtarget = nullptr;
405 const AArch64RegisterInfo *TRI = nullptr;
406 const AArch64FunctionInfo *AFI = nullptr;
407 const AArch64InstrInfo *TII = nullptr;
408 const LibcallLoweringInfo *LLI = nullptr;
409
410 MachineOptimizationRemarkEmitter *ORE = nullptr;
411 MachineRegisterInfo *MRI = nullptr;
412};
413
414static LiveRegs getPhysLiveRegs(LiveRegUnits const &LiveUnits) {
415 LiveRegs PhysLiveRegs = LiveRegs::None;
416 if (!LiveUnits.available(Reg: AArch64::NZCV))
417 PhysLiveRegs |= LiveRegs::NZCV;
418 // We have to track W0 and X0 separately as otherwise things can get
419 // confused if we attempt to preserve X0 but only W0 was defined.
420 if (!LiveUnits.available(Reg: AArch64::W0))
421 PhysLiveRegs |= LiveRegs::W0;
422 if (!LiveUnits.available(Reg: AArch64::W0_HI))
423 PhysLiveRegs |= LiveRegs::W0_HI;
424 return PhysLiveRegs;
425}
426
427static void setPhysLiveRegs(LiveRegUnits &LiveUnits, LiveRegs PhysLiveRegs) {
428 if (PhysLiveRegs & LiveRegs::NZCV)
429 LiveUnits.addReg(Reg: AArch64::NZCV);
430 if (PhysLiveRegs & LiveRegs::W0)
431 LiveUnits.addReg(Reg: AArch64::W0);
432 if (PhysLiveRegs & LiveRegs::W0_HI)
433 LiveUnits.addReg(Reg: AArch64::W0_HI);
434}
435
436[[maybe_unused]] bool isCallStartOpcode(unsigned Opc) {
437 switch (Opc) {
438 case AArch64::BLR:
439 case AArch64::BLRA:
440 case AArch64::TLSDESC_CALLSEQ:
441 case AArch64::TLSDESC_AUTH_CALLSEQ:
442 case AArch64::ADJCALLSTACKDOWN:
443 return true;
444 default:
445 return false;
446 }
447}
448
449FunctionInfo MachineSMEABI::collectNeededZAStates(SMEAttrs SMEFnAttrs) {
450 assert((SMEFnAttrs.hasAgnosticZAInterface() || SMEFnAttrs.hasZT0State() ||
451 SMEFnAttrs.hasZAState()) &&
452 "Expected function to have ZA/ZT0 state!");
453
454 SmallVector<BlockInfo> Blocks(MF->getNumBlockIDs());
455 LiveRegs PhysLiveRegsAfterSMEPrologue = LiveRegs::None;
456 std::optional<MachineBasicBlock::iterator> AfterSMEProloguePt;
457
458 for (MachineBasicBlock &MBB : *MF) {
459 BlockInfo &Block = Blocks[MBB.getNumber()];
460
461 if (MBB.isEntryBlock()) {
462 // Entry block:
463 Block.FixedEntryState = ZAState::ENTRY;
464 } else if (MBB.isEHPad()) {
465 // EH entry block:
466 Block.FixedEntryState = ZAState::LOCAL_COMMITTED;
467 }
468
469 LiveRegUnits LiveUnits(*TRI);
470 LiveUnits.addLiveOuts(MBB);
471
472 Block.PhysLiveRegsAtExit = getPhysLiveRegs(LiveUnits);
473 auto FirstTerminatorInsertPt = MBB.getFirstTerminator();
474 auto FirstNonPhiInsertPt = MBB.getFirstNonPHI();
475 for (MachineInstr &MI : reverse(C&: MBB)) {
476 if (MI.isDebugInstr())
477 continue;
478
479 MachineBasicBlock::iterator MBBI(MI);
480 LiveUnits.stepBackward(MI);
481 LiveRegs PhysLiveRegs = getPhysLiveRegs(LiveUnits);
482 // The SMEStateAllocPseudo marker is added to a function if the save
483 // buffer was allocated in SelectionDAG. It marks the end of the
484 // allocation -- which is a safe point for this pass to insert any TPIDR2
485 // block setup.
486 if (MI.getOpcode() == AArch64::SMEStateAllocPseudo) {
487 AfterSMEProloguePt = MBBI;
488 PhysLiveRegsAfterSMEPrologue = PhysLiveRegs;
489 }
490 // Note: We treat Agnostic ZA as inout_za with an alternate save/restore.
491 auto [NeededState, InsertPt] = getInstNeededZAState(TRI: *TRI, MI, SMEFnAttrs);
492 assert((InsertPt == MBBI || isCallStartOpcode(InsertPt->getOpcode())) &&
493 "Unexpected state change insertion point!");
494 if (MBBI == FirstTerminatorInsertPt)
495 Block.PhysLiveRegsAtExit = PhysLiveRegs;
496 if (MBBI == FirstNonPhiInsertPt)
497 Block.PhysLiveRegsAtEntry = PhysLiveRegs;
498 if (NeededState != ZAState::ANY)
499 Block.Insts.push_back(Elt: {.NeededState: NeededState, .InsertPt: InsertPt, .PhysLiveRegs: PhysLiveRegs});
500 }
501
502 // Reverse vector (as we had to iterate backwards for liveness).
503 std::reverse(first: Block.Insts.begin(), last: Block.Insts.end());
504
505 // Record the desired states on entry/exit of this block. These are the
506 // states that would not incur a state transition.
507 if (!Block.Insts.empty()) {
508 Block.DesiredIncomingState = Block.Insts.front().NeededState;
509 Block.DesiredOutgoingState = Block.Insts.back().NeededState;
510 }
511 }
512
513 return FunctionInfo{.Blocks: std::move(Blocks), .AfterSMEProloguePt: AfterSMEProloguePt,
514 .PhysLiveRegsAfterSMEPrologue: PhysLiveRegsAfterSMEPrologue};
515}
516
517/// Assigns each edge bundle a ZA state based on the desired states of incoming
518/// and outgoing blocks in the bundle.
519SmallVector<ZAState>
520MachineSMEABI::assignBundleZAStates(const EdgeBundles &Bundles,
521 const FunctionInfo &FnInfo) {
522 SmallVector<ZAState> BundleStates(Bundles.getNumBundles());
523 for (unsigned I = 0, E = Bundles.getNumBundles(); I != E; ++I) {
524 std::optional<ZAState> BundleState;
525 for (unsigned BlockID : Bundles.getBlocks(Bundle: I)) {
526 const BlockInfo &Block = FnInfo.Blocks[BlockID];
527 // Check if the block is an incoming block in the bundle. Note: We skip
528 // Block.FixedEntryState != ANY to ignore EH pads (which are only
529 // reachable via exceptions).
530 if (Block.FixedEntryState != ZAState::ANY ||
531 Bundles.getBundle(N: BlockID, /*Out=*/false) != I)
532 continue;
533
534 // Pick a state that matches all incoming blocks. Fall back to "ACTIVE" if
535 // any incoming state doesn't match. This will hoist the state from
536 // incoming blocks to outgoing blocks.
537 if (!BundleState)
538 BundleState = Block.DesiredIncomingState;
539 else if (BundleState != Block.DesiredIncomingState)
540 BundleState = ZAState::ACTIVE;
541 }
542
543 if (!BundleState || BundleState == ZAState::ANY)
544 BundleState = ZAState::ACTIVE;
545
546 BundleStates[I] = *BundleState;
547 }
548
549 return BundleStates;
550}
551
552std::pair<MachineBasicBlock::iterator, LiveRegs>
553MachineSMEABI::findStateChangeInsertionPoint(
554 MachineBasicBlock &MBB, const BlockInfo &Block,
555 SmallVectorImpl<InstInfo>::const_iterator Inst) {
556 LiveRegs PhysLiveRegs;
557 MachineBasicBlock::iterator InsertPt;
558 if (Inst != Block.Insts.end()) {
559 InsertPt = Inst->InsertPt;
560 PhysLiveRegs = Inst->PhysLiveRegs;
561 } else {
562 InsertPt = MBB.getFirstTerminator();
563 PhysLiveRegs = Block.PhysLiveRegsAtExit;
564 }
565
566 if (PhysLiveRegs == LiveRegs::None)
567 return {InsertPt, PhysLiveRegs}; // Nothing to do (no live regs).
568
569 // Find the previous state change. We can not move before this point.
570 MachineBasicBlock::iterator PrevStateChangeI;
571 if (Inst == Block.Insts.begin()) {
572 PrevStateChangeI = MBB.begin();
573 } else {
574 // Note: `std::prev(Inst)` is the previous InstInfo. We only create an
575 // InstInfo object for instructions that require a specific ZA state, so the
576 // InstInfo is the site of the previous state change in the block (which can
577 // be several MIs earlier).
578 PrevStateChangeI = std::prev(x: Inst)->InsertPt;
579 }
580
581 // Note: LiveUnits will only accurately track X0 and NZCV.
582 LiveRegUnits LiveUnits(*TRI);
583 setPhysLiveRegs(LiveUnits, PhysLiveRegs);
584 auto BestCandidate = std::make_pair(x&: InsertPt, y&: PhysLiveRegs);
585 for (MachineBasicBlock::iterator I = InsertPt; I != PrevStateChangeI; --I) {
586 if (I->isDebugInstr())
587 continue;
588
589 // Don't move before/into a call (which may have a state change before it).
590 if (I->getOpcode() == TII->getCallFrameDestroyOpcode() || I->isCall())
591 break;
592 LiveUnits.stepBackward(MI: *I);
593 LiveRegs CurrentPhysLiveRegs = getPhysLiveRegs(LiveUnits);
594 // Find places where NZCV is available, but keep looking for locations where
595 // both NZCV and X0 are available, which can avoid some copies.
596 if (!(CurrentPhysLiveRegs & LiveRegs::NZCV))
597 BestCandidate = {I, CurrentPhysLiveRegs};
598 if (CurrentPhysLiveRegs == LiveRegs::None)
599 break;
600 }
601 return BestCandidate;
602}
603
604void MachineSMEABI::insertStateChanges(EmitContext &Context,
605 const FunctionInfo &FnInfo,
606 const EdgeBundles &Bundles,
607 ArrayRef<ZAState> BundleStates) {
608 for (MachineBasicBlock &MBB : *MF) {
609 const BlockInfo &Block = FnInfo.Blocks[MBB.getNumber()];
610 ZAState InState = BundleStates[Bundles.getBundle(N: MBB.getNumber(),
611 /*Out=*/false)];
612
613 ZAState CurrentState = Block.FixedEntryState;
614 if (CurrentState == ZAState::ANY)
615 CurrentState = InState;
616
617 for (auto &Inst : Block.Insts) {
618 if (CurrentState != Inst.NeededState) {
619 auto [InsertPt, PhysLiveRegs] =
620 findStateChangeInsertionPoint(MBB, Block, Inst: &Inst);
621 emitStateChange(Context, MBB, MBBI: InsertPt, From: CurrentState, To: Inst.NeededState,
622 PhysLiveRegs);
623 CurrentState = Inst.NeededState;
624 }
625 }
626
627 if (MBB.succ_empty())
628 continue;
629
630 ZAState OutState =
631 BundleStates[Bundles.getBundle(N: MBB.getNumber(), /*Out=*/true)];
632 if (CurrentState != OutState) {
633 auto [InsertPt, PhysLiveRegs] =
634 findStateChangeInsertionPoint(MBB, Block, Inst: Block.Insts.end());
635 emitStateChange(Context, MBB, MBBI: InsertPt, From: CurrentState, To: OutState,
636 PhysLiveRegs);
637 }
638 }
639}
640
641static DebugLoc getDebugLoc(MachineBasicBlock &MBB,
642 MachineBasicBlock::iterator MBBI) {
643 if (MBB.empty())
644 return DebugLoc();
645 return MBBI != MBB.end() ? MBBI->getDebugLoc() : MBB.back().getDebugLoc();
646}
647
648/// Finds the first call (as determined by MachineInstr::isCall()) starting from
649/// \p MBBI in \p MBB marked with \p Marker (which is a marker opcode such as
650/// RequiresZASavePseudo). If a marked call is found, it is pushed to \p Calls
651/// and the function returns true.
652static bool findMarkedCall(const MachineBasicBlock &MBB,
653 MachineBasicBlock::const_iterator MBBI,
654 SmallVectorImpl<const MachineInstr *> &Calls,
655 unsigned Marker, unsigned CallDestroyOpcode) {
656 auto IsMarker = [&](auto &MI) { return MI.getOpcode() == Marker; };
657 auto MarkerInst = std::find_if(first: MBBI, last: MBB.end(), pred: IsMarker);
658 if (MarkerInst == MBB.end())
659 return false;
660 MachineBasicBlock::const_iterator I = MarkerInst;
661 while (++I != MBB.end()) {
662 if (I->isCall() || I->getOpcode() == CallDestroyOpcode)
663 break;
664 }
665 if (I != MBB.end() && I->isCall())
666 Calls.push_back(Elt: &*I);
667 // Note: This function always returns true if a "Marker" was found.
668 return true;
669}
670
671void MachineSMEABI::collectReachableMarkedCalls(
672 const MachineBasicBlock &StartMBB,
673 MachineBasicBlock::const_iterator StartInst,
674 SmallVectorImpl<const MachineInstr *> &Calls, unsigned Marker) const {
675 assert(Marker == AArch64::InOutZAUsePseudo ||
676 Marker == AArch64::RequiresZASavePseudo ||
677 Marker == AArch64::RequiresZT0SavePseudo);
678 unsigned CallDestroyOpcode = TII->getCallFrameDestroyOpcode();
679 if (findMarkedCall(MBB: StartMBB, MBBI: StartInst, Calls, Marker, CallDestroyOpcode))
680 return;
681
682 SmallPtrSet<const MachineBasicBlock *, 4> Visited;
683 SmallVector<const MachineBasicBlock *> Worklist(StartMBB.succ_rbegin(),
684 StartMBB.succ_rend());
685 while (!Worklist.empty()) {
686 const MachineBasicBlock *MBB = Worklist.pop_back_val();
687 auto [_, Inserted] = Visited.insert(Ptr: MBB);
688 if (!Inserted)
689 continue;
690
691 if (!findMarkedCall(MBB: *MBB, MBBI: MBB->begin(), Calls, Marker, CallDestroyOpcode))
692 Worklist.append(in_start: MBB->succ_rbegin(), in_end: MBB->succ_rend());
693 }
694}
695
696static StringRef getCalleeName(const MachineInstr &CallInst) {
697 assert(CallInst.isCall() && "expected a call");
698 for (const MachineOperand &MO : CallInst.operands()) {
699 if (MO.isSymbol())
700 return MO.getSymbolName();
701 if (MO.isGlobal())
702 return MO.getGlobal()->getName();
703 }
704 return {};
705}
706
707void MachineSMEABI::emitCallSaveRemarks(const MachineBasicBlock &MBB,
708 MachineBasicBlock::const_iterator MBBI,
709 DebugLoc DL, unsigned Marker,
710 StringRef RemarkName,
711 StringRef SaveName) const {
712 auto SaveRemark = [&](DebugLoc DL, const MachineBasicBlock &MBB) {
713 return MachineOptimizationRemarkAnalysis("sme", RemarkName, DL, &MBB);
714 };
715 StringRef StateName = Marker == AArch64::RequiresZT0SavePseudo ? "ZT0" : "ZA";
716 ORE->emit(RemarkBuilder: [&] {
717 return SaveRemark(DL, MBB) << SaveName << " of " << StateName
718 << " emitted in '" << MF->getName() << "'";
719 });
720 if (!ORE->allowExtraAnalysis(PassName: "sme"))
721 return;
722 SmallVector<const MachineInstr *> CallsRequiringSaves;
723 collectReachableMarkedCalls(StartMBB: MBB, StartInst: MBBI, Calls&: CallsRequiringSaves, Marker);
724 for (const MachineInstr *CallInst : CallsRequiringSaves) {
725 auto R = SaveRemark(CallInst->getDebugLoc(), *CallInst->getParent());
726 R << "call";
727 if (StringRef CalleeName = getCalleeName(CallInst: *CallInst); !CalleeName.empty())
728 R << " to '" << CalleeName << "'";
729 R << " requires " << StateName << " save";
730 ORE->emit(OptDiag&: R);
731 }
732}
733
734void MachineSMEABI::emitSetupLazySave(EmitContext &Context,
735 MachineBasicBlock &MBB,
736 MachineBasicBlock::iterator MBBI) {
737 DebugLoc DL = getDebugLoc(MBB, MBBI);
738
739 emitCallSaveRemarks(MBB, MBBI, DL, Marker: AArch64::RequiresZASavePseudo,
740 RemarkName: "SMELazySaveZA", SaveName: "lazy save");
741
742 // Get pointer to TPIDR2 block.
743 Register TPIDR2 = MRI->createVirtualRegister(RegClass: &AArch64::GPR64spRegClass);
744 Register TPIDR2Ptr = MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
745 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri), DestReg: TPIDR2)
746 .addFrameIndex(Idx: Context.getTPIDR2Block(MF&: *MF))
747 .addImm(Val: 0)
748 .addImm(Val: 0);
749 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: TPIDR2Ptr)
750 .addReg(RegNo: TPIDR2);
751 // Set TPIDR2_EL0 to point to TPIDR2 block.
752 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MSR))
753 .addImm(Val: AArch64SysReg::TPIDR2_EL0)
754 .addReg(RegNo: TPIDR2Ptr);
755}
756
757PhysRegSave MachineSMEABI::createPhysRegSave(LiveRegs PhysLiveRegs,
758 MachineBasicBlock &MBB,
759 MachineBasicBlock::iterator MBBI,
760 DebugLoc DL) {
761 PhysRegSave RegSave{.PhysLiveRegs: PhysLiveRegs};
762 if (PhysLiveRegs & LiveRegs::NZCV) {
763 RegSave.StatusFlags = MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
764 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MRS), DestReg: RegSave.StatusFlags)
765 .addImm(Val: AArch64SysReg::NZCV)
766 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit)
767 .setOperandDead(2); // implicit-def $nzcv
768 }
769 // Note: Preserving X0 is "free" as this is before register allocation, so
770 // the register allocator is still able to optimize these copies.
771 if (PhysLiveRegs & LiveRegs::W0) {
772 RegSave.X0Save = MRI->createVirtualRegister(RegClass: PhysLiveRegs & LiveRegs::W0_HI
773 ? &AArch64::GPR64RegClass
774 : &AArch64::GPR32RegClass);
775 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: RegSave.X0Save)
776 .addReg(RegNo: PhysLiveRegs & LiveRegs::W0_HI ? AArch64::X0 : AArch64::W0);
777 }
778 return RegSave;
779}
780
781void MachineSMEABI::restorePhyRegSave(const PhysRegSave &RegSave,
782 MachineBasicBlock &MBB,
783 MachineBasicBlock::iterator MBBI,
784 DebugLoc DL) {
785 if (RegSave.StatusFlags.isValid())
786 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MSR))
787 .addImm(Val: AArch64SysReg::NZCV)
788 .addReg(RegNo: RegSave.StatusFlags)
789 .addReg(RegNo: AArch64::NZCV, Flags: RegState::ImplicitDefine);
790
791 if (RegSave.X0Save.isValid())
792 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY),
793 DestReg: RegSave.PhysLiveRegs & LiveRegs::W0_HI ? AArch64::X0 : AArch64::W0)
794 .addReg(RegNo: RegSave.X0Save);
795}
796
797void MachineSMEABI::addSMELibCall(MachineInstrBuilder &MIB, RTLIB::Libcall LC,
798 CallingConv::ID ExpectedCC) {
799 RTLIB::LibcallImpl LCImpl = LLI->getLibcallImpl(Call: LC);
800 if (LCImpl == RTLIB::Unsupported)
801 emitError(Message: "cannot lower SME ABI (SME routines unsupported)");
802 CallingConv::ID CC = LLI->getLibcallImplCallingConv(Call: LCImpl);
803 StringRef ImplName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: LCImpl);
804 if (CC != ExpectedCC)
805 emitError(Message: "invalid calling convention for SME routine: '" + ImplName + "'");
806 // FIXME: This assumes the ImplName StringRef is null-terminated.
807 MIB.addExternalSymbol(FnName: ImplName.data());
808 MIB.addRegMask(Mask: TRI->getCallPreservedMask(MF: *MF, CC));
809}
810
811void MachineSMEABI::emitRestoreLazySave(EmitContext &Context,
812 MachineBasicBlock &MBB,
813 MachineBasicBlock::iterator MBBI,
814 LiveRegs PhysLiveRegs) {
815 DebugLoc DL = getDebugLoc(MBB, MBBI);
816 Register TPIDR2EL0 = MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
817 Register TPIDR2 = AArch64::X0;
818
819 // TODO: Emit these within the restore MBB to prevent unnecessary saves.
820 PhysRegSave RegSave = createPhysRegSave(PhysLiveRegs, MBB, MBBI, DL);
821
822 // Enable ZA.
823 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MSRpstatesvcrImm1))
824 .addImm(Val: AArch64SVCR::SVCRZA)
825 .addImm(Val: 1);
826 // Get current TPIDR2_EL0.
827 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MRS), DestReg: TPIDR2EL0)
828 .addImm(Val: AArch64SysReg::TPIDR2_EL0)
829 .setOperandDead(2); // implicit-def $nzcv
830 // Get pointer to TPIDR2 block.
831 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri), DestReg: TPIDR2)
832 .addFrameIndex(Idx: Context.getTPIDR2Block(MF&: *MF))
833 .addImm(Val: 0)
834 .addImm(Val: 0);
835 // (Conditionally) restore ZA state.
836 auto RestoreZA = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::RestoreZAPseudo))
837 .addReg(RegNo: TPIDR2EL0)
838 .addReg(RegNo: TPIDR2);
839 addSMELibCall(
840 MIB&: RestoreZA, LC: RTLIB::SMEABI_TPIDR2_RESTORE,
841 ExpectedCC: CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0);
842 // Zero TPIDR2_EL0.
843 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MSR))
844 .addImm(Val: AArch64SysReg::TPIDR2_EL0)
845 .addReg(RegNo: AArch64::XZR);
846
847 restorePhyRegSave(RegSave, MBB, MBBI, DL);
848}
849
850void MachineSMEABI::emitZAMode(MachineBasicBlock &MBB,
851 MachineBasicBlock::iterator MBBI,
852 bool ClearTPIDR2, bool On) {
853 DebugLoc DL = getDebugLoc(MBB, MBBI);
854
855 if (ClearTPIDR2)
856 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MSR))
857 .addImm(Val: AArch64SysReg::TPIDR2_EL0)
858 .addReg(RegNo: AArch64::XZR);
859
860 // Disable ZA.
861 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MSRpstatesvcrImm1))
862 .addImm(Val: AArch64SVCR::SVCRZA)
863 .addImm(Val: On ? 1 : 0);
864}
865
866void MachineSMEABI::emitAllocateLazySaveBuffer(
867 EmitContext &Context, MachineBasicBlock &MBB,
868 MachineBasicBlock::iterator MBBI) {
869 MachineFrameInfo &MFI = MF->getFrameInfo();
870 DebugLoc DL = getDebugLoc(MBB, MBBI);
871 Register SP = MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
872 Register SVL = MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
873 Register Buffer = AFI->getEarlyAllocSMESaveBuffer();
874
875 // Calculate SVL.
876 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::RDSVLI_XI), DestReg: SVL).addImm(Val: 1);
877
878 // 1. Allocate the lazy save buffer.
879 if (!Buffer.isValid()) {
880 // TODO: On Windows, we allocate the lazy save buffer in SelectionDAG (so
881 // Buffer is valid). This is done to reuse the existing
882 // expansions (which can insert stack checks). This works, but it means we
883 // will always allocate the lazy save buffer (even if the function contains
884 // no lazy saves). If we want to handle Windows here, we'll need to
885 // implement something similar to LowerWindowsDYNAMIC_STACKALLOC.
886 assert(!Subtarget->isTargetWindowsOrUEFI() &&
887 "Lazy ZA save is not yet supported on Windows or UEFI");
888 Buffer = MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
889 // Get original stack pointer.
890 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: SP)
891 .addReg(RegNo: AArch64::SP);
892 // Allocate a lazy-save buffer object of the size given, normally SVL * SVL
893 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MSUBXrrr), DestReg: Buffer)
894 .addReg(RegNo: SVL)
895 .addReg(RegNo: SVL)
896 .addReg(RegNo: SP);
897 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: AArch64::SP)
898 .addReg(RegNo: Buffer);
899 // We have just allocated a variable sized object, tell this to PEI.
900 MFI.CreateVariableSizedObject(Alignment: Align(16), Alloca: nullptr);
901 }
902
903 // 2. Setup the TPIDR2 block.
904 {
905 // Note: This case just needs to do `SVL << 48`. It is not implemented as we
906 // generally don't support big-endian SVE/SME.
907 if (!Subtarget->isLittleEndian())
908 reportFatalInternalError(
909 reason: "TPIDR2 block initialization is not supported on big-endian targets");
910
911 // Store buffer pointer and num_za_save_slices.
912 // Bytes 10-15 are implicitly zeroed.
913 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::STPXi))
914 .addReg(RegNo: Buffer)
915 .addReg(RegNo: SVL)
916 .addFrameIndex(Idx: Context.getTPIDR2Block(MF&: *MF))
917 .addImm(Val: 0);
918 }
919}
920
921static constexpr unsigned ZERO_ALL_ZA_MASK = 0b11111111;
922
923void MachineSMEABI::emitSMEPrologue(MachineBasicBlock &MBB,
924 MachineBasicBlock::iterator MBBI) {
925 DebugLoc DL = getDebugLoc(MBB, MBBI);
926
927 bool ZeroZA = AFI->getSMEFnAttrs().isNewZA();
928 bool ZeroZT0 = AFI->getSMEFnAttrs().isNewZT0();
929 if (AFI->getSMEFnAttrs().hasPrivateZAInterface()) {
930 // Get current TPIDR2_EL0.
931 Register TPIDR2EL0 = MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
932 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MRS))
933 .addReg(RegNo: TPIDR2EL0, Flags: RegState::Define)
934 .addImm(Val: AArch64SysReg::TPIDR2_EL0)
935 .setOperandDead(2); // implicit-def $nzcv
936 // If TPIDR2_EL0 is non-zero, commit the lazy save.
937 // NOTE: Functions that only use ZT0 don't need to zero ZA.
938 auto CommitZASave =
939 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::CommitZASavePseudo))
940 .addReg(RegNo: TPIDR2EL0)
941 .addImm(Val: ZeroZA)
942 .addImm(Val: ZeroZT0);
943 addSMELibCall(
944 MIB&: CommitZASave, LC: RTLIB::SMEABI_TPIDR2_SAVE,
945 ExpectedCC: CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0);
946 if (ZeroZA)
947 CommitZASave.addDef(RegNo: AArch64::ZAB0, Flags: RegState::ImplicitDefine);
948 if (ZeroZT0)
949 CommitZASave.addDef(RegNo: AArch64::ZT0, Flags: RegState::ImplicitDefine);
950 // Enable ZA (as ZA could have previously been in the OFF state).
951 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MSRpstatesvcrImm1))
952 .addImm(Val: AArch64SVCR::SVCRZA)
953 .addImm(Val: 1);
954 } else if (AFI->getSMEFnAttrs().hasSharedZAInterface()) {
955 if (ZeroZA)
956 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ZERO_M))
957 .addImm(Val: ZERO_ALL_ZA_MASK)
958 .addDef(RegNo: AArch64::ZAB0, Flags: RegState::ImplicitDefine);
959 if (ZeroZT0)
960 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ZERO_T)).addDef(RegNo: AArch64::ZT0);
961 }
962}
963
964void MachineSMEABI::emitFullZASaveRestore(EmitContext &Context,
965 MachineBasicBlock &MBB,
966 MachineBasicBlock::iterator MBBI,
967 LiveRegs PhysLiveRegs, bool IsSave) {
968 DebugLoc DL = getDebugLoc(MBB, MBBI);
969
970 if (IsSave)
971 emitCallSaveRemarks(MBB, MBBI, DL, Marker: AArch64::RequiresZASavePseudo,
972 RemarkName: "SMEFullZASave", SaveName: "full save");
973
974 PhysRegSave RegSave = createPhysRegSave(PhysLiveRegs, MBB, MBBI, DL);
975
976 // Copy the buffer pointer into X0.
977 Register BufferPtr = AArch64::X0;
978 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: BufferPtr)
979 .addReg(RegNo: Context.getAgnosticZABufferPtr(MF&: *MF));
980
981 // Call __arm_sme_save/__arm_sme_restore.
982 auto SaveRestoreZA = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::BL))
983 .addReg(RegNo: BufferPtr, Flags: RegState::Implicit);
984 addSMELibCall(
985 MIB&: SaveRestoreZA,
986 LC: IsSave ? RTLIB::SMEABI_SME_SAVE : RTLIB::SMEABI_SME_RESTORE,
987 ExpectedCC: CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1);
988
989 restorePhyRegSave(RegSave, MBB, MBBI, DL);
990}
991
992void MachineSMEABI::emitZT0SaveRestore(EmitContext &Context,
993 MachineBasicBlock &MBB,
994 MachineBasicBlock::iterator MBBI,
995 bool IsSave) {
996 DebugLoc DL = getDebugLoc(MBB, MBBI);
997
998 // Note: This will report calls that _only_ need ZT0 saved. Call that save
999 // both ZA and ZT0 will be under the SMELazySaveZA remark. This prevents
1000 // reporting the same calls twice.
1001 if (IsSave)
1002 emitCallSaveRemarks(MBB, MBBI, DL, Marker: AArch64::RequiresZT0SavePseudo,
1003 RemarkName: "SMEZT0Save", SaveName: "spill");
1004
1005 Register ZT0Save = MRI->createVirtualRegister(RegClass: &AArch64::GPR64spRegClass);
1006
1007 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri), DestReg: ZT0Save)
1008 .addFrameIndex(Idx: Context.getZT0SaveSlot(MF&: *MF))
1009 .addImm(Val: 0)
1010 .addImm(Val: 0);
1011
1012 if (IsSave) {
1013 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::STR_TX))
1014 .addReg(RegNo: AArch64::ZT0)
1015 .addReg(RegNo: ZT0Save);
1016 } else {
1017 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::LDR_TX), DestReg: AArch64::ZT0)
1018 .addReg(RegNo: ZT0Save);
1019 }
1020}
1021
1022void MachineSMEABI::emitAllocateFullZASaveBuffer(
1023 EmitContext &Context, MachineBasicBlock &MBB,
1024 MachineBasicBlock::iterator MBBI, LiveRegs PhysLiveRegs) {
1025 // Buffer already allocated in SelectionDAG.
1026 if (AFI->getEarlyAllocSMESaveBuffer())
1027 return;
1028
1029 DebugLoc DL = getDebugLoc(MBB, MBBI);
1030 Register BufferPtr = Context.getAgnosticZABufferPtr(MF&: *MF);
1031 Register BufferSize = MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
1032
1033 PhysRegSave RegSave = createPhysRegSave(PhysLiveRegs, MBB, MBBI, DL);
1034
1035 // Calculate the SME state size.
1036 {
1037 auto SMEStateSize = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::BL))
1038 .addReg(RegNo: AArch64::X0, Flags: RegState::ImplicitDefine);
1039 addSMELibCall(
1040 MIB&: SMEStateSize, LC: RTLIB::SMEABI_SME_STATE_SIZE,
1041 ExpectedCC: CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1);
1042 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: BufferSize)
1043 .addReg(RegNo: AArch64::X0);
1044 }
1045
1046 // Allocate a buffer object of the size given __arm_sme_state_size.
1047 {
1048 MachineFrameInfo &MFI = MF->getFrameInfo();
1049 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SUBXrx64), DestReg: AArch64::SP)
1050 .addReg(RegNo: AArch64::SP)
1051 .addReg(RegNo: BufferSize)
1052 .addImm(Val: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTX, Imm: 0));
1053 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: BufferPtr)
1054 .addReg(RegNo: AArch64::SP);
1055
1056 // We have just allocated a variable sized object, tell this to PEI.
1057 MFI.CreateVariableSizedObject(Alignment: Align(16), Alloca: nullptr);
1058 }
1059
1060 restorePhyRegSave(RegSave, MBB, MBBI, DL);
1061}
1062
1063struct FromState {
1064 ZAState From;
1065
1066 constexpr uint8_t to(ZAState To) const {
1067 static_assert(NUM_ZA_STATE < 16, "expected ZAState to fit in 4-bits");
1068 return uint8_t(From) << 4 | uint8_t(To);
1069 }
1070};
1071
1072constexpr FromState transitionFrom(ZAState From) { return FromState{.From: From}; }
1073
1074void MachineSMEABI::emitStateChange(EmitContext &Context,
1075 MachineBasicBlock &MBB,
1076 MachineBasicBlock::iterator InsertPt,
1077 ZAState From, ZAState To,
1078 LiveRegs PhysLiveRegs) {
1079 // ZA not used.
1080 if (From == ZAState::ANY || To == ZAState::ANY)
1081 return;
1082
1083 // If we're exiting from the ENTRY state that means that the function has not
1084 // used ZA, so in the case of private ZA/ZT0 functions we can omit any set up.
1085 if (From == ZAState::ENTRY && To == ZAState::OFF)
1086 return;
1087
1088 // TODO: Avoid setting up the save buffer if there's no transition to
1089 // LOCAL_SAVED.
1090 if (From == ZAState::ENTRY) {
1091 assert(&MBB == &MBB.getParent()->front() &&
1092 "ENTRY state only valid in entry block");
1093 emitSMEPrologue(MBB, MBBI: MBB.getFirstNonPHI());
1094 if (To == ZAState::ACTIVE)
1095 return; // Nothing more to do (ZA is active after the prologue).
1096
1097 // Note: "emitNewZAPrologue" zeros ZA, so we may need to setup a lazy save
1098 // if "To" is "ZAState::LOCAL_SAVED". It may be possible to improve this
1099 // case by changing the placement of the zero instruction.
1100 From = ZAState::ACTIVE;
1101 }
1102
1103 SMEAttrs SMEFnAttrs = AFI->getSMEFnAttrs();
1104 bool IsAgnosticZA = SMEFnAttrs.hasAgnosticZAInterface();
1105 bool HasZT0State = SMEFnAttrs.hasZT0State();
1106 bool HasZAState = IsAgnosticZA || SMEFnAttrs.hasZAState();
1107
1108 switch (transitionFrom(From).to(To)) {
1109 // This section handles: ACTIVE <-> ACTIVE_ZT0_SAVED
1110 case transitionFrom(From: ZAState::ACTIVE).to(To: ZAState::ACTIVE_ZT0_SAVED):
1111 emitZT0SaveRestore(Context, MBB, MBBI: InsertPt, /*IsSave=*/true);
1112 break;
1113 case transitionFrom(From: ZAState::ACTIVE_ZT0_SAVED).to(To: ZAState::ACTIVE):
1114 emitZT0SaveRestore(Context, MBB, MBBI: InsertPt, /*IsSave=*/false);
1115 break;
1116
1117 // This section handles: ACTIVE[_ZT0_SAVED] -> LOCAL_SAVED
1118 case transitionFrom(From: ZAState::ACTIVE).to(To: ZAState::LOCAL_SAVED):
1119 case transitionFrom(From: ZAState::ACTIVE_ZT0_SAVED).to(To: ZAState::LOCAL_SAVED):
1120 if (HasZT0State && From == ZAState::ACTIVE)
1121 emitZT0SaveRestore(Context, MBB, MBBI: InsertPt, /*IsSave=*/true);
1122 if (HasZAState)
1123 emitZASave(Context, MBB, MBBI: InsertPt, PhysLiveRegs);
1124 break;
1125
1126 // This section handles: ACTIVE -> LOCAL_COMMITTED
1127 case transitionFrom(From: ZAState::ACTIVE).to(To: ZAState::LOCAL_COMMITTED):
1128 // TODO: We could support ZA state here, but this transition is currently
1129 // only possible when we _don't_ have ZA state.
1130 assert(HasZT0State && !HasZAState && "Expect to only have ZT0 state.");
1131 emitZT0SaveRestore(Context, MBB, MBBI: InsertPt, /*IsSave=*/true);
1132 emitZAMode(MBB, MBBI: InsertPt, /*ClearTPIDR2=*/false, /*On=*/false);
1133 break;
1134
1135 // This section handles: LOCAL_COMMITTED -> (OFF|LOCAL_SAVED)
1136 case transitionFrom(From: ZAState::LOCAL_COMMITTED).to(To: ZAState::OFF):
1137 case transitionFrom(From: ZAState::LOCAL_COMMITTED).to(To: ZAState::LOCAL_SAVED):
1138 // These transitions are a no-op.
1139 break;
1140
1141 // This section handles: LOCAL_(SAVED|COMMITTED) -> ACTIVE[_ZT0_SAVED]
1142 case transitionFrom(From: ZAState::LOCAL_COMMITTED).to(To: ZAState::ACTIVE):
1143 case transitionFrom(From: ZAState::LOCAL_COMMITTED).to(To: ZAState::ACTIVE_ZT0_SAVED):
1144 case transitionFrom(From: ZAState::LOCAL_SAVED).to(To: ZAState::ACTIVE):
1145 case transitionFrom(From: ZAState::LOCAL_SAVED).to(To: ZAState::ACTIVE_ZT0_SAVED):
1146 if (HasZAState)
1147 emitZARestore(Context, MBB, MBBI: InsertPt, PhysLiveRegs);
1148 else
1149 emitZAMode(MBB, MBBI: InsertPt, /*ClearTPIDR2=*/false, /*On=*/true);
1150 if (HasZT0State && To == ZAState::ACTIVE)
1151 emitZT0SaveRestore(Context, MBB, MBBI: InsertPt, /*IsSave=*/false);
1152 break;
1153
1154 // This section handles transitions to OFF (not previously covered)
1155 case transitionFrom(From: ZAState::ACTIVE).to(To: ZAState::OFF):
1156 case transitionFrom(From: ZAState::ACTIVE_ZT0_SAVED).to(To: ZAState::OFF):
1157 case transitionFrom(From: ZAState::LOCAL_SAVED).to(To: ZAState::OFF):
1158 assert(SMEFnAttrs.hasPrivateZAInterface() &&
1159 "Did not expect to turn ZA off in shared/agnostic ZA function");
1160 emitZAMode(MBB, MBBI: InsertPt, /*ClearTPIDR2=*/From == ZAState::LOCAL_SAVED,
1161 /*On=*/false);
1162 break;
1163
1164 default:
1165 dbgs() << "Error: Transition from " << getZAStateString(State: From) << " to "
1166 << getZAStateString(State: To) << '\n';
1167 llvm_unreachable("Unimplemented state transition");
1168 }
1169}
1170
1171/// Returns true if private ZA setup can be elided. This occurs when there is
1172/// no instruction within the function that requires ZA to be active.
1173static bool canElidePrivateZASetup(const FunctionInfo &FnInfo) {
1174 for (const BlockInfo &BlockInfo : FnInfo.Blocks) {
1175 for (const InstInfo &InstInfo : BlockInfo.Insts) {
1176 if (InstInfo.NeededState == ZAState::ACTIVE ||
1177 InstInfo.NeededState == ZAState::ACTIVE_ZT0_SAVED)
1178 return false;
1179 }
1180 }
1181 return true;
1182}
1183
1184} // end anonymous namespace
1185
1186INITIALIZE_PASS(MachineSMEABI, "aarch64-machine-sme-abi", "Machine SME ABI",
1187 false, false)
1188
1189bool MachineSMEABI::runOnMachineFunction(MachineFunction &MF) {
1190 AFI = MF.getInfo<AArch64FunctionInfo>();
1191 SMEAttrs SMEFnAttrs = AFI->getSMEFnAttrs();
1192 if (!SMEFnAttrs.hasZAState() && !SMEFnAttrs.hasZT0State() &&
1193 !SMEFnAttrs.hasAgnosticZAInterface())
1194 return false;
1195
1196 Subtarget = &MF.getSubtarget<AArch64Subtarget>();
1197 if (!Subtarget->hasSME() && !SMEFnAttrs.hasAgnosticZAInterface())
1198 return false;
1199
1200 assert(MF.getRegInfo().isSSA() && "Expected to be run on SSA form!");
1201
1202 this->MF = &MF;
1203 ORE = &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE();
1204 LLI = &getAnalysis<LibcallLoweringInfoWrapper>().getLibcallLowering(
1205 M: *MF.getFunction().getParent(), Subtarget: *Subtarget);
1206 TII = Subtarget->getInstrInfo();
1207 TRI = Subtarget->getRegisterInfo();
1208 MRI = &MF.getRegInfo();
1209
1210 const EdgeBundles &Bundles =
1211 getAnalysis<EdgeBundlesWrapperLegacy>().getEdgeBundles();
1212
1213 FunctionInfo FnInfo = collectNeededZAStates(SMEFnAttrs);
1214
1215 if (SMEFnAttrs.hasPrivateZAInterface() && canElidePrivateZASetup(FnInfo))
1216 return false;
1217
1218 SmallVector<ZAState> BundleStates = assignBundleZAStates(Bundles, FnInfo);
1219
1220 EmitContext Context;
1221 insertStateChanges(Context, FnInfo, Bundles, BundleStates);
1222
1223 if (Context.needsSaveBuffer()) {
1224 if (FnInfo.AfterSMEProloguePt) {
1225 // Note: With inline stack probes the AfterSMEProloguePt may not be in the
1226 // entry block (due to the probing loop).
1227 MachineBasicBlock::iterator MBBI = *FnInfo.AfterSMEProloguePt;
1228 emitAllocateZASaveBuffer(Context, MBB&: *MBBI->getParent(), MBBI,
1229 PhysLiveRegs: FnInfo.PhysLiveRegsAfterSMEPrologue);
1230 } else {
1231 MachineBasicBlock &EntryBlock = MF.front();
1232 emitAllocateZASaveBuffer(
1233 Context, MBB&: EntryBlock, MBBI: EntryBlock.getFirstNonPHI(),
1234 PhysLiveRegs: FnInfo.Blocks[EntryBlock.getNumber()].PhysLiveRegsAtEntry);
1235 }
1236 }
1237
1238 return true;
1239}
1240
1241FunctionPass *llvm::createMachineSMEABIPass() { return new MachineSMEABI(); }
1242