1//===- AArch64PostCoalescerPass.cpp - AArch64 Post Coalescer pass ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AArch64.h"
10#include "AArch64MachineFunctionInfo.h"
11#include "llvm/CodeGen/LiveIntervals.h"
12#include "llvm/CodeGen/MachineFunctionPass.h"
13#include "llvm/CodeGen/MachineRegisterInfo.h"
14#include "llvm/CodeGen/Passes.h"
15#include "llvm/InitializePasses.h"
16
17using namespace llvm;
18
19#define DEBUG_TYPE "aarch64-post-coalescer"
20
21namespace {
22
23/// Expands FORM_TRANSPOSED_REG_TUPLE_{X2|X4}_PSEUDO instructions into copy
24/// sequences. Note: This expansion occurs immediately before greedy regalloc
25/// and after the coalescer and pre-RA scheduler.
26///
27/// Example:
28///
29/// %v2:zpr2 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO %v0.zsub0, %v1.zsub0
30///
31/// Expands to:
32///
33/// undef %v2.zsub0:zpr2 = COPY_INTO_TRANSPOSED_TUPLE %v0.zsub0, 2
34/// %v2.zsub1:zpr2 = COPY_INTO_TRANSPOSED_TUPLE %v1.zsub0, 2
35static bool expandFormTransposedRegTuple(MachineBasicBlock &MBB,
36 MachineInstr &MI, LiveIntervals *LIS) {
37 const TargetInstrInfo *TII =
38 MBB.getParent()->getSubtarget<AArch64Subtarget>().getInstrInfo();
39 unsigned TupleSize =
40 MI.getOpcode() == AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO ? 2 : 4;
41
42 DebugLoc DL = MI.getDebugLoc();
43 Register TupleReg = MI.getOperand(i: 0).getReg();
44 SmallVector<Register, 5> OrigRegs{TupleReg};
45 MachineBasicBlock::iterator FirstCopyMBBI;
46
47 for (unsigned I = 0; I < TupleSize; ++I) {
48 MachineOperand &SrcOp = MI.getOperand(i: I + 1);
49 OrigRegs.push_back(Elt: SrcOp.getReg());
50
51 RegState DefState = I == 0 ? RegState::Undef : RegState::NoFlags;
52 MachineInstr *CopyMI =
53 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AArch64::COPY_INTO_TRANSPOSED_TUPLE))
54 .addDef(RegNo: TupleReg, Flags: DefState, SubReg: AArch64::zsub0 + I)
55 .add(MO: SrcOp)
56 .addImm(Val: TupleSize);
57
58 if (I == 0)
59 FirstCopyMBBI = CopyMI;
60 }
61
62 MachineBasicBlock::iterator EndMBBI = std::next(x: MI.getIterator());
63 if (LIS)
64 LIS->RemoveMachineInstrFromMaps(MI);
65 MI.eraseFromParent();
66
67 if (LIS)
68 LIS->repairIntervalsInRange(MBB: &MBB, Begin: FirstCopyMBBI, End: EndMBBI, OrigRegs);
69 return true;
70}
71
72bool runAArch64PostCoalescer(MachineFunction &MF, LiveIntervals *LIS) {
73 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
74 if (!FuncInfo->hasStreamingModeChanges() &&
75 !MF.getSubtarget<AArch64Subtarget>().isStreaming())
76 return false;
77
78 MachineRegisterInfo &MRI = MF.getRegInfo();
79 bool Changed = false;
80
81 for (MachineBasicBlock &MBB : MF) {
82 for (MachineInstr &MI : make_early_inc_range(Range&: MBB)) {
83 switch (MI.getOpcode()) {
84 default:
85 break;
86 case AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO:
87 case AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO:
88 Changed |= expandFormTransposedRegTuple(MBB, MI, LIS);
89 break;
90 case AArch64::COALESCER_BARRIER_FPR16:
91 case AArch64::COALESCER_BARRIER_FPR32:
92 case AArch64::COALESCER_BARRIER_FPR64:
93 case AArch64::COALESCER_BARRIER_FPR128: {
94 Register Src = MI.getOperand(i: 1).getReg();
95 Register Dst = MI.getOperand(i: 0).getReg();
96 if (Src != Dst)
97 MRI.replaceRegWith(FromReg: Dst, ToReg: Src);
98
99 if (MI.getOperand(i: 1).isUndef())
100 for (MachineOperand &MO : MRI.use_operands(Reg: Dst))
101 MO.setIsUndef();
102
103 // MI must be erased from the basic block before recalculating the live
104 // interval.
105 if (LIS)
106 LIS->RemoveMachineInstrFromMaps(MI);
107 MI.eraseFromParent();
108
109 if (LIS) {
110 LIS->removeInterval(Reg: Src);
111 LIS->createAndComputeVirtRegInterval(Reg: Src);
112 }
113
114 Changed = true;
115 break;
116 }
117 }
118 }
119 }
120
121 return Changed;
122}
123
124struct AArch64PostCoalescerLegacy : public MachineFunctionPass {
125 static char ID;
126
127 AArch64PostCoalescerLegacy() : MachineFunctionPass(ID) {}
128
129 bool runOnMachineFunction(MachineFunction &MF) override;
130
131 StringRef getPassName() const override {
132 return "AArch64 Post Coalescer pass";
133 }
134
135 void getAnalysisUsage(AnalysisUsage &AU) const override {
136 AU.setPreservesCFG();
137 AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
138 AU.addPreserved<LiveIntervalsWrapperPass>();
139 AU.addPreserved<SlotIndexesWrapperPass>();
140 MachineFunctionPass::getAnalysisUsage(AU);
141 }
142};
143
144char AArch64PostCoalescerLegacy::ID = 0;
145
146} // end anonymous namespace
147
148INITIALIZE_PASS_BEGIN(AArch64PostCoalescerLegacy, "aarch64-post-coalescer",
149 "AArch64 Post Coalescer Pass", false, false)
150INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
151INITIALIZE_PASS_END(AArch64PostCoalescerLegacy, "aarch64-post-coalescer",
152 "AArch64 Post Coalescer Pass", false, false)
153
154bool AArch64PostCoalescerLegacy::runOnMachineFunction(MachineFunction &MF) {
155 if (skipFunction(F: MF.getFunction()))
156 return false;
157
158 auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
159 auto *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
160 return runAArch64PostCoalescer(MF, LIS);
161}
162
163PreservedAnalyses
164AArch64PostCoalescerPass::run(MachineFunction &MF,
165 MachineFunctionAnalysisManager &MFAM) {
166 auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(IR&: MF);
167 const bool Changed = runAArch64PostCoalescer(MF, LIS);
168 if (!Changed)
169 return PreservedAnalyses::all();
170 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
171 PA.preserveSet<CFGAnalyses>();
172 PA.preserve<LiveIntervalsAnalysis>();
173 PA.preserve<SlotIndexesAnalysis>();
174 return PA;
175}
176
177FunctionPass *llvm::createAArch64PostCoalescerPass() {
178 return new AArch64PostCoalescerLegacy();
179}
180