1//===- AArch64PostCoalescerPass.cpp - AArch64 Post Coalescer pass ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AArch64.h"
10#include "AArch64MachineFunctionInfo.h"
11#include "llvm/CodeGen/LiveIntervals.h"
12#include "llvm/CodeGen/MachineFunctionPass.h"
13#include "llvm/CodeGen/MachineRegisterInfo.h"
14#include "llvm/CodeGen/Passes.h"
15#include "llvm/InitializePasses.h"
16
17using namespace llvm;
18
19#define DEBUG_TYPE "aarch64-post-coalescer"
20
21namespace {
22
23/// Expands FORM_TRANSPOSED_REG_TUPLE_{X2|X4}_PSEUDO instructions into copy
24/// sequences. Note: This expansion occurs immediately before greedy regalloc
25/// and after the coalescer and pre-RA scheduler.
26///
27/// Example:
28///
29/// %v2:zpr2 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO %v0.zsub0, %v1.zsub0
30///
31/// Expands to:
32///
33/// undef %v2.zsub0:zpr2 = COPY_INTO_TRANSPOSED_TUPLE %v0.zsub0, 2
34/// %v2.zsub1:zpr2 = COPY_INTO_TRANSPOSED_TUPLE %v1.zsub0, 2
35static bool expandFormTransposedRegTuple(MachineBasicBlock &MBB,
36 MachineInstr &MI, LiveIntervals *LIS) {
37 const TargetInstrInfo *TII =
38 MBB.getParent()->getSubtarget<AArch64Subtarget>().getInstrInfo();
39 unsigned TupleSize =
40 MI.getOpcode() == AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO ? 2 : 4;
41
42 DebugLoc DL = MI.getDebugLoc();
43 Register TupleReg = MI.getOperand(i: 0).getReg();
44 SmallVector<Register, 5> OrigRegs{TupleReg};
45 MachineBasicBlock::iterator FirstCopyMBBI;
46
47 for (unsigned I = 0; I < TupleSize; ++I) {
48 MachineOperand &SrcOp = MI.getOperand(i: I + 1);
49 OrigRegs.push_back(Elt: SrcOp.getReg());
50
51 // Ensure that if operand is killed, the kill flag is placed on the final
52 // copy for that operand.
53 if (SrcOp.isKill()) {
54 for (unsigned J = I + 2; J < MI.getNumOperands(); ++J) {
55 MachineOperand &LaterOp = MI.getOperand(i: J);
56 if (LaterOp.getReg() == SrcOp.getReg()) {
57 LaterOp.setIsKill();
58 SrcOp.setIsKill(false);
59 }
60 }
61 }
62
63 RegState DefState = I == 0 ? RegState::Undef : RegState::NoFlags;
64 MachineInstr *CopyMI =
65 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AArch64::COPY_INTO_TRANSPOSED_TUPLE))
66 .addDef(RegNo: TupleReg, Flags: DefState, SubReg: AArch64::zsub0 + I)
67 .add(MO: SrcOp)
68 .addImm(Val: TupleSize);
69
70 if (I == 0)
71 FirstCopyMBBI = CopyMI;
72 }
73
74 MachineBasicBlock::iterator EndMBBI = std::next(x: MI.getIterator());
75 if (LIS)
76 LIS->RemoveMachineInstrFromMaps(MI);
77 MI.eraseFromParent();
78
79 if (LIS)
80 LIS->repairIntervalsInRange(MBB: &MBB, Begin: FirstCopyMBBI, End: EndMBBI, OrigRegs);
81 return true;
82}
83
84bool runAArch64PostCoalescer(MachineFunction &MF, LiveIntervals *LIS) {
85 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
86 if (!FuncInfo->hasStreamingModeChanges() &&
87 !MF.getSubtarget<AArch64Subtarget>().isStreaming())
88 return false;
89
90 MachineRegisterInfo &MRI = MF.getRegInfo();
91 bool Changed = false;
92
93 for (MachineBasicBlock &MBB : MF) {
94 for (MachineInstr &MI : make_early_inc_range(Range&: MBB)) {
95 switch (MI.getOpcode()) {
96 default:
97 break;
98 case AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO:
99 case AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO:
100 Changed |= expandFormTransposedRegTuple(MBB, MI, LIS);
101 break;
102 case AArch64::COALESCER_BARRIER_FPR16:
103 case AArch64::COALESCER_BARRIER_FPR32:
104 case AArch64::COALESCER_BARRIER_FPR64:
105 case AArch64::COALESCER_BARRIER_FPR128: {
106 Register Src = MI.getOperand(i: 1).getReg();
107 Register Dst = MI.getOperand(i: 0).getReg();
108 if (Src != Dst)
109 MRI.replaceRegWith(FromReg: Dst, ToReg: Src);
110
111 if (MI.getOperand(i: 1).isUndef())
112 for (MachineOperand &MO : MRI.use_operands(Reg: Dst))
113 MO.setIsUndef();
114
115 // MI must be erased from the basic block before recalculating the live
116 // interval.
117 if (LIS)
118 LIS->RemoveMachineInstrFromMaps(MI);
119 MI.eraseFromParent();
120
121 if (LIS) {
122 LIS->removeInterval(Reg: Src);
123 LIS->createAndComputeVirtRegInterval(Reg: Src);
124 }
125
126 Changed = true;
127 break;
128 }
129 }
130 }
131 }
132
133 return Changed;
134}
135
136struct AArch64PostCoalescerLegacy : public MachineFunctionPass {
137 static char ID;
138
139 AArch64PostCoalescerLegacy() : MachineFunctionPass(ID) {}
140
141 bool runOnMachineFunction(MachineFunction &MF) override;
142
143 StringRef getPassName() const override {
144 return "AArch64 Post Coalescer pass";
145 }
146
147 void getAnalysisUsage(AnalysisUsage &AU) const override {
148 AU.setPreservesCFG();
149 AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
150 AU.addPreserved<LiveIntervalsWrapperPass>();
151 AU.addPreserved<SlotIndexesWrapperPass>();
152 MachineFunctionPass::getAnalysisUsage(AU);
153 }
154};
155
156char AArch64PostCoalescerLegacy::ID = 0;
157
158} // end anonymous namespace
159
160INITIALIZE_PASS_BEGIN(AArch64PostCoalescerLegacy, "aarch64-post-coalescer",
161 "AArch64 Post Coalescer Pass", false, false)
162INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
163INITIALIZE_PASS_END(AArch64PostCoalescerLegacy, "aarch64-post-coalescer",
164 "AArch64 Post Coalescer Pass", false, false)
165
166bool AArch64PostCoalescerLegacy::runOnMachineFunction(MachineFunction &MF) {
167 if (skipFunction(F: MF.getFunction()))
168 return false;
169
170 auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
171 auto *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
172 return runAArch64PostCoalescer(MF, LIS);
173}
174
175PreservedAnalyses
176AArch64PostCoalescerPass::run(MachineFunction &MF,
177 MachineFunctionAnalysisManager &MFAM) {
178 auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(IR&: MF);
179 const bool Changed = runAArch64PostCoalescer(MF, LIS);
180 if (!Changed)
181 return PreservedAnalyses::all();
182 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
183 PA.preserveSet<CFGAnalyses>();
184 PA.preserve<LiveIntervalsAnalysis>();
185 PA.preserve<SlotIndexesAnalysis>();
186 return PA;
187}
188
189FunctionPass *llvm::createAArch64PostCoalescerPass() {
190 return new AArch64PostCoalescerLegacy();
191}
192