1//===-- SIPostRA16BitMovFolding.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass performs the post RA 16bit Mov folding
11///
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "GCNSubtarget.h"
16#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
17#include "llvm/CodeGen/MachineDominators.h"
18#include "llvm/CodeGen/MachineFunctionPass.h"
19#include "llvm/CodeGen/TargetSchedule.h"
20#include "llvm/Support/BranchProbability.h"
21using namespace llvm;
22
23#define DEBUG_TYPE "si-post-ra-16bit-mov-folding"
24
25namespace {
26
27class SIPostRA16BitMovFolding {
28private:
29 const SIInstrInfo *TII = nullptr;
30 const SIRegisterInfo *TRI = nullptr;
31
32 void getMovB16Info(const MachineInstr &MI, const SIRegisterInfo *TRI,
33 MCRegister &SrcReg16, bool &SrcIsVGPR,
34 MCRegister &SrcReg32, bool &SrcIsHi, bool &SrcIsImm,
35 int64_t &ImmVal) const;
36
37 bool mergeSingleMovB16Pair(MachineInstr &Lo, MachineInstr &Hi,
38 bool IsHiFirst) const;
39 bool mergeMovB16Pairs(MachineFunction &MF) const;
40
41public:
42 bool run(MachineFunction &MF);
43};
44
45class SIPostRA16BitMovFoldingLegacy : public MachineFunctionPass {
46public:
47 static char ID;
48
49 SIPostRA16BitMovFoldingLegacy() : MachineFunctionPass(ID) {}
50
51 StringRef getPassName() const override {
52 return "SI post-RA 16bit Mov Folding";
53 }
54
55 void getAnalysisUsage(AnalysisUsage &AU) const override {
56 AU.setPreservesAll();
57 MachineFunctionPass::getAnalysisUsage(AU);
58 }
59
60 bool runOnMachineFunction(MachineFunction &MF) override {
61 return SIPostRA16BitMovFolding().run(MF);
62 }
63};
64
65} // End anonymous namespace.
66
67INITIALIZE_PASS(SIPostRA16BitMovFoldingLegacy, DEBUG_TYPE,
68 "SI Post RA 16bit Mov Folding", false, false)
69
70char SIPostRA16BitMovFoldingLegacy::ID = 0;
71
72char &llvm::SIPostRA16BitMovFoldingLegacyID = SIPostRA16BitMovFoldingLegacy::ID;
73
74// Helper: extract the src operand and whether it is from the hi16 half.
75// Post-RA, both V_MOV_B16_t16_e32 and V_MOV_B16_t16_e64 use VGPR_16 dst
76// physical registers whose encoding already encodes hi/lo (IS_HI16 bit).
77void SIPostRA16BitMovFolding::getMovB16Info(
78 const MachineInstr &MI, const SIRegisterInfo *TRI, MCRegister &SrcReg16,
79 bool &SrcIsVGPR, MCRegister &SrcReg32, bool &SrcIsHi, bool &SrcIsImm,
80 int64_t &ImmVal) const {
81 SrcIsImm = false;
82 SrcIsHi = false;
83 SrcIsVGPR = false;
84 SrcReg16 = MCRegister();
85 SrcReg32 = MCRegister();
86
87 const MachineOperand *SrcOp = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
88
89 if (SrcOp->isImm()) {
90 SrcIsImm = true;
91 ImmVal = SrcOp->getImm();
92 return;
93 }
94
95 SrcReg16 = SrcOp->getReg().asMCReg();
96 SrcIsVGPR = AMDGPU::VGPR_16RegClass.contains(Reg: SrcReg16);
97 if (SrcIsVGPR) {
98 SrcIsHi = AMDGPU::isHi16Reg(Reg: SrcReg16, MRI: *TRI);
99 SrcReg32 = TRI->get32BitRegister(Reg: SrcReg16);
100 } else {
101 SrcIsHi = false;
102 SrcReg32 = SrcReg16;
103 }
104}
105
106// clang-format off
107// Try to merge a pair of v_mov_b16 instructions targeting the lo16 and hi16
108// halves of the same VGPR into a single 32-bit instruction.
109//
110// Caller guarantee the pair to be two v_mov_b16 and targets the same dst32
111//
112// Patterns:
113// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.l/s2 => v_and_b32 v0,0xffff,v2/s2
114// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.h => v_lshrrev_b32 v0,16,v2
115// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.l/s2 => v_lshlrev_b32 v0,16,v2/s2
116// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.h => v_and_b32 v0,0xffff0000,v2
117// v_mov_b16 v0.l, v.x/s v_mov_b16 v0.h, v.y/s => v_perm_b32_e64 v0, v.x/s, v.y/s, mask
118// v_mov_b16 v0.l, imm0 v_mov_b16 v0.h, imm1 => v_mov_b32 v0, (imm1<<16)|imm0
119//
120// A second mov reading the half the first wrote is rewritten to read the first
121// mov's source before matching.
122// clang-format on
123bool SIPostRA16BitMovFolding::mergeSingleMovB16Pair(MachineInstr &Lo,
124 MachineInstr &Hi,
125 bool IsHiFirst) const {
126 // Lo and Hi share the same Dst32
127 MCRegister LoDst = Lo.getOperand(i: 0).getReg().asMCReg();
128 MCRegister HiDst = Hi.getOperand(i: 0).getReg().asMCReg();
129 MCRegister Dst32 = TRI->get32BitRegister(Reg: LoDst);
130
131 // Extract source info for Lo and Hi.
132 MCRegister LoSrc16, LoSrc32, HiSrc16, HiSrc32;
133 bool LoSrcIsHi, HiSrcIsHi, LoSrcIsImm, HiSrcIsImm, LoSrcIsVGPR, HiSrcIsVGPR;
134 int64_t LoImm = 0, HiImm = 0;
135
136 getMovB16Info(MI: Lo, TRI, SrcReg16&: LoSrc16, SrcIsVGPR&: LoSrcIsVGPR, SrcReg32&: LoSrc32, SrcIsHi&: LoSrcIsHi, SrcIsImm&: LoSrcIsImm,
137 ImmVal&: LoImm);
138 getMovB16Info(MI: Hi, TRI, SrcReg16&: HiSrc16, SrcIsVGPR&: HiSrcIsVGPR, SrcReg32&: HiSrc32, SrcIsHi&: HiSrcIsHi, SrcIsImm&: HiSrcIsImm,
139 ImmVal&: HiImm);
140
141 MachineInstr &FirstMI = IsHiFirst ? Hi : Lo;
142 MachineInstr &SecondMI = IsHiFirst ? Lo : Hi;
143
144 // Data Conflict counter
145 MachineBasicBlock::iterator UpperBound = SecondMI.getIterator();
146 MachineBasicBlock::iterator LowerBound = FirstMI.getIterator();
147 unsigned LoopCnt = 0, UpperBoundCnt = UINT_MAX, LowerBoundCnt = 0;
148
149 MachineBasicBlock &MBB = *Lo.getParent();
150
151 // Check that between Lo and Hi, there are no instructions that:
152 // - modify Dst32
153 // - modify LoSrc16 or HiSrc16 depending on order (data dependency)
154 // We scan from the instruction after the first mov up to (but not including)
155 // the second mov.
156 MCRegister FirstSrc16 = IsHiFirst ? HiSrc16 : LoSrc16;
157 MCRegister FirstDst16 = IsHiFirst ? HiDst : LoDst;
158 MCRegister SecondSrc16 = IsHiFirst ? LoSrc16 : HiSrc16;
159 MCRegister SecondDst16 = IsHiFirst ? LoDst : HiDst;
160
161 // The merged instruction reads both sources before writing Dst32, so the
162 // second mov cannot read the half the first wrote. Forward the first mov's
163 // source into it instead.
164 if (SecondSrc16 && TRI->regsOverlap(RegA: SecondSrc16, RegB: FirstDst16)) {
165 // Overlap is always exact here; only that is forwardable.
166 if (SecondSrc16 != FirstDst16)
167 return false;
168 if (IsHiFirst) {
169 getMovB16Info(MI: FirstMI, TRI, SrcReg16&: LoSrc16, SrcIsVGPR&: LoSrcIsVGPR, SrcReg32&: LoSrc32, SrcIsHi&: LoSrcIsHi,
170 SrcIsImm&: LoSrcIsImm, ImmVal&: LoImm);
171 } else {
172 getMovB16Info(MI: FirstMI, TRI, SrcReg16&: HiSrc16, SrcIsVGPR&: HiSrcIsVGPR, SrcReg32&: HiSrc32, SrcIsHi&: HiSrcIsHi,
173 SrcIsImm&: HiSrcIsImm, ImmVal&: HiImm);
174 }
175 }
176
177 for (MachineInstr &Scan :
178 drop_begin(RangeOrContainer: make_range(x: FirstMI.getIterator(), y: SecondMI.getIterator()))) {
179 if (Scan.isDebugInstr())
180 continue;
181 if (Scan.modifiesRegister(Reg: Dst32, TRI))
182 return false;
183 LoopCnt++;
184 if (LoopCnt < UpperBoundCnt &&
185 ((FirstSrc16 && Scan.modifiesRegister(Reg: FirstSrc16, TRI)) ||
186 Scan.readsRegister(Reg: FirstDst16, TRI))) {
187 UpperBound = Scan.getIterator();
188 UpperBoundCnt = LoopCnt;
189 }
190 if (LoopCnt > LowerBoundCnt &&
191 ((SecondSrc16 && Scan.modifiesRegister(Reg: SecondSrc16, TRI)) ||
192 Scan.readsRegister(Reg: SecondDst16, TRI))) {
193 LowerBound = Scan.getIterator();
194 LowerBoundCnt = LoopCnt;
195 }
196 }
197
198 // No spot maintains data dependency
199 if (LowerBoundCnt >= UpperBoundCnt)
200 return false;
201
202 // Insert MI before selected. Any spots between (LowerBound, UpperBound] would
203 // work
204 MachineInstr &Selected = *UpperBound;
205 const DebugLoc &DL = Selected.getDebugLoc();
206
207 // Now match patterns and emit the replacement instruction.
208 // Insert on Selected MI location, then remove both mov.
209
210 // Pattern: v_mov_b16 v0.l, v2.x/s2 + v_mov_b16 v0.h, v3.y/s3
211 // => v_perm_b32_e64 v0,v3.y/s3,v2.x/s2, mask
212 if (!HiSrcIsImm && !LoSrcIsImm) {
213 // Violate constant bus restriction
214 if (!LoSrcIsVGPR && !HiSrcIsVGPR && HiSrc32 != LoSrc32)
215 return false;
216 unsigned MaskHiSrc = HiSrcIsHi ? 0x0706 : 0x0504;
217 unsigned MaskLoSrc = LoSrcIsHi ? 0x0302 : 0x0100;
218 BuildMI(BB&: MBB, I&: Selected, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_PERM_B32_e64), DestReg: Dst32)
219 .addReg(RegNo: HiSrc32)
220 .addReg(RegNo: LoSrc32)
221 .addImm(Val: (MaskHiSrc << 16) | MaskLoSrc);
222 Lo.eraseFromParent();
223 Hi.eraseFromParent();
224 return true;
225 }
226
227 bool Usevop2 =
228 AMDGPU::VGPR_32_Lo128RegClass.contains(Reg: Dst32) &&
229 (LoSrcIsImm ||
230 (LoSrcIsVGPR && AMDGPU::VGPR_32_Lo128RegClass.contains(Reg: LoSrc32))) &&
231 (HiSrcIsImm ||
232 (HiSrcIsVGPR && AMDGPU::VGPR_32_Lo128RegClass.contains(Reg: HiSrc32)));
233
234 // Pattern: v_mov_b16 v0.l, imm0 + v_mov_b16 v0.h, imm1
235 // => v_mov_b32 v0, (imm1 << 16) | imm0
236 if (LoSrcIsImm && HiSrcIsImm) {
237 BuildMI(BB&: MBB, I&: Selected, MIMD: DL,
238 MCID: TII->get(Opcode: Usevop2 ? AMDGPU::V_MOV_B32_e32 : AMDGPU::V_MOV_B32_e64),
239 DestReg: Dst32)
240 .addImm(Val: ((HiImm & 0xffff) << 16) | (LoImm & 0xffff));
241 Lo.eraseFromParent();
242 Hi.eraseFromParent();
243 return true;
244 }
245
246 // Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.l/s2
247 // => v_and_b32 v0, 0x0000ffff, v2/s2
248 if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && !LoSrcIsHi) {
249 BuildMI(BB&: MBB, I&: Selected, MIMD: DL,
250 MCID: TII->get(Opcode: Usevop2 ? AMDGPU::V_AND_B32_e32 : AMDGPU::V_AND_B32_e64),
251 DestReg: Dst32)
252 .addImm(Val: 0x0000ffff)
253 .addReg(RegNo: LoSrc32);
254 Lo.eraseFromParent();
255 Hi.eraseFromParent();
256 return true;
257 }
258
259 // Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.h
260 // => v_lshrrev_b32 v0, 16, v2
261 if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && LoSrcIsHi) {
262 BuildMI(BB&: MBB, I&: Selected, MIMD: DL,
263 MCID: TII->get(Opcode: Usevop2 ? AMDGPU::V_LSHRREV_B32_e32
264 : AMDGPU::V_LSHRREV_B32_e64),
265 DestReg: Dst32)
266 .addImm(Val: 16)
267 .addReg(RegNo: LoSrc32);
268 Lo.eraseFromParent();
269 Hi.eraseFromParent();
270 return true;
271 }
272
273 // Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.l/s2
274 // => v_lshlrev_b32 v0, 16, v2/s2
275 if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && !HiSrcIsHi) {
276 BuildMI(BB&: MBB, I&: Selected, MIMD: DL,
277 MCID: TII->get(Opcode: Usevop2 ? AMDGPU::V_LSHLREV_B32_e32
278 : AMDGPU::V_LSHLREV_B32_e64),
279 DestReg: Dst32)
280 .addImm(Val: 16)
281 .addReg(RegNo: HiSrc32);
282 Lo.eraseFromParent();
283 Hi.eraseFromParent();
284 return true;
285 }
286
287 // Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.h
288 // => v_and_b32 v0, 0xffff0000, v2
289 if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && HiSrcIsHi) {
290 BuildMI(BB&: MBB, I&: Selected, MIMD: DL,
291 MCID: TII->get(Opcode: Usevop2 ? AMDGPU::V_AND_B32_e32 : AMDGPU::V_AND_B32_e64),
292 DestReg: Dst32)
293 .addImm(Val: 0xffff0000)
294 .addReg(RegNo: HiSrc32);
295 Lo.eraseFromParent();
296 Hi.eraseFromParent();
297 return true;
298 }
299
300 return false;
301}
302
303// Merge pairs of v_mov_b16 targeting the lo16 and hi16 halves of the same
304// VGPR into a single 32-bit instruction (true16 mode only).
305bool SIPostRA16BitMovFolding::mergeMovB16Pairs(MachineFunction &MF) const {
306 bool Changed = false;
307 for (MachineBasicBlock &MBB : MF) {
308 // Map from 32-bit VGPR to the pending v_mov_b16 and its age.
309 // Tracks how many non-mov-b16 instructions have passed since the
310 // 16-bit write using a fixed size circular buffer
311 struct Pending {
312 MCRegister Dst32;
313 MachineInstr *MI;
314 unsigned IsHi;
315 };
316 // Search window size
317 const unsigned ScanLimit = 16;
318 std::array<Pending, ScanLimit> CirBuf = {};
319 SmallDenseMap<MCRegister, unsigned> PendingWrites;
320 unsigned Head = 0;
321
322 for (MachineInstr &MI : make_early_inc_range(Range&: MBB)) {
323 if (MI.isDebugInstr())
324 continue;
325
326 unsigned Opc = MI.getOpcode();
327 bool IsMovB16 = (Opc == AMDGPU::V_MOV_B16_t16_e32 ||
328 Opc == AMDGPU::V_MOV_B16_t16_e64);
329
330 if (++Head == ScanLimit)
331 Head = 0;
332
333 // Expire the last one
334 PendingWrites.erase(Val: CirBuf[Head].Dst32);
335
336 if (!IsMovB16) {
337 CirBuf[Head] = {.Dst32: MCRegister(), .MI: nullptr, .IsHi: false};
338 continue;
339 }
340
341 LLVM_DEBUG(dbgs() << "Checking MI:" << MI << "\n");
342 MCRegister DstReg = MI.getOperand(i: 0).getReg().asMCReg();
343 bool DstIsHi = AMDGPU::isHi16Reg(Reg: DstReg, MRI: *TRI);
344 MCRegister Dst32 = TRI->get32BitRegister(Reg: DstReg);
345
346 // Insert new one
347 CirBuf[Head] = {.Dst32: Dst32, .MI: &MI, .IsHi: DstIsHi};
348
349 auto [It, Inserted] = PendingWrites.insert(KV: {Dst32, Head});
350 if (!Inserted) {
351 if (CirBuf[It->second].IsHi == DstIsHi) {
352 It->second = Head;
353 continue;
354 }
355
356 // Look for a matching pending write.
357 MachineInstr &LoMI = !DstIsHi ? MI : *CirBuf[It->second].MI;
358 MachineInstr &HiMI = DstIsHi ? MI : *CirBuf[It->second].MI;
359 bool IsHiFirst = CirBuf[It->second].IsHi;
360 if (mergeSingleMovB16Pair(Lo&: LoMI, Hi&: HiMI, IsHiFirst)) {
361 Changed = true;
362 PendingWrites.erase(I: It);
363 } else {
364 It->second = Head;
365 }
366 }
367 }
368 }
369
370 return Changed;
371}
372
373PreservedAnalyses
374llvm::SIPostRA16BitMovFoldingPass::run(MachineFunction &MF,
375 MachineFunctionAnalysisManager &MFAM) {
376 SIPostRA16BitMovFolding().run(MF);
377 return PreservedAnalyses::all();
378}
379
380bool SIPostRA16BitMovFolding::run(MachineFunction &MF) {
381 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
382 TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
383 TII = ST.getInstrInfo();
384 bool Changed = false;
385
386 // Try merge B16 Pair in true16 mode
387 if (ST.useRealTrue16Insts())
388 Changed |= mergeMovB16Pairs(MF);
389
390 return Changed;
391}
392