1//===-- SIPostRA16BitMovFolding.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass performs the post RA 16bit Mov folding
11///
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "GCNSubtarget.h"
16#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/CodeGen/MachineDominators.h"
19#include "llvm/CodeGen/MachineFunctionPass.h"
20#include "llvm/CodeGen/MachineLoopInfo.h"
21#include "llvm/CodeGen/MachinePostDominators.h"
22#include "llvm/CodeGen/TargetSchedule.h"
23#include "llvm/Support/BranchProbability.h"
24using namespace llvm;
25
26#define DEBUG_TYPE "si-post-ra-16bit-mov-folding"
27
28namespace {
29
30class SIPostRA16BitMovFolding {
31private:
32 const SIInstrInfo *TII = nullptr;
33 const SIRegisterInfo *TRI = nullptr;
34
35 void getMovB16Info(const MachineInstr &MI, const SIRegisterInfo *TRI,
36 MCRegister &SrcReg16, bool &SrcIsVGPR,
37 MCRegister &SrcReg32, bool &SrcIsHi, bool &SrcIsImm,
38 int64_t &ImmVal) const;
39
40 bool mergeSingleMovB16Pair(MachineInstr &Lo, MachineInstr &Hi,
41 bool IsHiFirst) const;
42 bool mergeMovB16Pairs(MachineFunction &MF) const;
43
44public:
45 bool run(MachineFunction &MF);
46};
47
48class SIPostRA16BitMovFoldingLegacy : public MachineFunctionPass {
49public:
50 static char ID;
51
52 SIPostRA16BitMovFoldingLegacy() : MachineFunctionPass(ID) {}
53
54 StringRef getPassName() const override {
55 return "SI post-RA 16bit Mov Folding";
56 }
57
58 void getAnalysisUsage(AnalysisUsage &AU) const override {
59 AU.setPreservesAll();
60 MachineFunctionPass::getAnalysisUsage(AU);
61 }
62
63 bool runOnMachineFunction(MachineFunction &MF) override {
64 return SIPostRA16BitMovFolding().run(MF);
65 }
66};
67
68} // End anonymous namespace.
69
70INITIALIZE_PASS(SIPostRA16BitMovFoldingLegacy, DEBUG_TYPE,
71 "SI Post RA 16bit Mov Folding", false, false)
72
73char SIPostRA16BitMovFoldingLegacy::ID = 0;
74
75char &llvm::SIPostRA16BitMovFoldingLegacyID = SIPostRA16BitMovFoldingLegacy::ID;
76
77// Helper: extract the src operand and whether it is from the hi16 half.
78// Post-RA, both V_MOV_B16_t16_e32 and V_MOV_B16_t16_e64 use VGPR_16 dst
79// physical registers whose encoding already encodes hi/lo (IS_HI16 bit).
80void SIPostRA16BitMovFolding::getMovB16Info(
81 const MachineInstr &MI, const SIRegisterInfo *TRI, MCRegister &SrcReg16,
82 bool &SrcIsVGPR, MCRegister &SrcReg32, bool &SrcIsHi, bool &SrcIsImm,
83 int64_t &ImmVal) const {
84 SrcIsImm = false;
85 SrcIsHi = false;
86 SrcIsVGPR = false;
87 SrcReg16 = MCRegister();
88 SrcReg32 = MCRegister();
89
90 const MachineOperand *SrcOp = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
91
92 if (SrcOp->isImm()) {
93 SrcIsImm = true;
94 ImmVal = SrcOp->getImm();
95 return;
96 }
97
98 SrcReg16 = SrcOp->getReg().asMCReg();
99 SrcIsVGPR = AMDGPU::VGPR_16RegClass.contains(Reg: SrcReg16);
100 if (SrcIsVGPR) {
101 SrcIsHi = AMDGPU::isHi16Reg(Reg: SrcReg16, MRI: *TRI);
102 SrcReg32 = TRI->get32BitRegister(Reg: SrcReg16);
103 } else {
104 SrcIsHi = false;
105 SrcReg32 = SrcReg16;
106 }
107}
108
109// clang-format off
110// Try to merge a pair of v_mov_b16 instructions targeting the lo16 and hi16
111// halves of the same VGPR into a single 32-bit instruction.
112//
113// Caller guarantee the pair to be two v_mov_b16 and targets the same dst32
114//
115// Patterns:
116// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.l/s2 => v_and_b32 v0,0xffff,v2/s2
117// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.h => v_lshrrev_b32 v0,16,v2
118// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.l/s2 => v_lshlrev_b32 v0,16,v2/s2
119// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.h => v_and_b32 v0,0xffff0000,v2
120// v_mov_b16 v0.l, v.x/s v_mov_b16 v0.h, v.y/s => v_perm_b32_e64 v0, v.x/s, v.y/s, mask
121// v_mov_b16 v0.l, imm0 v_mov_b16 v0.h, imm1 => v_mov_b32 v0, (imm1<<16)|imm0
122//
123// A second mov reading the half the first wrote is rewritten to read the first
124// mov's source before matching.
125// clang-format on
126bool SIPostRA16BitMovFolding::mergeSingleMovB16Pair(MachineInstr &Lo,
127 MachineInstr &Hi,
128 bool IsHiFirst) const {
129 // Lo and Hi share the same Dst32
130 MCRegister LoDst = Lo.getOperand(i: 0).getReg().asMCReg();
131 MCRegister HiDst = Hi.getOperand(i: 0).getReg().asMCReg();
132 MCRegister Dst32 = TRI->get32BitRegister(Reg: LoDst);
133
134 // Extract source info for Lo and Hi.
135 MCRegister LoSrc16, LoSrc32, HiSrc16, HiSrc32;
136 bool LoSrcIsHi, HiSrcIsHi, LoSrcIsImm, HiSrcIsImm, LoSrcIsVGPR, HiSrcIsVGPR;
137 int64_t LoImm = 0, HiImm = 0;
138
139 getMovB16Info(MI: Lo, TRI, SrcReg16&: LoSrc16, SrcIsVGPR&: LoSrcIsVGPR, SrcReg32&: LoSrc32, SrcIsHi&: LoSrcIsHi, SrcIsImm&: LoSrcIsImm,
140 ImmVal&: LoImm);
141 getMovB16Info(MI: Hi, TRI, SrcReg16&: HiSrc16, SrcIsVGPR&: HiSrcIsVGPR, SrcReg32&: HiSrc32, SrcIsHi&: HiSrcIsHi, SrcIsImm&: HiSrcIsImm,
142 ImmVal&: HiImm);
143
144 MachineInstr &FirstMI = IsHiFirst ? Hi : Lo;
145 MachineInstr &SecondMI = IsHiFirst ? Lo : Hi;
146
147 // Data Conflict counter
148 MachineBasicBlock::iterator UpperBound = SecondMI.getIterator();
149 MachineBasicBlock::iterator LowerBound = FirstMI.getIterator();
150 unsigned LoopCnt = 0, UpperBoundCnt = UINT_MAX, LowerBoundCnt = 0;
151
152 MachineBasicBlock &MBB = *Lo.getParent();
153
154 // Check that between Lo and Hi, there are no instructions that:
155 // - modify Dst32
156 // - modify LoSrc16 or HiSrc16 depending on order (data dependency)
157 // We scan from the instruction after the first mov up to (but not including)
158 // the second mov.
159 MCRegister FirstSrc16 = IsHiFirst ? HiSrc16 : LoSrc16;
160 MCRegister FirstDst16 = IsHiFirst ? HiDst : LoDst;
161 MCRegister SecondSrc16 = IsHiFirst ? LoSrc16 : HiSrc16;
162 MCRegister SecondDst16 = IsHiFirst ? LoDst : HiDst;
163
164 // The merged instruction reads both sources before writing Dst32, so the
165 // second mov cannot read the half the first wrote. Forward the first mov's
166 // source into it instead.
167 if (SecondSrc16 && TRI->regsOverlap(RegA: SecondSrc16, RegB: FirstDst16)) {
168 // Overlap is always exact here; only that is forwardable.
169 if (SecondSrc16 != FirstDst16)
170 return false;
171 if (IsHiFirst) {
172 getMovB16Info(MI: FirstMI, TRI, SrcReg16&: LoSrc16, SrcIsVGPR&: LoSrcIsVGPR, SrcReg32&: LoSrc32, SrcIsHi&: LoSrcIsHi,
173 SrcIsImm&: LoSrcIsImm, ImmVal&: LoImm);
174 } else {
175 getMovB16Info(MI: FirstMI, TRI, SrcReg16&: HiSrc16, SrcIsVGPR&: HiSrcIsVGPR, SrcReg32&: HiSrc32, SrcIsHi&: HiSrcIsHi,
176 SrcIsImm&: HiSrcIsImm, ImmVal&: HiImm);
177 }
178 }
179
180 for (MachineInstr &Scan :
181 drop_begin(RangeOrContainer: make_range(x: FirstMI.getIterator(), y: SecondMI.getIterator()))) {
182 if (Scan.isDebugInstr())
183 continue;
184 if (Scan.modifiesRegister(Reg: Dst32, TRI))
185 return false;
186 LoopCnt++;
187 if (LoopCnt < UpperBoundCnt &&
188 ((FirstSrc16 && Scan.modifiesRegister(Reg: FirstSrc16, TRI)) ||
189 Scan.readsRegister(Reg: FirstDst16, TRI))) {
190 UpperBound = Scan.getIterator();
191 UpperBoundCnt = LoopCnt;
192 }
193 if (LoopCnt > LowerBoundCnt &&
194 ((SecondSrc16 && Scan.modifiesRegister(Reg: SecondSrc16, TRI)) ||
195 Scan.readsRegister(Reg: SecondDst16, TRI))) {
196 LowerBound = Scan.getIterator();
197 LowerBoundCnt = LoopCnt;
198 }
199 }
200
201 // No spot maintains data dependency
202 if (LowerBoundCnt >= UpperBoundCnt)
203 return false;
204
205 // Insert MI before selected. Any spots between (LowerBound, UpperBound] would
206 // work
207 MachineInstr &Selected = *UpperBound;
208 const DebugLoc &DL = Selected.getDebugLoc();
209
210 // Now match patterns and emit the replacement instruction.
211 // Insert on Selected MI location, then remove both mov.
212
213 // Pattern: v_mov_b16 v0.l, v2.x/s2 + v_mov_b16 v0.h, v3.y/s3
214 // => v_perm_b32_e64 v0,v3.y/s3,v2.x/s2, mask
215 if (!HiSrcIsImm && !LoSrcIsImm) {
216 // Violate constant bus restriction
217 if (!LoSrcIsVGPR && !HiSrcIsVGPR && HiSrc32 != LoSrc32)
218 return false;
219 unsigned MaskHiSrc = HiSrcIsHi ? 0x0706 : 0x0504;
220 unsigned MaskLoSrc = LoSrcIsHi ? 0x0302 : 0x0100;
221 BuildMI(BB&: MBB, I&: Selected, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_PERM_B32_e64), DestReg: Dst32)
222 .addReg(RegNo: HiSrc32)
223 .addReg(RegNo: LoSrc32)
224 .addImm(Val: (MaskHiSrc << 16) | MaskLoSrc);
225 Lo.eraseFromParent();
226 Hi.eraseFromParent();
227 return true;
228 }
229
230 bool Usevop2 =
231 AMDGPU::VGPR_32_Lo128RegClass.contains(Reg: Dst32) &&
232 (LoSrcIsImm ||
233 (LoSrcIsVGPR && AMDGPU::VGPR_32_Lo128RegClass.contains(Reg: LoSrc32))) &&
234 (HiSrcIsImm ||
235 (HiSrcIsVGPR && AMDGPU::VGPR_32_Lo128RegClass.contains(Reg: HiSrc32)));
236
237 // Pattern: v_mov_b16 v0.l, imm0 + v_mov_b16 v0.h, imm1
238 // => v_mov_b32 v0, (imm1 << 16) | imm0
239 if (LoSrcIsImm && HiSrcIsImm) {
240 BuildMI(BB&: MBB, I&: Selected, MIMD: DL,
241 MCID: TII->get(Opcode: Usevop2 ? AMDGPU::V_MOV_B32_e32 : AMDGPU::V_MOV_B32_e64),
242 DestReg: Dst32)
243 .addImm(Val: ((HiImm & 0xffff) << 16) | (LoImm & 0xffff));
244 Lo.eraseFromParent();
245 Hi.eraseFromParent();
246 return true;
247 }
248
249 // Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.l/s2
250 // => v_and_b32 v0, 0x0000ffff, v2/s2
251 if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && !LoSrcIsHi) {
252 BuildMI(BB&: MBB, I&: Selected, MIMD: DL,
253 MCID: TII->get(Opcode: Usevop2 ? AMDGPU::V_AND_B32_e32 : AMDGPU::V_AND_B32_e64),
254 DestReg: Dst32)
255 .addImm(Val: 0x0000ffff)
256 .addReg(RegNo: LoSrc32);
257 Lo.eraseFromParent();
258 Hi.eraseFromParent();
259 return true;
260 }
261
262 // Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.h
263 // => v_lshrrev_b32 v0, 16, v2
264 if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && LoSrcIsHi) {
265 BuildMI(BB&: MBB, I&: Selected, MIMD: DL,
266 MCID: TII->get(Opcode: Usevop2 ? AMDGPU::V_LSHRREV_B32_e32
267 : AMDGPU::V_LSHRREV_B32_e64),
268 DestReg: Dst32)
269 .addImm(Val: 16)
270 .addReg(RegNo: LoSrc32);
271 Lo.eraseFromParent();
272 Hi.eraseFromParent();
273 return true;
274 }
275
276 // Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.l/s2
277 // => v_lshlrev_b32 v0, 16, v2/s2
278 if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && !HiSrcIsHi) {
279 BuildMI(BB&: MBB, I&: Selected, MIMD: DL,
280 MCID: TII->get(Opcode: Usevop2 ? AMDGPU::V_LSHLREV_B32_e32
281 : AMDGPU::V_LSHLREV_B32_e64),
282 DestReg: Dst32)
283 .addImm(Val: 16)
284 .addReg(RegNo: HiSrc32);
285 Lo.eraseFromParent();
286 Hi.eraseFromParent();
287 return true;
288 }
289
290 // Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.h
291 // => v_and_b32 v0, 0xffff0000, v2
292 if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && HiSrcIsHi) {
293 BuildMI(BB&: MBB, I&: Selected, MIMD: DL,
294 MCID: TII->get(Opcode: Usevop2 ? AMDGPU::V_AND_B32_e32 : AMDGPU::V_AND_B32_e64),
295 DestReg: Dst32)
296 .addImm(Val: 0xffff0000)
297 .addReg(RegNo: HiSrc32);
298 Lo.eraseFromParent();
299 Hi.eraseFromParent();
300 return true;
301 }
302
303 return false;
304}
305
306// Merge pairs of v_mov_b16 targeting the lo16 and hi16 halves of the same
307// VGPR into a single 32-bit instruction (true16 mode only).
308bool SIPostRA16BitMovFolding::mergeMovB16Pairs(MachineFunction &MF) const {
309 bool Changed = false;
310 for (MachineBasicBlock &MBB : MF) {
311 // Map from 32-bit VGPR to the pending v_mov_b16 and its age.
312 // Tracks how many non-mov-b16 instructions have passed since the
313 // 16-bit write using a fixed size circular buffer
314 struct Pending {
315 MCRegister Dst32;
316 MachineInstr *MI;
317 unsigned IsHi;
318 };
319 // Search window size
320 const unsigned ScanLimit = 16;
321 std::array<Pending, ScanLimit> CirBuf = {};
322 SmallDenseMap<MCRegister, unsigned> PendingWrites;
323 unsigned Head = 0;
324
325 for (MachineInstr &MI : make_early_inc_range(Range&: MBB)) {
326 if (MI.isDebugInstr())
327 continue;
328
329 unsigned Opc = MI.getOpcode();
330 bool IsMovB16 = (Opc == AMDGPU::V_MOV_B16_t16_e32 ||
331 Opc == AMDGPU::V_MOV_B16_t16_e64);
332
333 if (++Head == ScanLimit)
334 Head = 0;
335
336 // Expire the last one
337 PendingWrites.erase(Val: CirBuf[Head].Dst32);
338
339 if (!IsMovB16) {
340 CirBuf[Head] = {.Dst32: MCRegister(), .MI: nullptr, .IsHi: false};
341 continue;
342 }
343
344 LLVM_DEBUG(dbgs() << "Checking MI:" << MI << "\n");
345 MCRegister DstReg = MI.getOperand(i: 0).getReg().asMCReg();
346 bool DstIsHi = AMDGPU::isHi16Reg(Reg: DstReg, MRI: *TRI);
347 MCRegister Dst32 = TRI->get32BitRegister(Reg: DstReg);
348
349 // Insert new one
350 CirBuf[Head] = {.Dst32: Dst32, .MI: &MI, .IsHi: DstIsHi};
351
352 auto [It, Inserted] = PendingWrites.insert(KV: {Dst32, Head});
353 if (!Inserted) {
354 if (CirBuf[It->second].IsHi == DstIsHi) {
355 It->second = Head;
356 continue;
357 }
358
359 // Look for a matching pending write.
360 MachineInstr &LoMI = !DstIsHi ? MI : *CirBuf[It->second].MI;
361 MachineInstr &HiMI = DstIsHi ? MI : *CirBuf[It->second].MI;
362 bool IsHiFirst = CirBuf[It->second].IsHi;
363 if (mergeSingleMovB16Pair(Lo&: LoMI, Hi&: HiMI, IsHiFirst)) {
364 Changed = true;
365 PendingWrites.erase(I: It);
366 } else {
367 It->second = Head;
368 }
369 }
370 }
371 }
372
373 return Changed;
374}
375
376PreservedAnalyses
377llvm::SIPostRA16BitMovFoldingPass::run(MachineFunction &MF,
378 MachineFunctionAnalysisManager &MFAM) {
379 SIPostRA16BitMovFolding().run(MF);
380 return PreservedAnalyses::all();
381}
382
383bool SIPostRA16BitMovFolding::run(MachineFunction &MF) {
384 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
385 TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
386 TII = ST.getInstrInfo();
387 bool Changed = false;
388
389 // Try merge B16 Pair in true16 mode
390 if (ST.useRealTrue16Insts())
391 Changed |= mergeMovB16Pairs(MF);
392
393 return Changed;
394}
395