1//===-- SIPreEmitPeephole.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass performs the peephole optimizations before code emission.
11///
12/// Additionally, this pass also unpacks packed instructions (V_PK_MUL_F32/F16,
13/// V_PK_ADD_F32/F16, V_PK_FMA_F32) adjacent to MFMAs such that they can be
14/// co-issued. This helps with overlapping MFMA and certain vector instructions
15/// in machine schedules and is expected to improve performance. Only those
16/// packed instructions are unpacked that are overlapped by the MFMA latency.
17/// Rest should remain untouched.
18/// TODO: Add support for F16 packed instructions
19//===----------------------------------------------------------------------===//
20
21#include "AMDGPU.h"
22#include "GCNSubtarget.h"
23#include "llvm/ADT/Statistic.h"
24#include "llvm/CodeGen/MachineDominators.h"
25#include "llvm/CodeGen/MachineFunctionPass.h"
26#include "llvm/CodeGen/MachineLoopInfo.h"
27#include "llvm/CodeGen/TargetSchedule.h"
28#include "llvm/Support/BranchProbability.h"
29using namespace llvm;
30
31#define DEBUG_TYPE "si-pre-emit-peephole"
32
33STATISTIC(NumModeWritesRemoved,
34 "Number of redundant mode register writes removed");
35
36namespace {
37
38/// The state of one independent field of the MODE register, as tracked by
39/// removeRedundantModeWrites.
40struct ModeFieldState {
41 std::optional<int64_t> Value;
42 std::optional<int64_t> ValueBeforePendingWrite;
43 MachineInstr *PendingWrite = nullptr;
44
45 bool isTracked() const { return PendingWrite || Value; }
46};
47
48class SIPreEmitPeephole {
49private:
50 const SIInstrInfo *TII = nullptr;
51 const SIRegisterInfo *TRI = nullptr;
52 MachineLoopInfo *MLI = nullptr;
53
54 std::optional<int64_t> matchVccMask(MachineInstr &A, Register &SReg,
55 bool &Changed) const;
56 bool optimizeVccBranch(MachineInstr &MI) const;
57 void updateMLIBeforeRemovingEdge(MachineBasicBlock *From,
58 MachineBasicBlock *To) const;
59 bool optimizeSetGPR(MachineInstr &First, MachineInstr &MI) const;
60 bool getBlockDestinations(MachineBasicBlock &SrcMBB,
61 MachineBasicBlock *&TrueMBB,
62 MachineBasicBlock *&FalseMBB,
63 SmallVectorImpl<MachineOperand> &Cond);
64 bool mustRetainExeczBranch(const MachineInstr &Branch,
65 const MachineBasicBlock &From,
66 const MachineBasicBlock &To) const;
67 bool removeExeczBranch(MachineInstr &MI, MachineBasicBlock &SrcMBB);
68 bool removeRedundantModeWrites(MachineBasicBlock &SrcMBB) const;
69 // Creates a list of packed instructions following an MFMA that are suitable
70 // for unpacking.
71 void collectUnpackingCandidates(MachineInstr &BeginMI,
72 SetVector<MachineInstr *> &InstrsToUnpack,
73 uint16_t NumMFMACycles);
74 // v_pk_fma_f32 v[0:1], v[0:1], v[2:3], v[2:3] op_sel:[1,1,1]
75 // op_sel_hi:[0,0,0]
76 // ==>
77 // v_fma_f32 v0, v1, v3, v3
78 // v_fma_f32 v1, v0, v2, v2
79 // Here, we have overwritten v0 before we use it. This function checks if
80 // unpacking can lead to such a situation.
81 bool canUnpackingClobberRegister(const MachineInstr &MI);
82 // Unpack and insert F32 packed instructions, such as V_PK_MUL, V_PK_ADD, and
83 // V_PK_FMA. Currently, only V_PK_MUL, V_PK_ADD, V_PK_FMA are supported for
84 // this transformation.
85 void performF32Unpacking(MachineInstr &I);
86 // Select corresponding unpacked instruction
87 uint32_t mapToUnpackedOpcode(MachineInstr &I);
88 // Creates the unpacked instruction to be inserted. Adds source modifiers to
89 // the unpacked instructions based on the source modifiers in the packed
90 // instruction.
91 MachineInstrBuilder createUnpackedMI(MachineInstr &I, uint32_t UnpackedOpcode,
92 bool IsHiBits);
93 // Process operands/source modifiers from packed instructions and insert the
94 // appropriate source modifers and operands into the unpacked instructions.
95 void addOperandAndMods(MachineInstrBuilder &NewMI, unsigned SrcMods,
96 bool IsHiBits, const MachineOperand &SrcMO);
97
98public:
99 bool run(MachineFunction &MF, MachineLoopInfo *MLI);
100};
101
102class SIPreEmitPeepholeLegacy : public MachineFunctionPass {
103public:
104 static char ID;
105
106 SIPreEmitPeepholeLegacy() : MachineFunctionPass(ID) {}
107
108 void getAnalysisUsage(AnalysisUsage &AU) const override {
109 AU.addUsedIfAvailable<MachineLoopInfoWrapperPass>();
110 AU.addPreserved<MachineLoopInfoWrapperPass>();
111 MachineFunctionPass::getAnalysisUsage(AU);
112 }
113
114 bool runOnMachineFunction(MachineFunction &MF) override {
115 auto *MLIWrapper = getAnalysisIfAvailable<MachineLoopInfoWrapperPass>();
116 MachineLoopInfo *MLI = MLIWrapper ? &MLIWrapper->getLI() : nullptr;
117 return SIPreEmitPeephole().run(MF, MLI);
118 }
119};
120
121} // End anonymous namespace.
122
123INITIALIZE_PASS(SIPreEmitPeepholeLegacy, DEBUG_TYPE,
124 "SI peephole optimizations", false, false)
125
126char SIPreEmitPeepholeLegacy::ID = 0;
127
128char &llvm::SIPreEmitPeepholeID = SIPreEmitPeepholeLegacy::ID;
129
130void SIPreEmitPeephole::updateMLIBeforeRemovingEdge(
131 MachineBasicBlock *From, MachineBasicBlock *To) const {
132 if (!MLI)
133 return;
134
135 // Only handle back-edges: To must be a loop header with From inside the loop.
136 MachineLoop *Loop = MLI->getLoopFor(BB: To);
137 if (!Loop || Loop->getHeader() != To || !Loop->contains(BB: From))
138 return;
139
140 // Count back-edges
141 unsigned BackEdgeCount = 0;
142 for (MachineBasicBlock *Pred : To->predecessors()) {
143 if (Loop->contains(BB: Pred))
144 BackEdgeCount++;
145 }
146
147 if (BackEdgeCount > 1)
148 return;
149
150 MachineLoop *ParentLoop = Loop->getParentLoop();
151
152 // Re-map blocks directly owned by this loop to the parent.
153 for (MachineBasicBlock *BB : Loop->blocks()) {
154 if (MLI->getLoopFor(BB) == Loop)
155 MLI->changeLoopFor(BB, L: ParentLoop);
156 }
157
158 // Reparent all child loops.
159 while (!Loop->isInnermost()) {
160 MachineLoop *Child = Loop->removeChildLoop(I: std::prev(x: Loop->end()));
161 if (ParentLoop)
162 ParentLoop->addChildLoop(NewChild: Child);
163 else
164 MLI->addTopLevelLoop(New: Child);
165 }
166
167 if (ParentLoop)
168 ParentLoop->removeChildLoop(Child: Loop);
169 else
170 MLI->removeLoop(I: llvm::find(Range&: *MLI, Val: Loop));
171
172 MLI->destroy(L: Loop);
173}
174
175// Match the instruction A that defines vcc for optimizeVccBranch. Returns 0 if
176// vcc == 0, -1 if vcc == exec, or std::nullopt otherwise. SReg is set to the
177// mask register when A is an S_AND/S_ANDN2 of exec with a register.
178std::optional<int64_t> SIPreEmitPeephole::matchVccMask(MachineInstr &A,
179 Register &SReg,
180 bool &Changed) const {
181 const GCNSubtarget &ST = A.getMF()->getSubtarget<GCNSubtarget>();
182 const bool IsWave32 = ST.isWave32();
183 const unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
184 const unsigned AndN2 = IsWave32 ? AMDGPU::S_ANDN2_B32 : AMDGPU::S_ANDN2_B64;
185 const unsigned Mov = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
186
187 const unsigned Opc = A.getOpcode();
188 if (Opc == Mov || Opc == AMDGPU::COPY) {
189 const MachineOperand &Src = A.getOperand(i: 1);
190 if (Src.isImm() && Src.getImm() == 0)
191 return 0;
192 if (Src.isReg() && Src.getReg() == ExecReg)
193 return -1;
194 return std::nullopt;
195 }
196
197 MachineOperand &Op1 = A.getOperand(i: 1);
198 MachineOperand &Op2 = A.getOperand(i: 2);
199 if ((!Op1.isReg() || Op1.getReg() != ExecReg) && Op2.isReg() &&
200 Op2.getReg() == ExecReg) {
201 TII->commuteInstruction(MI&: A);
202 Changed = true;
203 }
204 if (!Op1.isReg() || Op1.getReg() != ExecReg)
205 return std::nullopt;
206 if (Op2.isImm() && !(Op2.getImm() == -1 || Op2.getImm() == 0))
207 return std::nullopt;
208
209 int64_t MaskValue = 0;
210 if (Op2.isReg()) {
211 SReg = Op2.getReg();
212 MachineBasicBlock::reverse_iterator M =
213 std::next(x: MachineBasicBlock::reverse_iterator(A));
214 MachineBasicBlock::reverse_iterator E = A.getParent()->rend();
215 bool ReadsSreg = false;
216 for (; M != E; ++M) {
217 if (M->definesRegister(Reg: SReg, TRI))
218 break;
219 if (M->modifiesRegister(Reg: SReg, TRI))
220 return std::nullopt;
221 ReadsSreg |= M->readsRegister(Reg: SReg, TRI);
222 }
223 if (M == E)
224 return std::nullopt;
225
226 if (!M->isMoveImmediate() || !M->getOperand(i: 1).isImm() ||
227 (M->getOperand(i: 1).getImm() != -1 && M->getOperand(i: 1).getImm() != 0))
228 return std::nullopt;
229 MaskValue = M->getOperand(i: 1).getImm();
230 // First if sreg is only used in the AND instruction fold the immediate
231 // into the AND.
232 if (!ReadsSreg && Op2.isKill()) {
233 A.getOperand(i: 2).ChangeToImmediate(ImmVal: MaskValue);
234 M->eraseFromParent();
235 }
236 } else if (Op2.isImm()) {
237 MaskValue = Op2.getImm();
238 } else {
239 llvm_unreachable("Op2 must be register or immediate");
240 }
241
242 // Invert mask for s_andn2
243 assert(MaskValue == 0 || MaskValue == -1);
244 if (Opc == AndN2)
245 MaskValue = ~MaskValue;
246 return MaskValue;
247}
248
249bool SIPreEmitPeephole::optimizeVccBranch(MachineInstr &MI) const {
250 // Match:
251 // vcc = S_AND exec, 0/-1 (or S_ANDN2) / S_MOV 0 / S_MOV|COPY exec
252 // S_CBRANCH_VCC[N]Z
253 // =>
254 // S_CBRANCH_EXEC[N]Z
255
256 bool Changed = false;
257 MachineBasicBlock &MBB = *MI.getParent();
258 const GCNSubtarget &ST = MBB.getParent()->getSubtarget<GCNSubtarget>();
259 const bool IsWave32 = ST.isWave32();
260 const unsigned CondReg = TRI->getVCC();
261 const unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
262 const unsigned And = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
263 const unsigned AndN2 = IsWave32 ? AMDGPU::S_ANDN2_B32 : AMDGPU::S_ANDN2_B64;
264 const unsigned Mov = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
265
266 MachineBasicBlock::reverse_iterator A = MI.getReverseIterator(),
267 E = MBB.rend();
268 bool ReadsCond = false;
269 unsigned Threshold = 5;
270 for (++A; A != E; ++A) {
271 if (!--Threshold)
272 return false;
273 if (A->modifiesRegister(Reg: ExecReg, TRI))
274 return false;
275 if (A->modifiesRegister(Reg: CondReg, TRI)) {
276 unsigned Opc = A->getOpcode();
277 if (!A->definesRegister(Reg: CondReg, TRI) ||
278 (Opc != And && Opc != AndN2 && Opc != Mov && Opc != AMDGPU::COPY))
279 return false;
280 break;
281 }
282 ReadsCond |= A->readsRegister(Reg: CondReg, TRI);
283 }
284 if (A == E)
285 return false;
286
287 Register SReg;
288 std::optional<int64_t> Match = matchVccMask(A&: *A, SReg, Changed);
289 if (!Match)
290 return Changed;
291 int64_t MaskValue = *Match;
292
293 const bool IsAnd = A->getOpcode() == And || A->getOpcode() == AndN2;
294 if (!ReadsCond &&
295 (!IsAnd || A->registerDefIsDead(Reg: AMDGPU::SCC, /*TRI=*/nullptr))) {
296 const bool KillsCond = MI.killsRegister(Reg: CondReg, TRI);
297 if (IsAnd && !KillsCond) {
298 // Replace AND with MOV
299 if (MaskValue == 0) {
300 BuildMI(BB&: *A->getParent(), I&: *A, MIMD: A->getDebugLoc(), MCID: TII->get(Opcode: Mov), DestReg: CondReg)
301 .addImm(Val: 0);
302 } else {
303 BuildMI(BB&: *A->getParent(), I&: *A, MIMD: A->getDebugLoc(), MCID: TII->get(Opcode: Mov), DestReg: CondReg)
304 .addReg(RegNo: ExecReg);
305 }
306 }
307 if (IsAnd || KillsCond)
308 A->eraseFromParent();
309 }
310
311 bool IsVCCZ = MI.getOpcode() == AMDGPU::S_CBRANCH_VCCZ;
312 if (SReg == ExecReg) {
313 // EXEC is updated directly
314 if (IsVCCZ) {
315 MI.eraseFromParent();
316 return true;
317 }
318 MI.setDesc(TII->get(Opcode: AMDGPU::S_BRANCH));
319 } else if (IsVCCZ && MaskValue == 0) {
320 // Will always branch
321 // Remove all successors shadowed by new unconditional branch
322 MachineBasicBlock *Parent = MI.getParent();
323 SmallVector<MachineInstr *, 4> ToRemove;
324 bool Found = false;
325 for (MachineInstr &Term : Parent->terminators()) {
326 if (Found) {
327 if (Term.isBranch())
328 ToRemove.push_back(Elt: &Term);
329 } else {
330 Found = Term.isIdenticalTo(Other: MI);
331 }
332 }
333 assert(Found && "conditional branch is not terminator");
334 for (auto *BranchMI : ToRemove) {
335 MachineOperand &Dst = BranchMI->getOperand(i: 0);
336 assert(Dst.isMBB() && "destination is not basic block");
337 updateMLIBeforeRemovingEdge(From: Parent, To: Dst.getMBB());
338 Parent->removeSuccessor(Succ: Dst.getMBB());
339 BranchMI->eraseFromParent();
340 }
341
342 if (MachineBasicBlock *Succ = Parent->getFallThrough()) {
343 updateMLIBeforeRemovingEdge(From: Parent, To: Succ);
344 Parent->removeSuccessor(Succ);
345 }
346
347 // Rewrite to unconditional branch
348 MI.setDesc(TII->get(Opcode: AMDGPU::S_BRANCH));
349 } else if (!IsVCCZ && MaskValue == 0) {
350 // Will never branch
351 MachineOperand &Dst = MI.getOperand(i: 0);
352 assert(Dst.isMBB() && "destination is not basic block");
353 MachineBasicBlock *Parent = MI.getParent();
354 updateMLIBeforeRemovingEdge(From: Parent, To: Dst.getMBB());
355 Parent->removeSuccessor(Succ: Dst.getMBB());
356 MI.eraseFromParent();
357 return true;
358 } else if (MaskValue == -1) {
359 // Depends only on EXEC
360 MI.setDesc(
361 TII->get(Opcode: IsVCCZ ? AMDGPU::S_CBRANCH_EXECZ : AMDGPU::S_CBRANCH_EXECNZ));
362 }
363
364 MI.removeOperand(OpNo: MI.findRegisterUseOperandIdx(Reg: CondReg, TRI, isKill: false /*Kill*/));
365 MI.addImplicitDefUseOperands(MF&: *MBB.getParent());
366
367 return true;
368}
369
370bool SIPreEmitPeephole::optimizeSetGPR(MachineInstr &First,
371 MachineInstr &MI) const {
372 MachineBasicBlock &MBB = *MI.getParent();
373 const MachineFunction &MF = *MBB.getParent();
374 const MachineRegisterInfo &MRI = MF.getRegInfo();
375 MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
376 Register IdxReg = Idx->isReg() ? Idx->getReg() : Register();
377 SmallVector<MachineInstr *, 4> ToRemove;
378 bool IdxOn = true;
379
380 if (!MI.isIdenticalTo(Other: First))
381 return false;
382
383 // Scan back to find an identical S_SET_GPR_IDX_ON
384 for (MachineBasicBlock::instr_iterator I = std::next(x: First.getIterator()),
385 E = MI.getIterator();
386 I != E; ++I) {
387 if (I->isBundle() || I->isDebugInstr())
388 continue;
389 switch (I->getOpcode()) {
390 case AMDGPU::S_SET_GPR_IDX_MODE:
391 return false;
392 case AMDGPU::S_SET_GPR_IDX_OFF:
393 IdxOn = false;
394 ToRemove.push_back(Elt: &*I);
395 break;
396 default:
397 if (I->modifiesRegister(Reg: AMDGPU::M0, TRI))
398 return false;
399 if (IdxReg && I->modifiesRegister(Reg: IdxReg, TRI))
400 return false;
401 if (llvm::any_of(Range: I->operands(), P: [&MRI, this](const MachineOperand &MO) {
402 return MO.isReg() && TRI->isVectorRegister(MRI, Reg: MO.getReg());
403 })) {
404 // The only exception allowed here is another indirect vector move
405 // with the same mode.
406 if (!IdxOn || !(I->getOpcode() == AMDGPU::V_MOV_B32_indirect_write ||
407 I->getOpcode() == AMDGPU::V_MOV_B32_indirect_read))
408 return false;
409 }
410 }
411 }
412
413 MI.eraseFromBundle();
414 for (MachineInstr *RI : ToRemove)
415 RI->eraseFromBundle();
416 return true;
417}
418
419bool SIPreEmitPeephole::getBlockDestinations(
420 MachineBasicBlock &SrcMBB, MachineBasicBlock *&TrueMBB,
421 MachineBasicBlock *&FalseMBB, SmallVectorImpl<MachineOperand> &Cond) {
422 if (TII->analyzeBranch(MBB&: SrcMBB, TBB&: TrueMBB, FBB&: FalseMBB, Cond))
423 return false;
424
425 if (!FalseMBB)
426 FalseMBB = SrcMBB.getNextNode();
427
428 return true;
429}
430
431namespace {
432class BranchWeightCostModel {
433 const SIInstrInfo &TII;
434 const TargetSchedModel &SchedModel;
435 BranchProbability BranchProb;
436 static constexpr uint64_t BranchNotTakenCost = 1;
437 uint64_t BranchTakenCost;
438 uint64_t ThenCyclesCost = 0;
439
440public:
441 BranchWeightCostModel(const SIInstrInfo &TII, const MachineInstr &Branch,
442 const MachineBasicBlock &Succ)
443 : TII(TII), SchedModel(TII.getSchedModel()) {
444 const MachineBasicBlock &Head = *Branch.getParent();
445 const auto *FromIt = find(Range: Head.successors(), Val: &Succ);
446 assert(FromIt != Head.succ_end());
447
448 BranchProb = Head.getSuccProbability(Succ: FromIt);
449 if (BranchProb.isUnknown())
450 BranchProb = BranchProbability::getZero();
451 BranchTakenCost = SchedModel.computeInstrLatency(MI: &Branch);
452 }
453
454 bool isProfitable(const MachineInstr &MI) {
455 if (TII.isWaitcnt(Opcode: MI.getOpcode()))
456 return false;
457
458 ThenCyclesCost += SchedModel.computeInstrLatency(MI: &MI);
459
460 // Consider `P = N/D` to be the probability of execz being false (skipping
461 // the then-block) The transformation is profitable if always executing the
462 // 'then' block is cheaper than executing sometimes 'then' and always
463 // executing s_cbranch_execz:
464 // * ThenCost <= P*ThenCost + (1-P)*BranchTakenCost + P*BranchNotTakenCost
465 // * (1-P) * ThenCost <= (1-P)*BranchTakenCost + P*BranchNotTakenCost
466 // * (D-N)/D * ThenCost <= (D-N)/D * BranchTakenCost + N/D *
467 // BranchNotTakenCost
468 uint64_t Numerator = BranchProb.getNumerator();
469 uint64_t Denominator = BranchProb.getDenominator();
470 return (Denominator - Numerator) * ThenCyclesCost <=
471 ((Denominator - Numerator) * BranchTakenCost +
472 Numerator * BranchNotTakenCost);
473 }
474};
475
476bool SIPreEmitPeephole::mustRetainExeczBranch(
477 const MachineInstr &Branch, const MachineBasicBlock &From,
478 const MachineBasicBlock &To) const {
479 assert(is_contained(Branch.getParent()->successors(), &From));
480 BranchWeightCostModel CostModel{*TII, Branch, From};
481
482 const MachineFunction *MF = From.getParent();
483 for (MachineFunction::const_iterator MBBI(&From), ToI(&To), End = MF->end();
484 MBBI != End && MBBI != ToI; ++MBBI) {
485 const MachineBasicBlock &MBB = *MBBI;
486
487 for (const MachineInstr &MI : MBB) {
488 // When a uniform loop is inside non-uniform control flow, the branch
489 // leaving the loop might never be taken when EXEC = 0.
490 // Hence we should retain cbranch out of the loop lest it become infinite.
491 if (MI.isConditionalBranch())
492 return true;
493
494 if (MI.isUnconditionalBranch() &&
495 TII->getBranchDestBlock(MI) != MBB.getNextNode())
496 return true;
497
498 if (MI.isMetaInstruction())
499 continue;
500
501 if (TII->hasUnwantedEffectsWhenEXECEmpty(MI))
502 return true;
503
504 if (!CostModel.isProfitable(MI))
505 return true;
506 }
507 }
508
509 return false;
510}
511} // namespace
512
513// Returns true if the skip branch instruction is removed.
514bool SIPreEmitPeephole::removeExeczBranch(MachineInstr &MI,
515 MachineBasicBlock &SrcMBB) {
516
517 if (!TII->getSchedModel().hasInstrSchedModel())
518 return false;
519
520 MachineBasicBlock *TrueMBB = nullptr;
521 MachineBasicBlock *FalseMBB = nullptr;
522 SmallVector<MachineOperand, 1> Cond;
523
524 if (!getBlockDestinations(SrcMBB, TrueMBB, FalseMBB, Cond))
525 return false;
526
527 // Consider only the forward branches.
528 if (SrcMBB.getNumber() >= TrueMBB->getNumber())
529 return false;
530
531 // Consider only when it is legal and profitable
532 if (mustRetainExeczBranch(Branch: MI, From: *FalseMBB, To: *TrueMBB))
533 return false;
534
535 LLVM_DEBUG(dbgs() << "Removing the execz branch: " << MI);
536 MI.eraseFromParent();
537 SrcMBB.removeSuccessor(Succ: TrueMBB);
538
539 return true;
540}
541
542/// Remove writes to the FP round mode and FP denorm mode that can never be
543/// observed: either the value written is already live in MODE, or a mode write
544/// replaces the whole mode field before anything reads it.
545///
546/// s_round_mode and s_denorm_mode each assign one field of MODE and preserve
547/// the rest of the register, so the two fields are tracked independently and a
548/// write to one is transparent to the other.
549///
550/// This is a purely intra-block analysis: the mode on entry to \p SrcMBB is
551/// unknown, and a write that is still live at the end of the block is kept for
552/// the benefit of the successors.
553bool SIPreEmitPeephole::removeRedundantModeWrites(
554 MachineBasicBlock &SrcMBB) const {
555 bool Changed = false;
556 ModeFieldState DenormMode;
557 ModeFieldState RoundMode;
558
559 for (MachineInstr &MI : make_early_inc_range(Range&: SrcMBB)) {
560 if (MI.isDebugInstr())
561 continue;
562
563 unsigned Opc = MI.getOpcode();
564 if (Opc == AMDGPU::S_DENORM_MODE || Opc == AMDGPU::S_ROUND_MODE) {
565 ModeFieldState &Field =
566 Opc == AMDGPU::S_DENORM_MODE ? DenormMode : RoundMode;
567 int64_t NewValue = MI.getOperand(i: 0).getImm();
568
569 if (Field.PendingWrite) {
570 LLVM_DEBUG(dbgs() << "Removing dead mode write: "
571 << *Field.PendingWrite);
572 Field.PendingWrite->eraseFromParent();
573 ++NumModeWritesRemoved;
574 Changed = true;
575 Field.PendingWrite = nullptr;
576 Field.Value = Field.ValueBeforePendingWrite;
577 }
578
579 if (Field.Value == NewValue) {
580 LLVM_DEBUG(dbgs() << "Removing redundant mode write: " << MI);
581 MI.eraseFromParent();
582 ++NumModeWritesRemoved;
583 Changed = true;
584 continue;
585 }
586
587 Field.ValueBeforePendingWrite = Field.Value;
588 Field.PendingWrite = &MI;
589 Field.Value = NewValue;
590 continue;
591 }
592
593 // Nothing tracked yet; skip register checks below.
594 if (!DenormMode.isTracked() && !RoundMode.isTracked())
595 continue;
596
597 // Inline asm cannot declare a MODE clobber, so assume it writes both.
598 if (MI.isInlineAsm() || MI.modifiesRegister(Reg: AMDGPU::MODE, TRI)) {
599 DenormMode = ModeFieldState();
600 RoundMode = ModeFieldState();
601 continue;
602 }
603
604 if (MI.readsRegister(Reg: AMDGPU::MODE, TRI) || MI.hasUnmodeledSideEffects()) {
605 DenormMode.PendingWrite = nullptr;
606 RoundMode.PendingWrite = nullptr;
607 }
608 }
609 return Changed;
610}
611
612bool SIPreEmitPeephole::canUnpackingClobberRegister(const MachineInstr &MI) {
613 unsigned OpCode = MI.getOpcode();
614 Register DstReg = MI.getOperand(i: 0).getReg();
615 // Only the first register in the register pair needs to be checked due to the
616 // unpacking order. Packed instructions are unpacked such that the lower 32
617 // bits (i.e., the first register in the pair) are written first. This can
618 // introduce dependencies if the first register is written in one instruction
619 // and then read as part of the higher 32 bits in the subsequent instruction.
620 // Such scenarios can arise due to specific combinations of op_sel and
621 // op_sel_hi modifiers.
622 Register UnpackedDstReg = TRI->getSubReg(Reg: DstReg, Idx: AMDGPU::sub0);
623
624 const MachineOperand *Src0MO = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
625 if (Src0MO && Src0MO->isReg()) {
626 Register SrcReg0 = Src0MO->getReg();
627 unsigned Src0Mods =
628 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers)->getImm();
629 Register HiSrc0Reg = (Src0Mods & SISrcMods::OP_SEL_1)
630 ? TRI->getSubReg(Reg: SrcReg0, Idx: AMDGPU::sub1)
631 : TRI->getSubReg(Reg: SrcReg0, Idx: AMDGPU::sub0);
632 // Check if the register selected by op_sel_hi is the same as the first
633 // register in the destination register pair.
634 if (TRI->regsOverlap(RegA: UnpackedDstReg, RegB: HiSrc0Reg))
635 return true;
636 }
637
638 const MachineOperand *Src1MO = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
639 if (Src1MO && Src1MO->isReg()) {
640 Register SrcReg1 = Src1MO->getReg();
641 unsigned Src1Mods =
642 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers)->getImm();
643 Register HiSrc1Reg = (Src1Mods & SISrcMods::OP_SEL_1)
644 ? TRI->getSubReg(Reg: SrcReg1, Idx: AMDGPU::sub1)
645 : TRI->getSubReg(Reg: SrcReg1, Idx: AMDGPU::sub0);
646 if (TRI->regsOverlap(RegA: UnpackedDstReg, RegB: HiSrc1Reg))
647 return true;
648 }
649
650 // Applicable for packed instructions with 3 source operands, such as
651 // V_PK_FMA.
652 if (AMDGPU::hasNamedOperand(Opcode: OpCode, NamedIdx: AMDGPU::OpName::src2)) {
653 const MachineOperand *Src2MO =
654 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
655 if (Src2MO && Src2MO->isReg()) {
656 Register SrcReg2 = Src2MO->getReg();
657 unsigned Src2Mods =
658 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers)->getImm();
659 Register HiSrc2Reg = (Src2Mods & SISrcMods::OP_SEL_1)
660 ? TRI->getSubReg(Reg: SrcReg2, Idx: AMDGPU::sub1)
661 : TRI->getSubReg(Reg: SrcReg2, Idx: AMDGPU::sub0);
662 if (TRI->regsOverlap(RegA: UnpackedDstReg, RegB: HiSrc2Reg))
663 return true;
664 }
665 }
666 return false;
667}
668
669uint32_t SIPreEmitPeephole::mapToUnpackedOpcode(MachineInstr &I) {
670 unsigned Opcode = I.getOpcode();
671 // Use 64 bit encoding to allow use of VOP3 instructions.
672 // VOP3 e64 instructions allow source modifiers
673 // e32 instructions don't allow source modifiers.
674 switch (Opcode) {
675 case AMDGPU::V_PK_ADD_F32:
676 case AMDGPU::V_PK_ADD_F32_gfx1250:
677 return AMDGPU::V_ADD_F32_e64;
678 case AMDGPU::V_PK_MUL_F32:
679 case AMDGPU::V_PK_MUL_F32_gfx1250:
680 return AMDGPU::V_MUL_F32_e64;
681 case AMDGPU::V_PK_FMA_F32:
682 case AMDGPU::V_PK_FMA_F32_gfx1250:
683 return AMDGPU::V_FMA_F32_e64;
684 default:
685 return std::numeric_limits<uint32_t>::max();
686 }
687 llvm_unreachable("Fully covered switch");
688}
689
690void SIPreEmitPeephole::addOperandAndMods(MachineInstrBuilder &NewMI,
691 unsigned SrcMods, bool IsHiBits,
692 const MachineOperand &SrcMO) {
693 unsigned NewSrcMods = 0;
694 unsigned NegModifier = IsHiBits ? SISrcMods::NEG_HI : SISrcMods::NEG;
695 unsigned OpSelModifier = IsHiBits ? SISrcMods::OP_SEL_1 : SISrcMods::OP_SEL_0;
696 // Packed instructions (VOP3P) do not support ABS. Hence, no checks are done
697 // for ABS modifiers.
698 // If NEG or NEG_HI is true, we need to negate the corresponding 32 bit
699 // lane.
700 // NEG_HI shares the same bit position with ABS. But packed instructions do
701 // not support ABS. Therefore, NEG_HI must be translated to NEG source
702 // modifier for the higher 32 bits. Unpacked VOP3 instructions support
703 // ABS, but do not support NEG_HI. Therefore we need to explicitly add the
704 // NEG modifier if present in the packed instruction.
705 if (SrcMods & NegModifier)
706 NewSrcMods |= SISrcMods::NEG;
707 // Src modifiers. Only negative modifiers are added if needed. Unpacked
708 // operations do not have op_sel, therefore it must be handled explicitly as
709 // done below.
710 NewMI.addImm(Val: NewSrcMods);
711 if (SrcMO.isImm()) {
712 NewMI.addImm(Val: SrcMO.getImm());
713 return;
714 }
715 // If op_sel == 0, select register 0 of reg:sub0_sub1.
716 Register UnpackedSrcReg = (SrcMods & OpSelModifier)
717 ? TRI->getSubReg(Reg: SrcMO.getReg(), Idx: AMDGPU::sub1)
718 : TRI->getSubReg(Reg: SrcMO.getReg(), Idx: AMDGPU::sub0);
719
720 MachineOperand UnpackedSrcMO =
721 MachineOperand::CreateReg(Reg: UnpackedSrcReg, /*isDef=*/false);
722 if (SrcMO.isKill()) {
723 // For each unpacked instruction, mark its source registers as killed if the
724 // corresponding source register in the original packed instruction was
725 // marked as killed.
726 //
727 // Exception:
728 // If the op_sel and op_sel_hi modifiers require both unpacked instructions
729 // to use the same register (e.g., due to overlapping access to low/high
730 // bits of the same packed register), then only the *second* (latter)
731 // instruction should mark the register as killed. This is because the
732 // second instruction handles the higher bits and is effectively the last
733 // user of the full register pair.
734
735 bool OpSel = SrcMods & SISrcMods::OP_SEL_0;
736 bool OpSelHi = SrcMods & SISrcMods::OP_SEL_1;
737 bool KillState = true;
738 if ((OpSel == OpSelHi) && !IsHiBits)
739 KillState = false;
740 UnpackedSrcMO.setIsKill(KillState);
741 }
742 NewMI.add(MO: UnpackedSrcMO);
743}
744
745void SIPreEmitPeephole::collectUnpackingCandidates(
746 MachineInstr &BeginMI, SetVector<MachineInstr *> &InstrsToUnpack,
747 uint16_t NumMFMACycles) {
748 auto *BB = BeginMI.getParent();
749 auto E = BB->end();
750 int TotalCyclesBetweenCandidates = 0;
751 const TargetSchedModel &SchedModel = TII->getSchedModel();
752 Register MFMADef = BeginMI.getOperand(i: 0).getReg();
753
754 for (auto I = std::next(x: BeginMI.getIterator()); I != E; ++I) {
755 MachineInstr &Instr = *I;
756 uint32_t UnpackedOpCode = mapToUnpackedOpcode(I&: Instr);
757 bool IsUnpackable =
758 !(UnpackedOpCode == std::numeric_limits<uint32_t>::max());
759 if (Instr.isMetaInstruction())
760 continue;
761 if ((Instr.isTerminator()) ||
762 (TII->isNeverCoissue(MI&: Instr) && !IsUnpackable) ||
763 (SIInstrInfo::modifiesModeRegister(MI: Instr) &&
764 Instr.modifiesRegister(Reg: AMDGPU::EXEC, TRI)))
765 return;
766
767 const MCSchedClassDesc *InstrSchedClassDesc =
768 SchedModel.resolveSchedClass(MI: &Instr);
769 uint16_t Latency =
770 SchedModel.getWriteProcResBegin(SC: InstrSchedClassDesc)->ReleaseAtCycle;
771 TotalCyclesBetweenCandidates += Latency;
772
773 if (TotalCyclesBetweenCandidates >= NumMFMACycles - 1)
774 return;
775 // Identify register dependencies between those used by the MFMA
776 // instruction and the following packed instructions. Also checks for
777 // transitive dependencies between the MFMA def and candidate instruction
778 // def and uses. Conservatively ensures that we do not incorrectly
779 // read/write registers.
780 for (const MachineOperand &InstrMO : Instr.operands()) {
781 if (!InstrMO.isReg() || !InstrMO.getReg().isValid())
782 continue;
783 if (TRI->regsOverlap(RegA: MFMADef, RegB: InstrMO.getReg()))
784 return;
785 }
786 if (!IsUnpackable)
787 continue;
788
789 if (canUnpackingClobberRegister(MI: Instr))
790 return;
791 // If it's a packed instruction, adjust latency: remove the packed
792 // latency, add latency of two unpacked instructions (currently estimated
793 // as 2 cycles).
794 TotalCyclesBetweenCandidates -= Latency;
795 // TODO: improve latency handling based on instruction modeling.
796 TotalCyclesBetweenCandidates += 2;
797 // Subtract 1 to account for MFMA issue latency.
798 if (TotalCyclesBetweenCandidates < NumMFMACycles - 1)
799 InstrsToUnpack.insert(X: &Instr);
800 }
801}
802
803void SIPreEmitPeephole::performF32Unpacking(MachineInstr &I) {
804 const MachineOperand &DstOp = I.getOperand(i: 0);
805
806 uint32_t UnpackedOpcode = mapToUnpackedOpcode(I);
807 assert(UnpackedOpcode != std::numeric_limits<uint32_t>::max() &&
808 "Unsupported Opcode");
809
810 MachineInstrBuilder Op0LOp1L =
811 createUnpackedMI(I, UnpackedOpcode, /*IsHiBits=*/false);
812 MachineOperand &LoDstOp = Op0LOp1L->getOperand(i: 0);
813
814 LoDstOp.setIsUndef(DstOp.isUndef());
815
816 MachineInstrBuilder Op0HOp1H =
817 createUnpackedMI(I, UnpackedOpcode, /*IsHiBits=*/true);
818 MachineOperand &HiDstOp = Op0HOp1H->getOperand(i: 0);
819
820 uint32_t IFlags = I.getFlags();
821 Op0LOp1L->setFlags(IFlags);
822 Op0HOp1H->setFlags(IFlags);
823 LoDstOp.setIsRenamable(DstOp.isRenamable());
824 HiDstOp.setIsRenamable(DstOp.isRenamable());
825
826 I.eraseFromParent();
827}
828
829MachineInstrBuilder SIPreEmitPeephole::createUnpackedMI(MachineInstr &I,
830 uint32_t UnpackedOpcode,
831 bool IsHiBits) {
832 MachineBasicBlock &MBB = *I.getParent();
833 const DebugLoc &DL = I.getDebugLoc();
834 const MachineOperand *SrcMO0 = TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src0);
835 const MachineOperand *SrcMO1 = TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src1);
836 Register DstReg = I.getOperand(i: 0).getReg();
837 unsigned OpCode = I.getOpcode();
838 Register UnpackedDstReg = IsHiBits ? TRI->getSubReg(Reg: DstReg, Idx: AMDGPU::sub1)
839 : TRI->getSubReg(Reg: DstReg, Idx: AMDGPU::sub0);
840
841 int64_t ClampVal = TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::clamp)->getImm();
842 unsigned Src0Mods =
843 TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src0_modifiers)->getImm();
844 unsigned Src1Mods =
845 TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src1_modifiers)->getImm();
846
847 MachineInstrBuilder NewMI = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: UnpackedOpcode));
848 NewMI.addDef(RegNo: UnpackedDstReg); // vdst
849 addOperandAndMods(NewMI, SrcMods: Src0Mods, IsHiBits, SrcMO: *SrcMO0);
850 addOperandAndMods(NewMI, SrcMods: Src1Mods, IsHiBits, SrcMO: *SrcMO1);
851
852 if (AMDGPU::hasNamedOperand(Opcode: OpCode, NamedIdx: AMDGPU::OpName::src2)) {
853 const MachineOperand *SrcMO2 =
854 TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src2);
855 unsigned Src2Mods =
856 TII->getNamedOperand(MI&: I, OperandName: AMDGPU::OpName::src2_modifiers)->getImm();
857 addOperandAndMods(NewMI, SrcMods: Src2Mods, IsHiBits, SrcMO: *SrcMO2);
858 }
859 NewMI.addImm(Val: ClampVal); // clamp
860 // Packed instructions do not support output modifiers. safe to assign them 0
861 // for this use case
862 NewMI.addImm(Val: 0); // omod
863 return NewMI;
864}
865
866PreservedAnalyses
867llvm::SIPreEmitPeepholePass::run(MachineFunction &MF,
868 MachineFunctionAnalysisManager &MFAM) {
869 auto *MLI = MFAM.getCachedResult<MachineLoopAnalysis>(IR&: MF);
870 SIPreEmitPeephole Impl;
871
872 if (Impl.run(MF, MLI)) {
873 auto PA = getMachineFunctionPassPreservedAnalyses();
874 PA.preserve<MachineLoopAnalysis>();
875 return PA;
876 }
877
878 return PreservedAnalyses::all();
879}
880
881bool SIPreEmitPeephole::run(MachineFunction &MF, MachineLoopInfo *LoopInfo) {
882 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
883 TII = ST.getInstrInfo();
884 TRI = &TII->getRegisterInfo();
885 MLI = LoopInfo;
886 bool Changed = false;
887
888 MF.RenumberBlocks();
889
890 for (MachineBasicBlock &MBB : MF) {
891 Changed |= removeRedundantModeWrites(SrcMBB&: MBB);
892
893 MachineBasicBlock::iterator TermI = MBB.getFirstTerminator();
894 // Check first terminator for branches to optimize
895 if (TermI != MBB.end()) {
896 MachineInstr &MI = *TermI;
897 switch (MI.getOpcode()) {
898 case AMDGPU::S_CBRANCH_VCCZ:
899 case AMDGPU::S_CBRANCH_VCCNZ:
900 Changed |= optimizeVccBranch(MI);
901 break;
902 case AMDGPU::S_CBRANCH_EXECZ:
903 Changed |= removeExeczBranch(MI, SrcMBB&: MBB);
904 break;
905 }
906 }
907
908 if (!ST.hasVGPRIndexMode())
909 continue;
910
911 MachineInstr *SetGPRMI = nullptr;
912 const unsigned Threshold = 20;
913 unsigned Count = 0;
914 // Scan the block for two S_SET_GPR_IDX_ON instructions to see if a
915 // second is not needed. Do expensive checks in the optimizeSetGPR()
916 // and limit the distance to 20 instructions for compile time purposes.
917 // Note: this needs to work on bundles as S_SET_GPR_IDX* instructions
918 // may be bundled with the instructions they modify.
919 for (auto &MI : make_early_inc_range(Range: MBB.instrs())) {
920 if (Count == Threshold)
921 SetGPRMI = nullptr;
922 else
923 ++Count;
924
925 if (MI.getOpcode() != AMDGPU::S_SET_GPR_IDX_ON)
926 continue;
927
928 Count = 0;
929 if (!SetGPRMI) {
930 SetGPRMI = &MI;
931 continue;
932 }
933
934 if (optimizeSetGPR(First&: *SetGPRMI, MI))
935 Changed = true;
936 else
937 SetGPRMI = &MI;
938 }
939 }
940
941 // TODO: Fold this into previous block, if possible. Evaluate and handle any
942 // side effects.
943
944 // Perform the extra MF scans only for supported archs
945 if (!ST.hasGFX940Insts())
946 return Changed;
947 for (MachineBasicBlock &MBB : MF) {
948 // Unpack packed instructions overlapped by MFMAs. This allows the
949 // compiler to co-issue unpacked instructions with MFMA
950 const TargetSchedModel &SchedModel = TII->getSchedModel();
951 SetVector<MachineInstr *> InstrsToUnpack;
952 for (auto &MI : make_early_inc_range(Range: MBB.instrs())) {
953 if (!SIInstrInfo::isMFMA(MI))
954 continue;
955 const MCSchedClassDesc *SchedClassDesc =
956 SchedModel.resolveSchedClass(MI: &MI);
957 uint16_t NumMFMACycles =
958 SchedModel.getWriteProcResBegin(SC: SchedClassDesc)->ReleaseAtCycle;
959 collectUnpackingCandidates(BeginMI&: MI, InstrsToUnpack, NumMFMACycles);
960 }
961 for (MachineInstr *MI : InstrsToUnpack) {
962 performF32Unpacking(I&: *MI);
963 }
964 if (!InstrsToUnpack.empty())
965 Changed = true;
966 }
967
968 return Changed;
969}
970