1//===- AMDGPULowerVGPREncoding.cpp - lower VGPRs above v255 ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Lower VGPRs above first 256 on gfx1250.
11///
12/// The pass scans used VGPRs and inserts S_SET_VGPR_MSB instructions to switch
13/// VGPR addressing mode. The mode change is effective until the next change.
14/// This instruction provides high bits of a VGPR address for four of the
15/// operands: vdst, src0, src1, and src2, or other 4 operands depending on the
16/// instruction encoding. If bits are set they are added as MSB to the
17/// corresponding operand VGPR number.
18///
19/// There is no need to replace actual register operands because encoding of the
20/// high and low VGPRs is the same. I.e. v0 has the encoding 0x100, so does
21/// v256. v1 has the encoding 0x101 and v257 has the same encoding. So high
22/// VGPRs will survive until actual encoding and will result in a same actual
23/// bit encoding.
24///
25/// As a result the pass only inserts S_SET_VGPR_MSB to provide an actual offset
26/// to a VGPR address of the subseqent instructions. The InstPrinter will take
27/// care of the printing a low VGPR instead of a high one. In prinicple this
28/// shall be viable to print actual high VGPR numbers, but that would disagree
29/// with a disasm printing and create a situation where asm text is not
30/// deterministic.
31///
32/// This pass creates a convention where non-fall through basic blocks shall
33/// start with all 4 MSBs zero. Otherwise a disassembly would not be readable.
34/// An optimization here is possible but deemed not desirable because of the
35/// readbility concerns.
36///
37/// Consequentially the ABI is set to expect all 4 MSBs to be zero on entry.
38/// The pass must run very late in the pipeline to make sure no changes to VGPR
39/// operands will be made after it.
40//
41//===----------------------------------------------------------------------===//
42
43#include "AMDGPULowerVGPREncoding.h"
44#include "AMDGPU.h"
45#include "GCNSubtarget.h"
46#include "SIDefines.h"
47#include "SIInstrInfo.h"
48#include "llvm/ADT/bit.h"
49#include "llvm/Support/MathExtras.h"
50
51using namespace llvm;
52
53#define DEBUG_TYPE "amdgpu-lower-vgpr-encoding"
54
55namespace {
56
57class AMDGPULowerVGPREncoding {
58 static constexpr unsigned OpNum = 4;
59 static constexpr unsigned BitsPerField = 2;
60 static constexpr unsigned NumFields = 4;
61 static constexpr unsigned ModeWidth = NumFields * BitsPerField;
62 static constexpr unsigned ModeMask = (1 << ModeWidth) - 1;
63 static constexpr unsigned VGPRMSBShift =
64 llvm::countr_zero_constexpr<unsigned>(Val: AMDGPU::Hwreg::DST_VGPR_MSB);
65
66 struct OpMode {
67 // No MSBs set means they are not required to be of a particular value.
68 std::optional<unsigned> MSBits;
69
70 bool update(const OpMode &New, bool &Rewritten) {
71 bool Updated = false;
72 if (New.MSBits) {
73 if (*New.MSBits != MSBits.value_or(u: 0)) {
74 Updated = true;
75 Rewritten |= MSBits.has_value();
76 }
77 MSBits = New.MSBits;
78 }
79 return Updated;
80 }
81 };
82
83 struct ModeTy {
84 OpMode Ops[OpNum];
85
86 bool update(const ModeTy &New, bool &Rewritten) {
87 bool Updated = false;
88 for (unsigned I : seq(Size: OpNum))
89 Updated |= Ops[I].update(New: New.Ops[I], Rewritten);
90 return Updated;
91 }
92
93 unsigned encode() const {
94 // Layout: [src0 msb, src1 msb, src2 msb, dst msb].
95 unsigned V = 0;
96 for (const auto &[I, Op] : enumerate(First: Ops))
97 V |= Op.MSBits.value_or(u: 0) << (I * 2);
98 return V;
99 }
100 };
101
102public:
103 bool run(MachineFunction &MF);
104
105private:
106 const SIInstrInfo *TII;
107 const SIRegisterInfo *TRI;
108
109 // Current basic block.
110 MachineBasicBlock *MBB;
111
112 /// Most recent s_set_* instruction.
113 MachineInstr *MostRecentModeSet;
114
115 /// Current mode bits.
116 ModeTy CurrentMode;
117
118 /// Number of current hard clause instructions.
119 unsigned ClauseLen;
120
121 /// Number of hard clause instructions remaining.
122 unsigned ClauseRemaining;
123
124 /// Clause group breaks.
125 unsigned ClauseBreaks;
126
127 /// Last hard clause instruction.
128 MachineInstr *Clause;
129
130 /// Insert mode change before \p I. \returns true if mode was changed.
131 bool setMode(ModeTy NewMode, MachineBasicBlock::instr_iterator I);
132
133 /// Reset mode to default.
134 void resetMode(MachineBasicBlock::instr_iterator I) {
135 ModeTy Mode;
136 for (OpMode &Op : Mode.Ops)
137 Op.MSBits = 0;
138 setMode(NewMode: Mode, I);
139 }
140
141 /// If \p MO references VGPRs, return the MSBs. Otherwise, return nullopt.
142 std::optional<unsigned> getMSBs(const MachineOperand &MO) const;
143
144 /// Handle single \p MI. \return true if changed.
145 bool runOnMachineInstr(MachineInstr &MI);
146
147 /// Compute the mode for a single \p MI given \p Ops operands
148 /// bit mapping. Optionally takes second array \p Ops2 for VOPD.
149 /// If provided and an operand from \p Ops is not a VGPR, then \p Ops2
150 /// is checked.
151 void computeMode(ModeTy &NewMode, MachineInstr &MI,
152 const AMDGPU::OpName Ops[OpNum],
153 const AMDGPU::OpName *Ops2 = nullptr);
154
155 /// Check if an instruction \p I is within a clause and returns a suitable
156 /// iterator to insert mode change. It may also modify the S_CLAUSE
157 /// instruction to extend it or drop the clause if it cannot be adjusted.
158 MachineBasicBlock::instr_iterator
159 handleClause(MachineBasicBlock::instr_iterator I);
160
161 /// Check if an instruction \p I is immediately after another program state
162 /// instruction which it cannot coissue with. If so, insert before that
163 /// instruction to encourage more coissuing.
164 MachineBasicBlock::instr_iterator
165 handleCoissue(MachineBasicBlock::instr_iterator I);
166
167 /// Handle S_SETREG_IMM32_B32 targeting MODE register. On certain hardware,
168 /// this instruction clobbers VGPR MSB bits[12:19], so we need to restore
169 /// the current mode. \returns true if the instruction was modified or a
170 /// new one was inserted.
171 bool handleSetregMode(MachineInstr &MI);
172
173 /// Update bits[12:19] of the imm operand in S_SETREG_IMM32_B32 to contain
174 /// the VGPR MSB mode value. \returns true if the immediate was changed.
175 bool updateSetregModeImm(MachineInstr &MI, int64_t ModeValue);
176};
177
178bool AMDGPULowerVGPREncoding::setMode(ModeTy NewMode,
179 MachineBasicBlock::instr_iterator I) {
180 // Record previous mode into high 8 bits of the immediate.
181 int64_t OldModeBits = CurrentMode.encode() << ModeWidth;
182
183 bool Rewritten = false;
184 if (!CurrentMode.update(New: NewMode, Rewritten))
185 return false;
186
187 if (MostRecentModeSet && !Rewritten) {
188 // Update MostRecentModeSet with the new mode. It can be either
189 // S_SET_VGPR_MSB or S_SETREG_IMM32_B32 (with Size <= 12).
190 if (MostRecentModeSet->getOpcode() == AMDGPU::S_SET_VGPR_MSB) {
191 MachineOperand &Op = MostRecentModeSet->getOperand(i: 0);
192 // Carry old mode bits from the existing instruction.
193 int64_t OldModeBits = Op.getImm() & (ModeMask << ModeWidth);
194 Op.setImm(CurrentMode.encode() | OldModeBits);
195 } else {
196 assert(MostRecentModeSet->getOpcode() == AMDGPU::S_SETREG_IMM32_B32 &&
197 "unexpected MostRecentModeSet opcode");
198 updateSetregModeImm(MI&: *MostRecentModeSet, ModeValue: CurrentMode.encode());
199 }
200
201 return true;
202 }
203
204 I = handleClause(I);
205 I = handleCoissue(I);
206 MostRecentModeSet = BuildMI(BB&: *MBB, I, MIMD: {}, MCID: TII->get(Opcode: AMDGPU::S_SET_VGPR_MSB))
207 .addImm(Val: NewMode.encode() | OldModeBits);
208
209 CurrentMode = NewMode;
210 return true;
211}
212
213std::optional<unsigned>
214AMDGPULowerVGPREncoding::getMSBs(const MachineOperand &MO) const {
215 if (!MO.isReg())
216 return std::nullopt;
217
218 MCRegister Reg = MO.getReg();
219 const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Reg);
220 if (!RC || !TRI->isVGPRClass(RC))
221 return std::nullopt;
222
223 unsigned Idx = TRI->getHWRegIndex(Reg);
224 return Idx >> 8;
225}
226
227void AMDGPULowerVGPREncoding::computeMode(ModeTy &NewMode, MachineInstr &MI,
228 const AMDGPU::OpName Ops[OpNum],
229 const AMDGPU::OpName *Ops2) {
230 NewMode = {};
231
232 for (unsigned I = 0; I < OpNum; ++I) {
233 MachineOperand *Op = TII->getNamedOperand(MI, OperandName: Ops[I]);
234
235 std::optional<unsigned> MSBits;
236 if (Op)
237 MSBits = getMSBs(MO: *Op);
238
239#if !defined(NDEBUG)
240 if (MSBits.has_value() && Ops2) {
241 auto Op2 = TII->getNamedOperand(MI, Ops2[I]);
242 if (Op2) {
243 std::optional<unsigned> MSBits2;
244 MSBits2 = getMSBs(*Op2);
245 if (MSBits2.has_value() && MSBits != MSBits2)
246 llvm_unreachable("Invalid VOPD pair was created");
247 }
248 }
249#endif
250
251 if (!MSBits.has_value() && Ops2) {
252 Op = TII->getNamedOperand(MI, OperandName: Ops2[I]);
253 if (Op)
254 MSBits = getMSBs(MO: *Op);
255 }
256
257 if (!MSBits.has_value())
258 continue;
259
260 // Skip tied uses of src2 of VOP2, these will be handled along with defs and
261 // only vdst bit affects these operands. We cannot skip tied uses of VOP3,
262 // these uses are real even if must match the vdst.
263 if (Ops[I] == AMDGPU::OpName::src2 && !Op->isDef() && Op->isTied() &&
264 (SIInstrInfo::isVOP2(MI) ||
265 (SIInstrInfo::isVOP3(MI) &&
266 TII->hasVALU32BitEncoding(Opcode: MI.getOpcode()))))
267 continue;
268
269 NewMode.Ops[I].MSBits = MSBits.value();
270 }
271}
272
273bool AMDGPULowerVGPREncoding::runOnMachineInstr(MachineInstr &MI) {
274 auto Ops = AMDGPU::getVGPRLoweringOperandTables(Desc: MI.getDesc());
275 if (Ops.first) {
276 ModeTy NewMode;
277 computeMode(NewMode, MI, Ops: Ops.first, Ops2: Ops.second);
278 return setMode(NewMode, I: MI.getIterator());
279 }
280 assert(!TII->hasVGPRUses(MI) || MI.isMetaInstruction() || MI.isPseudo());
281
282 return false;
283}
284
285MachineBasicBlock::instr_iterator
286AMDGPULowerVGPREncoding::handleClause(MachineBasicBlock::instr_iterator I) {
287 if (!ClauseRemaining)
288 return I;
289
290 // A clause cannot start with a special instruction, place it right before
291 // the clause.
292 if (ClauseRemaining == ClauseLen) {
293 I = Clause->getPrevNode()->getIterator();
294 assert(I->isBundle());
295 return I;
296 }
297
298 // If a clause defines breaks each group cannot start with a mode change.
299 // just drop the clause.
300 if (ClauseBreaks) {
301 Clause->eraseFromBundle();
302 ClauseRemaining = 0;
303 return I;
304 }
305
306 // Otherwise adjust a number of instructions in the clause if it fits.
307 // If it does not clause will just become shorter. Since the length
308 // recorded in the clause is one less, increment the length after the
309 // update. Note that SIMM16[5:0] must be 1-62, not 0 or 63.
310 if (ClauseLen < 63)
311 Clause->getOperand(i: 0).setImm(ClauseLen | (ClauseBreaks << 8));
312
313 ++ClauseLen;
314
315 return I;
316}
317
318MachineBasicBlock::instr_iterator
319AMDGPULowerVGPREncoding::handleCoissue(MachineBasicBlock::instr_iterator I) {
320 if (I.isEnd())
321 return I;
322
323 // "Program State instructions" are instructions which are used to control
324 // operation of the GPU rather than performing arithmetic. Such instructions
325 // have different coissuing rules w.r.t s_set_vgpr_msb.
326 auto isProgramStateInstr = [this](MachineInstr *MI) {
327 unsigned Opc = MI->getOpcode();
328 return TII->isBarrier(Opcode: Opc) || TII->isWaitcnt(Opcode: Opc) ||
329 Opc == AMDGPU::S_DELAY_ALU;
330 };
331
332 while (!I.isEnd() && I != I->getParent()->begin()) {
333 auto Prev = std::prev(x: I);
334 if (!isProgramStateInstr(&*Prev))
335 return I;
336 I = Prev;
337 }
338
339 return I;
340}
341
342/// Convert mode value from S_SET_VGPR_MSB format to MODE register format.
343/// S_SET_VGPR_MSB uses: (src0[0-1], src1[2-3], src2[4-5], dst[6-7])
344/// MODE register uses: (dst[0-1], src0[2-3], src1[4-5], src2[6-7])
345/// This is a left rotation by 2 bits on an 8-bit value.
346static int64_t convertModeToSetregFormat(int64_t Mode) {
347 assert(isUInt<8>(Mode) && "Mode expected to be 8-bit");
348 return llvm::rotl<uint8_t>(V: static_cast<uint8_t>(Mode), /*R=*/2);
349}
350
351bool AMDGPULowerVGPREncoding::updateSetregModeImm(MachineInstr &MI,
352 int64_t ModeValue) {
353 assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32);
354
355 // Convert from S_SET_VGPR_MSB format to MODE register format
356 int64_t SetregMode = convertModeToSetregFormat(Mode: ModeValue);
357
358 MachineOperand *ImmOp = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::imm);
359 int64_t OldImm = ImmOp->getImm();
360 int64_t NewImm =
361 (OldImm & ~AMDGPU::Hwreg::VGPR_MSB_MASK) | (SetregMode << VGPRMSBShift);
362 ImmOp->setImm(NewImm);
363 return NewImm != OldImm;
364}
365
366bool AMDGPULowerVGPREncoding::handleSetregMode(MachineInstr &MI) {
367 using namespace AMDGPU::Hwreg;
368
369 assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 &&
370 "only S_SETREG_IMM32_B32 needs to be handled");
371
372 MachineOperand *SIMM16Op = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::simm16);
373 assert(SIMM16Op && "SIMM16Op must be present");
374
375 auto [HwRegId, Offset, Size] = HwregEncoding::decode(Encoded: SIMM16Op->getImm());
376 (void)Offset;
377 if (HwRegId != ID_MODE)
378 return false;
379
380 int64_t ModeValue = CurrentMode.encode();
381
382 // Case 1: Size <= 12 - the original instruction uses imm32[0:Size-1], so
383 // imm32[12:19] is unused. Safe to set imm32[12:19] to the correct VGPR
384 // MSBs.
385 if (Size <= VGPRMSBShift) {
386 // This instruction now acts as MostRecentModeSet so it can be updated if
387 // CurrentMode changes via piggybacking.
388 MostRecentModeSet = &MI;
389 return updateSetregModeImm(MI, ModeValue);
390 }
391
392 // Case 2: Size > 12 - the original instruction uses bits beyond 11, so we
393 // cannot arbitrarily modify imm32[12:19]. Check if it already matches VGPR
394 // MSBs. Note: imm32[12:19] is in MODE register format, while ModeValue is
395 // in S_SET_VGPR_MSB format, so we need to convert before comparing.
396 MachineOperand *ImmOp = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::imm);
397 assert(ImmOp && "ImmOp must be present");
398 int64_t ImmBits12To19 = (ImmOp->getImm() & VGPR_MSB_MASK) >> VGPRMSBShift;
399 int64_t SetregModeValue = convertModeToSetregFormat(Mode: ModeValue);
400 if (ImmBits12To19 == SetregModeValue) {
401 // Already correct, but we must invalidate MostRecentModeSet because this
402 // instruction will overwrite mode[12:19]. We can't update this instruction
403 // via piggybacking (bits[12:19] are meaningful), so if CurrentMode changes,
404 // a new s_set_vgpr_msb will be inserted after this instruction.
405 MostRecentModeSet = nullptr;
406 return false;
407 }
408
409 // imm32[12:19] doesn't match VGPR MSBs - insert s_set_vgpr_msb after
410 // the original instruction to restore the correct value.
411 MachineBasicBlock::iterator InsertPt = std::next(x: MI.getIterator());
412 MostRecentModeSet = BuildMI(BB&: *MBB, I: InsertPt, MIMD: MI.getDebugLoc(),
413 MCID: TII->get(Opcode: AMDGPU::S_SET_VGPR_MSB))
414 .addImm(Val: ModeValue);
415 return true;
416}
417
418bool AMDGPULowerVGPREncoding::run(MachineFunction &MF) {
419 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
420 if (!ST.has1024AddressableVGPRs())
421 return false;
422
423 TII = ST.getInstrInfo();
424 TRI = ST.getRegisterInfo();
425
426 bool Changed = false;
427 ClauseLen = ClauseRemaining = 0;
428 CurrentMode = {};
429 for (auto &MBB : MF) {
430 MostRecentModeSet = nullptr;
431 this->MBB = &MBB;
432
433 for (auto &MI : llvm::make_early_inc_range(Range: MBB.instrs())) {
434 if (MI.isMetaInstruction())
435 continue;
436
437 if (MI.isTerminator() || MI.isCall()) {
438 if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
439 MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED)
440 CurrentMode = {};
441 else
442 resetMode(I: MI.getIterator());
443 continue;
444 }
445
446 if (MI.isInlineAsm()) {
447 if (TII->hasVGPRUses(MI))
448 resetMode(I: MI.getIterator());
449 continue;
450 }
451
452 if (MI.getOpcode() == AMDGPU::S_CLAUSE) {
453 assert(!ClauseRemaining && "Nested clauses are not supported");
454 ClauseLen = MI.getOperand(i: 0).getImm();
455 ClauseBreaks = (ClauseLen >> 8) & 15;
456 ClauseLen = ClauseRemaining = (ClauseLen & 63) + 1;
457 Clause = &MI;
458 continue;
459 }
460
461 if (MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 &&
462 ST.hasSetregVGPRMSBFixup()) {
463 Changed |= handleSetregMode(MI);
464 continue;
465 }
466
467 Changed |= runOnMachineInstr(MI);
468
469 if (ClauseRemaining)
470 --ClauseRemaining;
471 }
472
473 // Reset the mode if we are falling through.
474 resetMode(I: MBB.instr_end());
475 }
476
477 return Changed;
478}
479
480class AMDGPULowerVGPREncodingLegacy : public MachineFunctionPass {
481public:
482 static char ID;
483
484 AMDGPULowerVGPREncodingLegacy() : MachineFunctionPass(ID) {}
485
486 bool runOnMachineFunction(MachineFunction &MF) override {
487 return AMDGPULowerVGPREncoding().run(MF);
488 }
489
490 void getAnalysisUsage(AnalysisUsage &AU) const override {
491 AU.setPreservesCFG();
492 MachineFunctionPass::getAnalysisUsage(AU);
493 }
494};
495
496} // namespace
497
498char AMDGPULowerVGPREncodingLegacy::ID = 0;
499
500char &llvm::AMDGPULowerVGPREncodingLegacyID = AMDGPULowerVGPREncodingLegacy::ID;
501
502INITIALIZE_PASS(AMDGPULowerVGPREncodingLegacy, DEBUG_TYPE,
503 "AMDGPU Lower VGPR Encoding", false, false)
504
505PreservedAnalyses
506AMDGPULowerVGPREncodingPass::run(MachineFunction &MF,
507 MachineFunctionAnalysisManager &MFAM) {
508 if (!AMDGPULowerVGPREncoding().run(MF))
509 return PreservedAnalyses::all();
510
511 return getMachineFunctionPassPreservedAnalyses().preserveSet<CFGAnalyses>();
512}
513