1//===- R600ExpandSpecialInstrs.cpp - Expand special instructions ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Vector, Reduction, and Cube instructions need to fill the entire instruction
11/// group to work correctly. This pass expands these individual instructions
12/// into several instructions that will completely fill the instruction group.
13//
14//===----------------------------------------------------------------------===//
15
16#include "MCTargetDesc/R600MCTargetDesc.h"
17#include "R600.h"
18#include "R600Defines.h"
19#include "R600Subtarget.h"
20#include "llvm/CodeGen/MachineFunction.h"
21#include "llvm/CodeGen/MachineFunctionPass.h"
22
23using namespace llvm;
24
25#define DEBUG_TYPE "r600-expand-special-instrs"
26
27namespace {
28
29class R600ExpandSpecialInstrsPass : public MachineFunctionPass {
30private:
31 const R600InstrInfo *TII = nullptr;
32
33 void SetFlagInNewMI(MachineInstr *NewMI, const MachineInstr *OldMI,
34 unsigned Op);
35
36public:
37 static char ID;
38
39 R600ExpandSpecialInstrsPass() : MachineFunctionPass(ID) {}
40
41 bool runOnMachineFunction(MachineFunction &MF) override;
42
43 StringRef getPassName() const override {
44 return "R600 Expand special instructions pass";
45 }
46};
47
48} // end anonymous namespace
49
50INITIALIZE_PASS_BEGIN(R600ExpandSpecialInstrsPass, DEBUG_TYPE,
51 "R600 Expand Special Instrs", false, false)
52INITIALIZE_PASS_END(R600ExpandSpecialInstrsPass, DEBUG_TYPE,
53 "R600ExpandSpecialInstrs", false, false)
54
55char R600ExpandSpecialInstrsPass::ID = 0;
56
57char &llvm::R600ExpandSpecialInstrsPassID = R600ExpandSpecialInstrsPass::ID;
58
59FunctionPass *llvm::createR600ExpandSpecialInstrsPass() {
60 return new R600ExpandSpecialInstrsPass();
61}
62
63void R600ExpandSpecialInstrsPass::SetFlagInNewMI(MachineInstr *NewMI,
64 const MachineInstr *OldMI, unsigned Op) {
65 int OpIdx = TII->getOperandIdx(MI: *OldMI, Op);
66 if (OpIdx > -1) {
67 uint64_t Val = OldMI->getOperand(i: OpIdx).getImm();
68 TII->setImmOperand(MI&: *NewMI, Op, Imm: Val);
69 }
70}
71
72bool R600ExpandSpecialInstrsPass::runOnMachineFunction(MachineFunction &MF) {
73 const R600Subtarget &ST = MF.getSubtarget<R600Subtarget>();
74 TII = ST.getInstrInfo();
75
76 const R600RegisterInfo &TRI = TII->getRegisterInfo();
77
78 for (MachineBasicBlock &MBB : MF) {
79 MachineBasicBlock::iterator I = MBB.begin();
80 while (I != MBB.end()) {
81 MachineInstr &MI = *I;
82 I = std::next(x: I);
83
84 // Expand LDS_*_RET instructions
85 if (TII->isLDSRetInstr(Opcode: MI.getOpcode())) {
86 int DstIdx = TII->getOperandIdx(Opcode: MI.getOpcode(), Op: R600::OpName::dst);
87 assert(DstIdx != -1);
88 MachineOperand &DstOp = MI.getOperand(i: DstIdx);
89 MachineInstr *Mov = TII->buildMovInstr(MBB: &MBB, I,
90 DstReg: DstOp.getReg(), SrcReg: R600::OQAP);
91 DstOp.setReg(R600::OQAP);
92 int LDSPredSelIdx = TII->getOperandIdx(Opcode: MI.getOpcode(),
93 Op: R600::OpName::pred_sel);
94 int MovPredSelIdx = TII->getOperandIdx(Opcode: Mov->getOpcode(),
95 Op: R600::OpName::pred_sel);
96 // Copy the pred_sel bit
97 Mov->getOperand(i: MovPredSelIdx).setReg(
98 MI.getOperand(i: LDSPredSelIdx).getReg());
99 }
100
101 switch (MI.getOpcode()) {
102 default: break;
103 // Expand PRED_X to one of the PRED_SET instructions.
104 case R600::PRED_X: {
105 uint64_t Flags = MI.getOperand(i: 3).getImm();
106 // The native opcode used by PRED_X is stored as an immediate in the
107 // third operand.
108 MachineInstr *PredSet = TII->buildDefaultInstruction(MBB, I,
109 Opcode: MI.getOperand(i: 2).getImm(), // opcode
110 DstReg: MI.getOperand(i: 0).getReg(), // dst
111 Src0Reg: MI.getOperand(i: 1).getReg(), // src0
112 Src1Reg: R600::ZERO); // src1
113 TII->addFlag(MI&: *PredSet, Operand: 0, MO_FLAG_MASK);
114 if (Flags & MO_FLAG_PUSH) {
115 TII->setImmOperand(MI&: *PredSet, Op: R600::OpName::update_exec_mask, Imm: 1);
116 } else {
117 TII->setImmOperand(MI&: *PredSet, Op: R600::OpName::update_pred, Imm: 1);
118 }
119 MI.eraseFromParent();
120 continue;
121 }
122 case R600::DOT_4: {
123
124 const R600RegisterInfo &TRI = TII->getRegisterInfo();
125
126 Register DstReg = MI.getOperand(i: 0).getReg();
127 unsigned DstBase = TRI.getEncodingValue(RegNo: DstReg) & HW_REG_MASK;
128
129 for (unsigned Chan = 0; Chan < 4; ++Chan) {
130 bool Mask = (Chan != TRI.getHWRegChan(reg: DstReg));
131 unsigned SubDstReg =
132 R600::R600_TReg32RegClass.getRegister(i: (DstBase * 4) + Chan);
133 MachineInstr *BMI =
134 TII->buildSlotOfVectorInstruction(MBB, MI: &MI, Slot: Chan, DstReg: SubDstReg);
135 if (Chan > 0) {
136 BMI->bundleWithPred();
137 }
138 if (Mask) {
139 TII->addFlag(MI&: *BMI, Operand: 0, MO_FLAG_MASK);
140 }
141 if (Chan != 3)
142 TII->addFlag(MI&: *BMI, Operand: 0, MO_FLAG_NOT_LAST);
143 unsigned Opcode = BMI->getOpcode();
144 // While not strictly necessary from hw point of view, we force
145 // all src operands of a dot4 inst to belong to the same slot.
146 Register Src0 =
147 BMI->getOperand(i: TII->getOperandIdx(Opcode, Op: R600::OpName::src0))
148 .getReg();
149 Register Src1 =
150 BMI->getOperand(i: TII->getOperandIdx(Opcode, Op: R600::OpName::src1))
151 .getReg();
152 (void) Src0;
153 (void) Src1;
154 if ((TRI.getEncodingValue(RegNo: Src0) & 0xff) < 127 &&
155 (TRI.getEncodingValue(RegNo: Src1) & 0xff) < 127)
156 assert(TRI.getHWRegChan(Src0) == TRI.getHWRegChan(Src1));
157 }
158 MI.eraseFromParent();
159 continue;
160 }
161 }
162
163 bool IsReduction = TII->isReductionOp(opcode: MI.getOpcode());
164 bool IsVector = TII->isVector(MI);
165 bool IsCube = TII->isCubeOp(opcode: MI.getOpcode());
166 if (!IsReduction && !IsVector && !IsCube) {
167 continue;
168 }
169
170 // Expand the instruction
171 //
172 // Reduction instructions:
173 // T0_X = DP4 T1_XYZW, T2_XYZW
174 // becomes:
175 // TO_X = DP4 T1_X, T2_X
176 // TO_Y (write masked) = DP4 T1_Y, T2_Y
177 // TO_Z (write masked) = DP4 T1_Z, T2_Z
178 // TO_W (write masked) = DP4 T1_W, T2_W
179 //
180 // Vector instructions:
181 // T0_X = MULLO_INT T1_X, T2_X
182 // becomes:
183 // T0_X = MULLO_INT T1_X, T2_X
184 // T0_Y (write masked) = MULLO_INT T1_X, T2_X
185 // T0_Z (write masked) = MULLO_INT T1_X, T2_X
186 // T0_W (write masked) = MULLO_INT T1_X, T2_X
187 //
188 // Cube instructions:
189 // T0_XYZW = CUBE T1_XYZW
190 // becomes:
191 // TO_X = CUBE T1_Z, T1_Y
192 // T0_Y = CUBE T1_Z, T1_X
193 // T0_Z = CUBE T1_X, T1_Z
194 // T0_W = CUBE T1_Y, T1_Z
195 for (unsigned Chan = 0; Chan < 4; Chan++) {
196 Register DstReg =
197 MI.getOperand(i: TII->getOperandIdx(MI, Op: R600::OpName::dst)).getReg();
198 Register Src0 =
199 MI.getOperand(i: TII->getOperandIdx(MI, Op: R600::OpName::src0)).getReg();
200 unsigned Src1 = 0;
201
202 // Determine the correct source registers
203 if (!IsCube) {
204 int Src1Idx = TII->getOperandIdx(MI, Op: R600::OpName::src1);
205 if (Src1Idx != -1) {
206 Src1 = MI.getOperand(i: Src1Idx).getReg();
207 }
208 }
209 if (IsReduction) {
210 unsigned SubRegIndex = R600RegisterInfo::getSubRegFromChannel(Channel: Chan);
211 Src0 = TRI.getSubReg(Reg: Src0, Idx: SubRegIndex);
212 Src1 = TRI.getSubReg(Reg: Src1, Idx: SubRegIndex);
213 } else if (IsCube) {
214 static const int CubeSrcSwz[] = {2, 2, 0, 1};
215 unsigned SubRegIndex0 = R600RegisterInfo::getSubRegFromChannel(Channel: CubeSrcSwz[Chan]);
216 unsigned SubRegIndex1 = R600RegisterInfo::getSubRegFromChannel(Channel: CubeSrcSwz[3 - Chan]);
217 Src1 = TRI.getSubReg(Reg: Src0, Idx: SubRegIndex1);
218 Src0 = TRI.getSubReg(Reg: Src0, Idx: SubRegIndex0);
219 }
220
221 // Determine the correct destination registers;
222 bool Mask = false;
223 bool NotLast = true;
224 if (IsCube) {
225 unsigned SubRegIndex = R600RegisterInfo::getSubRegFromChannel(Channel: Chan);
226 DstReg = TRI.getSubReg(Reg: DstReg, Idx: SubRegIndex);
227 } else {
228 // Mask the write if the original instruction does not write to
229 // the current Channel.
230 Mask = (Chan != TRI.getHWRegChan(reg: DstReg));
231 unsigned DstBase = TRI.getEncodingValue(RegNo: DstReg) & HW_REG_MASK;
232 DstReg = R600::R600_TReg32RegClass.getRegister(i: (DstBase * 4) + Chan);
233 }
234
235 // Set the IsLast bit
236 NotLast = (Chan != 3 );
237
238 // Add the new instruction
239 unsigned Opcode = MI.getOpcode();
240 switch (Opcode) {
241 case R600::CUBE_r600_pseudo:
242 Opcode = R600::CUBE_r600_real;
243 break;
244 case R600::CUBE_eg_pseudo:
245 Opcode = R600::CUBE_eg_real;
246 break;
247 default:
248 break;
249 }
250
251 MachineInstr *NewMI =
252 TII->buildDefaultInstruction(MBB, I, Opcode, DstReg, Src0Reg: Src0, Src1Reg: Src1);
253
254 if (Chan != 0)
255 NewMI->bundleWithPred();
256 if (Mask) {
257 TII->addFlag(MI&: *NewMI, Operand: 0, MO_FLAG_MASK);
258 }
259 if (NotLast) {
260 TII->addFlag(MI&: *NewMI, Operand: 0, MO_FLAG_NOT_LAST);
261 }
262 SetFlagInNewMI(NewMI, OldMI: &MI, Op: R600::OpName::clamp);
263 SetFlagInNewMI(NewMI, OldMI: &MI, Op: R600::OpName::literal);
264 SetFlagInNewMI(NewMI, OldMI: &MI, Op: R600::OpName::src0_abs);
265 SetFlagInNewMI(NewMI, OldMI: &MI, Op: R600::OpName::src1_abs);
266 SetFlagInNewMI(NewMI, OldMI: &MI, Op: R600::OpName::src0_neg);
267 SetFlagInNewMI(NewMI, OldMI: &MI, Op: R600::OpName::src1_neg);
268 }
269 MI.eraseFromParent();
270 }
271 }
272 return false;
273}
274