1//===-- R600ISelLowering.cpp - R600 DAG Lowering Implementation -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Custom DAG lowering for R600
11//
12//===----------------------------------------------------------------------===//
13
14#include "R600ISelLowering.h"
15#include "AMDGPU.h"
16#include "AMDGPUSelectionDAGInfo.h"
17#include "MCTargetDesc/R600MCTargetDesc.h"
18#include "R600Defines.h"
19#include "R600MachineFunctionInfo.h"
20#include "R600Subtarget.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/IR/IntrinsicsAMDGPU.h"
23#include "llvm/IR/IntrinsicsR600.h"
24
25using namespace llvm;
26
27#define GET_CALLING_CONV_IMPL
28#include "R600GenCallingConv.inc"
29
30R600TargetLowering::R600TargetLowering(const TargetMachine &TM,
31 const R600Subtarget &STI)
32 : AMDGPUTargetLowering(TM, STI, STI), Subtarget(&STI),
33 Gen(STI.getGeneration()) {
34 addRegisterClass(VT: MVT::f32, RC: &R600::R600_Reg32RegClass);
35 addRegisterClass(VT: MVT::i32, RC: &R600::R600_Reg32RegClass);
36 addRegisterClass(VT: MVT::v2f32, RC: &R600::R600_Reg64RegClass);
37 addRegisterClass(VT: MVT::v2i32, RC: &R600::R600_Reg64RegClass);
38 addRegisterClass(VT: MVT::v4f32, RC: &R600::R600_Reg128RegClass);
39 addRegisterClass(VT: MVT::v4i32, RC: &R600::R600_Reg128RegClass);
40
41 setBooleanContents(ZeroOrNegativeOneBooleanContent);
42 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
43
44 computeRegisterProperties(TRI: Subtarget->getRegisterInfo());
45
46 // Legalize loads and stores to the private address space.
47 setOperationAction(Ops: ISD::LOAD, VTs: {MVT::i32, MVT::v2i32, MVT::v4i32}, Action: Custom);
48
49 // EXTLOAD should be the same as ZEXTLOAD. It is legal for some address
50 // spaces, so it is custom lowered to handle those where it isn't.
51 for (auto Op : {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD})
52 for (MVT VT : MVT::integer_valuetypes()) {
53 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i1, Action: Promote);
54 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i8, Action: Custom);
55 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i16, Action: Custom);
56 }
57
58 // Workaround for LegalizeDAG asserting on expansion of i1 vector loads.
59 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::v2i32,
60 MemVT: MVT::v2i1, Action: Expand);
61
62 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::v4i32,
63 MemVT: MVT::v4i1, Action: Expand);
64
65 setOperationAction(Ops: ISD::STORE, VTs: {MVT::i8, MVT::i32, MVT::v2i32, MVT::v4i32},
66 Action: Custom);
67
68 setTruncStoreAction(ValVT: MVT::i32, MemVT: MVT::i8, Action: Custom);
69 setTruncStoreAction(ValVT: MVT::i32, MemVT: MVT::i16, Action: Custom);
70 // We need to include these since trunc STORES to PRIVATE need
71 // special handling to accommodate RMW
72 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i16, Action: Custom);
73 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i16, Action: Custom);
74 setTruncStoreAction(ValVT: MVT::v8i32, MemVT: MVT::v8i16, Action: Custom);
75 setTruncStoreAction(ValVT: MVT::v16i32, MemVT: MVT::v16i16, Action: Custom);
76 setTruncStoreAction(ValVT: MVT::v32i32, MemVT: MVT::v32i16, Action: Custom);
77 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i8, Action: Custom);
78 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i8, Action: Custom);
79 setTruncStoreAction(ValVT: MVT::v8i32, MemVT: MVT::v8i8, Action: Custom);
80 setTruncStoreAction(ValVT: MVT::v16i32, MemVT: MVT::v16i8, Action: Custom);
81 setTruncStoreAction(ValVT: MVT::v32i32, MemVT: MVT::v32i8, Action: Custom);
82
83 // Workaround for LegalizeDAG asserting on expansion of i1 vector stores.
84 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i1, Action: Expand);
85 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i1, Action: Expand);
86
87 // Set condition code actions
88 setCondCodeAction(CCs: {ISD::SETO, ISD::SETUO, ISD::SETLT, ISD::SETLE, ISD::SETOLT,
89 ISD::SETOLE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGE,
90 ISD::SETUGT, ISD::SETULT, ISD::SETULE},
91 VT: MVT::f32, Action: Expand);
92
93 setCondCodeAction(CCs: {ISD::SETLE, ISD::SETLT, ISD::SETULE, ISD::SETULT},
94 VT: MVT::i32, Action: Expand);
95
96 setOperationAction(Ops: {ISD::FCOS, ISD::FSIN}, VT: MVT::f32, Action: Custom);
97
98 setOperationAction(Ops: ISD::SETCC, VTs: {MVT::v4i32, MVT::v2i32}, Action: Expand);
99
100 setOperationAction(Ops: ISD::BR_CC, VTs: {MVT::i32, MVT::f32}, Action: Expand);
101 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
102
103 setOperationAction(Op: ISD::FSUB, VT: MVT::f32, Action: Expand);
104
105 setOperationAction(Ops: ISD::IS_FPCLASS,
106 VTs: {MVT::f32, MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
107 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v16f32},
108 Action: Expand);
109
110 setOperationAction(Ops: {ISD::FCEIL, ISD::FTRUNC, ISD::FROUNDEVEN, ISD::FFLOOR},
111 VT: MVT::f64, Action: Custom);
112
113 setOperationAction(Op: ISD::FPOW, VT: MVT::f32, Action: Legal);
114
115 setOperationAction(Ops: ISD::SELECT_CC, VTs: {MVT::f32, MVT::i32}, Action: Custom);
116
117 setOperationAction(Ops: ISD::SETCC, VTs: {MVT::i32, MVT::f32}, Action: Expand);
118 setOperationAction(Ops: {ISD::FP_TO_UINT, ISD::FP_TO_SINT}, VTs: {MVT::i1, MVT::i64},
119 Action: Custom);
120
121 setOperationAction(Ops: ISD::SELECT, VTs: {MVT::i32, MVT::f32, MVT::v2i32, MVT::v4i32},
122 Action: Expand);
123
124 // ADD, SUB overflow.
125 // TODO: turn these into Legal?
126 if (Subtarget->hasCARRY())
127 setOperationAction(Op: ISD::UADDO, VT: MVT::i32, Action: Custom);
128
129 if (Subtarget->hasBORROW())
130 setOperationAction(Op: ISD::USUBO, VT: MVT::i32, Action: Custom);
131
132 // Expand sign extension of vectors
133 if (!Subtarget->hasBFE())
134 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
135
136 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG, VTs: {MVT::v2i1, MVT::v4i1}, Action: Expand);
137
138 if (!Subtarget->hasBFE())
139 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i8, Action: Expand);
140 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG, VTs: {MVT::v2i8, MVT::v4i8}, Action: Expand);
141
142 if (!Subtarget->hasBFE())
143 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i16, Action: Expand);
144 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG, VTs: {MVT::v2i16, MVT::v4i16}, Action: Expand);
145
146 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i32, Action: Legal);
147 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG, VTs: {MVT::v2i32, MVT::v4i32}, Action: Expand);
148
149 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::Other, Action: Expand);
150
151 setOperationAction(Op: ISD::FrameIndex, VT: MVT::i32, Action: Custom);
152
153 setOperationAction(Ops: ISD::EXTRACT_VECTOR_ELT,
154 VTs: {MVT::v2i32, MVT::v2f32, MVT::v4i32, MVT::v4f32}, Action: Custom);
155
156 setOperationAction(Ops: ISD::INSERT_VECTOR_ELT,
157 VTs: {MVT::v2i32, MVT::v2f32, MVT::v4i32, MVT::v4f32}, Action: Custom);
158
159 // We don't have 64-bit shifts. Thus we need either SHX i64 or SHX_PARTS i32
160 // to be Legal/Custom in order to avoid library calls.
161 setOperationAction(Ops: {ISD::SHL_PARTS, ISD::SRL_PARTS, ISD::SRA_PARTS}, VT: MVT::i32,
162 Action: Custom);
163
164 if (!Subtarget->hasFMA())
165 setOperationAction(Ops: ISD::FMA, VTs: {MVT::f32, MVT::f64}, Action: Expand);
166
167 // FIXME: May need no denormals check
168 setOperationAction(Op: ISD::FMAD, VT: MVT::f32, Action: Legal);
169
170 if (!Subtarget->hasBFI())
171 // fcopysign can be done in a single instruction with BFI.
172 setOperationAction(Ops: ISD::FCOPYSIGN, VTs: {MVT::f32, MVT::f64}, Action: Expand);
173
174 if (!Subtarget->hasBCNT(Size: 32))
175 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Expand);
176
177 if (!Subtarget->hasBCNT(Size: 64))
178 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Expand);
179
180 if (Subtarget->hasFFBH())
181 setOperationAction(Op: ISD::CTLZ_ZERO_POISON, VT: MVT::i32, Action: Custom);
182
183 if (Subtarget->hasFFBL())
184 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::i32, Action: Custom);
185
186 // FIXME: This was moved from AMDGPUTargetLowering, I'm not sure if we
187 // need it for R600.
188 if (Subtarget->hasBFE())
189 setHasExtractBitsInsn(true);
190
191 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i32, Action: Custom);
192 setOperationAction(Op: ISD::ADDRSPACECAST, VT: MVT::i32, Action: Custom);
193
194 // LLVM will expand these to atomic_cmp_swap(0)
195 // and atomic_swap, respectively.
196 setOperationAction(Ops: {ISD::ATOMIC_LOAD, ISD::ATOMIC_STORE}, VT: MVT::i32, Action: Expand);
197
198 // We need to custom lower some of the intrinsics
199 setOperationAction(Ops: {ISD::INTRINSIC_VOID, ISD::INTRINSIC_WO_CHAIN}, VT: MVT::Other,
200 Action: Custom);
201
202 setSchedulingPreference(Sched::Source);
203
204 setTargetDAGCombine({ISD::FP_ROUND, ISD::FP_TO_SINT, ISD::EXTRACT_VECTOR_ELT,
205 ISD::SELECT_CC, ISD::INSERT_VECTOR_ELT, ISD::LOAD});
206}
207
208static inline bool isEOP(MachineBasicBlock::iterator I) {
209 if (std::next(x: I) == I->getParent()->end())
210 return false;
211 return std::next(x: I)->getOpcode() == R600::RETURN;
212}
213
214MachineBasicBlock *
215R600TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
216 MachineBasicBlock *BB) const {
217 MachineFunction *MF = BB->getParent();
218 MachineRegisterInfo &MRI = MF->getRegInfo();
219 MachineBasicBlock::iterator I = MI;
220 const R600InstrInfo *TII = Subtarget->getInstrInfo();
221
222 switch (MI.getOpcode()) {
223 default:
224 // Replace LDS_*_RET instruction that don't have any uses with the
225 // equivalent LDS_*_NORET instruction.
226 if (TII->isLDSRetInstr(Opcode: MI.getOpcode())) {
227 int DstIdx = TII->getOperandIdx(Opcode: MI.getOpcode(), Op: R600::OpName::dst);
228 assert(DstIdx != -1);
229 MachineInstrBuilder NewMI;
230 // FIXME: getLDSNoRetOp method only handles LDS_1A1D LDS ops. Add
231 // LDS_1A2D support and remove this special case.
232 if (!MRI.use_empty(RegNo: MI.getOperand(i: DstIdx).getReg()) ||
233 MI.getOpcode() == R600::LDS_CMPST_RET)
234 return BB;
235
236 NewMI = BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I),
237 MCID: TII->get(Opcode: R600::getLDSNoRetOp(Opcode: MI.getOpcode())));
238 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands()))
239 NewMI.add(MO);
240 } else {
241 return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, MBB: BB);
242 }
243 break;
244
245 case R600::FABS_R600: {
246 MachineInstr *NewMI = TII->buildDefaultInstruction(
247 MBB&: *BB, I, Opcode: R600::MOV, DstReg: MI.getOperand(i: 0).getReg(),
248 Src0Reg: MI.getOperand(i: 1).getReg());
249 TII->addFlag(MI&: *NewMI, SrcIdx: 0, MO_FLAG_ABS);
250 break;
251 }
252
253 case R600::FNEG_R600: {
254 MachineInstr *NewMI = TII->buildDefaultInstruction(
255 MBB&: *BB, I, Opcode: R600::MOV, DstReg: MI.getOperand(i: 0).getReg(),
256 Src0Reg: MI.getOperand(i: 1).getReg());
257 TII->addFlag(MI&: *NewMI, SrcIdx: 0, MO_FLAG_NEG);
258 break;
259 }
260
261 case R600::MASK_WRITE: {
262 Register maskedRegister = MI.getOperand(i: 0).getReg();
263 assert(maskedRegister.isVirtual());
264 MachineInstr * defInstr = MRI.getVRegDef(Reg: maskedRegister);
265 TII->addFlag(MI&: *defInstr, SrcIdx: 0, MO_FLAG_MASK);
266 break;
267 }
268
269 case R600::MOV_IMM_F32:
270 TII->buildMovImm(BB&: *BB, I, DstReg: MI.getOperand(i: 0).getReg(), Imm: MI.getOperand(i: 1)
271 .getFPImm()
272 ->getValueAPF()
273 .bitcastToAPInt()
274 .getZExtValue());
275 break;
276
277 case R600::MOV_IMM_I32:
278 TII->buildMovImm(BB&: *BB, I, DstReg: MI.getOperand(i: 0).getReg(),
279 Imm: MI.getOperand(i: 1).getImm());
280 break;
281
282 case R600::MOV_IMM_GLOBAL_ADDR: {
283 //TODO: Perhaps combine this instruction with the next if possible
284 auto MIB = TII->buildDefaultInstruction(
285 MBB&: *BB, I: MI, Opcode: R600::MOV, DstReg: MI.getOperand(i: 0).getReg(), Src0Reg: R600::ALU_LITERAL_X);
286 int Idx = TII->getOperandIdx(MI: *MIB, Op: R600::OpName::literal);
287 //TODO: Ugh this is rather ugly
288 const MachineOperand &MO = MI.getOperand(i: 1);
289 MIB->getOperand(i: Idx).ChangeToGA(GV: MO.getGlobal(), Offset: MO.getOffset(),
290 TargetFlags: MO.getTargetFlags());
291 break;
292 }
293
294 case R600::CONST_COPY: {
295 MachineInstr *NewMI = TII->buildDefaultInstruction(
296 MBB&: *BB, I: MI, Opcode: R600::MOV, DstReg: MI.getOperand(i: 0).getReg(), Src0Reg: R600::ALU_CONST);
297 TII->setImmOperand(MI&: *NewMI, Op: R600::OpName::src0_sel,
298 Imm: MI.getOperand(i: 1).getImm());
299 break;
300 }
301
302 case R600::RAT_WRITE_CACHELESS_32_eg:
303 case R600::RAT_WRITE_CACHELESS_64_eg:
304 case R600::RAT_WRITE_CACHELESS_128_eg:
305 BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I), MCID: TII->get(Opcode: MI.getOpcode()))
306 .add(MO: MI.getOperand(i: 0))
307 .add(MO: MI.getOperand(i: 1))
308 .addImm(Val: isEOP(I)); // Set End of program bit
309 break;
310
311 case R600::RAT_STORE_TYPED_eg:
312 BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I), MCID: TII->get(Opcode: MI.getOpcode()))
313 .add(MO: MI.getOperand(i: 0))
314 .add(MO: MI.getOperand(i: 1))
315 .add(MO: MI.getOperand(i: 2))
316 .addImm(Val: isEOP(I)); // Set End of program bit
317 break;
318
319 case R600::BRANCH:
320 BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I), MCID: TII->get(Opcode: R600::JUMP))
321 .add(MO: MI.getOperand(i: 0));
322 break;
323
324 case R600::BRANCH_COND_f32: {
325 MachineInstr *NewMI =
326 BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I), MCID: TII->get(Opcode: R600::PRED_X),
327 DestReg: R600::PREDICATE_BIT)
328 .add(MO: MI.getOperand(i: 1))
329 .addImm(Val: R600::PRED_SETNE)
330 .addImm(Val: 0); // Flags
331 TII->addFlag(MI&: *NewMI, SrcIdx: 0, MO_FLAG_PUSH);
332 BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I), MCID: TII->get(Opcode: R600::JUMP_COND))
333 .add(MO: MI.getOperand(i: 0))
334 .addReg(RegNo: R600::PREDICATE_BIT, Flags: RegState::Kill);
335 break;
336 }
337
338 case R600::BRANCH_COND_i32: {
339 MachineInstr *NewMI =
340 BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I), MCID: TII->get(Opcode: R600::PRED_X),
341 DestReg: R600::PREDICATE_BIT)
342 .add(MO: MI.getOperand(i: 1))
343 .addImm(Val: R600::PRED_SETNE_INT)
344 .addImm(Val: 0); // Flags
345 TII->addFlag(MI&: *NewMI, SrcIdx: 0, MO_FLAG_PUSH);
346 BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I), MCID: TII->get(Opcode: R600::JUMP_COND))
347 .add(MO: MI.getOperand(i: 0))
348 .addReg(RegNo: R600::PREDICATE_BIT, Flags: RegState::Kill);
349 break;
350 }
351
352 case R600::EG_ExportSwz:
353 case R600::R600_ExportSwz: {
354 // Instruction is left unmodified if its not the last one of its type
355 bool isLastInstructionOfItsType = true;
356 unsigned InstExportType = MI.getOperand(i: 1).getImm();
357 for (MachineBasicBlock::iterator NextExportInst = std::next(x: I),
358 EndBlock = BB->end(); NextExportInst != EndBlock;
359 NextExportInst = std::next(x: NextExportInst)) {
360 if (NextExportInst->getOpcode() == R600::EG_ExportSwz ||
361 NextExportInst->getOpcode() == R600::R600_ExportSwz) {
362 unsigned CurrentInstExportType = NextExportInst->getOperand(i: 1)
363 .getImm();
364 if (CurrentInstExportType == InstExportType) {
365 isLastInstructionOfItsType = false;
366 break;
367 }
368 }
369 }
370 bool EOP = isEOP(I);
371 if (!EOP && !isLastInstructionOfItsType)
372 return BB;
373 unsigned CfInst = (MI.getOpcode() == R600::EG_ExportSwz) ? 84 : 40;
374 BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I), MCID: TII->get(Opcode: MI.getOpcode()))
375 .add(MO: MI.getOperand(i: 0))
376 .add(MO: MI.getOperand(i: 1))
377 .add(MO: MI.getOperand(i: 2))
378 .add(MO: MI.getOperand(i: 3))
379 .add(MO: MI.getOperand(i: 4))
380 .add(MO: MI.getOperand(i: 5))
381 .add(MO: MI.getOperand(i: 6))
382 .addImm(Val: CfInst)
383 .addImm(Val: EOP);
384 break;
385 }
386 case R600::RETURN: {
387 return BB;
388 }
389 }
390
391 MI.eraseFromParent();
392 return BB;
393}
394
395//===----------------------------------------------------------------------===//
396// Custom DAG Lowering Operations
397//===----------------------------------------------------------------------===//
398
399SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
400 MachineFunction &MF = DAG.getMachineFunction();
401 R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
402 switch (Op.getOpcode()) {
403 default: return AMDGPUTargetLowering::LowerOperation(Op, DAG);
404 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
405 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
406 case ISD::SHL_PARTS:
407 case ISD::SRA_PARTS:
408 case ISD::SRL_PARTS: return LowerShiftParts(Op, DAG);
409 case ISD::UADDO: return LowerUADDSUBO(Op, DAG, mainop: ISD::ADD, ovf: AMDGPUISD::CARRY);
410 case ISD::USUBO: return LowerUADDSUBO(Op, DAG, mainop: ISD::SUB, ovf: AMDGPUISD::BORROW);
411 case ISD::FCOS:
412 case ISD::FSIN: return LowerTrig(Op, DAG);
413 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
414 case ISD::STORE: return LowerSTORE(Op, DAG);
415 case ISD::LOAD: {
416 SDValue Result = LowerLOAD(Op, DAG);
417 assert((!Result.getNode() ||
418 Result.getNode()->getNumValues() == 2) &&
419 "Load should return a value and a chain");
420 return Result;
421 }
422
423 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
424 case ISD::GlobalAddress: return LowerGlobalAddress(MFI, Op, DAG);
425 case ISD::FrameIndex: return lowerFrameIndex(Op, DAG);
426 case ISD::ADDRSPACECAST:
427 return lowerADDRSPACECAST(Op, DAG);
428 case ISD::INTRINSIC_VOID: {
429 SDValue Chain = Op.getOperand(i: 0);
430 unsigned IntrinsicID = Op.getConstantOperandVal(i: 1);
431 switch (IntrinsicID) {
432 case Intrinsic::r600_store_swizzle: {
433 SDLoc DL(Op);
434 const SDValue Args[8] = {
435 Chain,
436 Op.getOperand(i: 2), // Export Value
437 Op.getOperand(i: 3), // ArrayBase
438 Op.getOperand(i: 4), // Type
439 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // SWZ_X
440 DAG.getConstant(Val: 1, DL, VT: MVT::i32), // SWZ_Y
441 DAG.getConstant(Val: 2, DL, VT: MVT::i32), // SWZ_Z
442 DAG.getConstant(Val: 3, DL, VT: MVT::i32) // SWZ_W
443 };
444 return DAG.getNode(Opcode: AMDGPUISD::R600_EXPORT, DL, VT: Op.getValueType(), Ops: Args);
445 }
446
447 // default for switch(IntrinsicID)
448 default: break;
449 }
450 // break out of case ISD::INTRINSIC_VOID in switch(Op.getOpcode())
451 break;
452 }
453 case ISD::INTRINSIC_WO_CHAIN: {
454 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
455 EVT VT = Op.getValueType();
456 SDLoc DL(Op);
457 switch (IntrinsicID) {
458 case Intrinsic::r600_tex:
459 case Intrinsic::r600_texc: {
460 unsigned TextureOp;
461 switch (IntrinsicID) {
462 case Intrinsic::r600_tex:
463 TextureOp = 0;
464 break;
465 case Intrinsic::r600_texc:
466 TextureOp = 1;
467 break;
468 default:
469 llvm_unreachable("unhandled texture operation");
470 }
471
472 SDValue TexArgs[19] = {
473 DAG.getConstant(Val: TextureOp, DL, VT: MVT::i32),
474 Op.getOperand(i: 1),
475 DAG.getConstant(Val: 0, DL, VT: MVT::i32),
476 DAG.getConstant(Val: 1, DL, VT: MVT::i32),
477 DAG.getConstant(Val: 2, DL, VT: MVT::i32),
478 DAG.getConstant(Val: 3, DL, VT: MVT::i32),
479 Op.getOperand(i: 2),
480 Op.getOperand(i: 3),
481 Op.getOperand(i: 4),
482 DAG.getConstant(Val: 0, DL, VT: MVT::i32),
483 DAG.getConstant(Val: 1, DL, VT: MVT::i32),
484 DAG.getConstant(Val: 2, DL, VT: MVT::i32),
485 DAG.getConstant(Val: 3, DL, VT: MVT::i32),
486 Op.getOperand(i: 5),
487 Op.getOperand(i: 6),
488 Op.getOperand(i: 7),
489 Op.getOperand(i: 8),
490 Op.getOperand(i: 9),
491 Op.getOperand(i: 10)
492 };
493 return DAG.getNode(Opcode: AMDGPUISD::TEXTURE_FETCH, DL, VT: MVT::v4f32, Ops: TexArgs);
494 }
495 case Intrinsic::r600_dot4: {
496 SDValue Args[8] = {
497 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: Op.getOperand(i: 1),
498 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32)),
499 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: Op.getOperand(i: 2),
500 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32)),
501 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: Op.getOperand(i: 1),
502 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32)),
503 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: Op.getOperand(i: 2),
504 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32)),
505 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: Op.getOperand(i: 1),
506 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32)),
507 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: Op.getOperand(i: 2),
508 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32)),
509 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: Op.getOperand(i: 1),
510 N2: DAG.getConstant(Val: 3, DL, VT: MVT::i32)),
511 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: Op.getOperand(i: 2),
512 N2: DAG.getConstant(Val: 3, DL, VT: MVT::i32))
513 };
514 return DAG.getNode(Opcode: AMDGPUISD::DOT4, DL, VT: MVT::f32, Ops: Args);
515 }
516
517 case Intrinsic::r600_implicitarg_ptr: {
518 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout(), AS: AMDGPUAS::PARAM_I_ADDRESS);
519 uint32_t ByteOffset = getImplicitParameterOffset(MF, Param: FIRST_IMPLICIT);
520 return DAG.getConstant(Val: ByteOffset, DL, VT: PtrVT);
521 }
522 case Intrinsic::r600_read_ngroups_x:
523 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 0);
524 case Intrinsic::r600_read_ngroups_y:
525 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 1);
526 case Intrinsic::r600_read_ngroups_z:
527 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 2);
528 case Intrinsic::r600_read_global_size_x:
529 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 3);
530 case Intrinsic::r600_read_global_size_y:
531 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 4);
532 case Intrinsic::r600_read_global_size_z:
533 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 5);
534 case Intrinsic::r600_read_local_size_x:
535 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 6);
536 case Intrinsic::r600_read_local_size_y:
537 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 7);
538 case Intrinsic::r600_read_local_size_z:
539 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 8);
540
541 case Intrinsic::r600_read_tgid_x:
542 case Intrinsic::amdgcn_workgroup_id_x:
543 return CreateLiveInRegisterRaw(DAG, RC: &R600::R600_TReg32RegClass,
544 Reg: R600::T1_X, VT);
545 case Intrinsic::r600_read_tgid_y:
546 case Intrinsic::amdgcn_workgroup_id_y:
547 return CreateLiveInRegisterRaw(DAG, RC: &R600::R600_TReg32RegClass,
548 Reg: R600::T1_Y, VT);
549 case Intrinsic::r600_read_tgid_z:
550 case Intrinsic::amdgcn_workgroup_id_z:
551 return CreateLiveInRegisterRaw(DAG, RC: &R600::R600_TReg32RegClass,
552 Reg: R600::T1_Z, VT);
553 case Intrinsic::r600_read_tidig_x:
554 case Intrinsic::amdgcn_workitem_id_x:
555 return CreateLiveInRegisterRaw(DAG, RC: &R600::R600_TReg32RegClass,
556 Reg: R600::T0_X, VT);
557 case Intrinsic::r600_read_tidig_y:
558 case Intrinsic::amdgcn_workitem_id_y:
559 return CreateLiveInRegisterRaw(DAG, RC: &R600::R600_TReg32RegClass,
560 Reg: R600::T0_Y, VT);
561 case Intrinsic::r600_read_tidig_z:
562 case Intrinsic::amdgcn_workitem_id_z:
563 return CreateLiveInRegisterRaw(DAG, RC: &R600::R600_TReg32RegClass,
564 Reg: R600::T0_Z, VT);
565
566 case Intrinsic::r600_recipsqrt_ieee:
567 return DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: Op.getOperand(i: 1));
568
569 case Intrinsic::r600_recipsqrt_clamped:
570 return DAG.getNode(Opcode: AMDGPUISD::RSQ_CLAMP, DL, VT, Operand: Op.getOperand(i: 1));
571 default:
572 return Op;
573 }
574
575 // break out of case ISD::INTRINSIC_WO_CHAIN in switch(Op.getOpcode())
576 break;
577 }
578 } // end switch(Op.getOpcode())
579 return SDValue();
580}
581
582void R600TargetLowering::ReplaceNodeResults(SDNode *N,
583 SmallVectorImpl<SDValue> &Results,
584 SelectionDAG &DAG) const {
585 switch (N->getOpcode()) {
586 default:
587 AMDGPUTargetLowering::ReplaceNodeResults(N, Results, DAG);
588 return;
589 case ISD::FP_TO_UINT:
590 if (N->getValueType(ResNo: 0) == MVT::i1) {
591 Results.push_back(Elt: lowerFP_TO_UINT(Op: N->getOperand(Num: 0), DAG));
592 return;
593 }
594 // Since we don't care about out of bounds values we can use FP_TO_SINT for
595 // uints too. The DAGLegalizer code for uint considers some extra cases
596 // which are not necessary here.
597 [[fallthrough]];
598 case ISD::FP_TO_SINT: {
599 if (N->getValueType(ResNo: 0) == MVT::i1) {
600 Results.push_back(Elt: lowerFP_TO_SINT(Op: N->getOperand(Num: 0), DAG));
601 return;
602 }
603
604 SDValue Result;
605 if (expandFP_TO_SINT(N, Result, DAG))
606 Results.push_back(Elt: Result);
607 return;
608 }
609 case ISD::SDIVREM: {
610 SDValue Op = SDValue(N, 1);
611 SDValue RES = LowerSDIVREM(Op, DAG);
612 Results.push_back(Elt: RES);
613 Results.push_back(Elt: RES.getValue(R: 1));
614 break;
615 }
616 case ISD::UDIVREM: {
617 SDValue Op = SDValue(N, 0);
618 LowerUDIVREM64(Op, DAG, Results);
619 break;
620 }
621 }
622}
623
624SDValue R600TargetLowering::vectorToVerticalVector(SelectionDAG &DAG,
625 SDValue Vector) const {
626 SDLoc DL(Vector);
627 EVT VecVT = Vector.getValueType();
628 EVT EltVT = VecVT.getVectorElementType();
629 SmallVector<SDValue, 8> Args;
630
631 for (unsigned i = 0, e = VecVT.getVectorNumElements(); i != e; ++i) {
632 Args.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Vector,
633 N2: DAG.getVectorIdxConstant(Val: i, DL)));
634 }
635
636 return DAG.getNode(Opcode: AMDGPUISD::BUILD_VERTICAL_VECTOR, DL, VT: VecVT, Ops: Args);
637}
638
639SDValue R600TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
640 SelectionDAG &DAG) const {
641 SDLoc DL(Op);
642 SDValue Vector = Op.getOperand(i: 0);
643 SDValue Index = Op.getOperand(i: 1);
644
645 if (isa<ConstantSDNode>(Val: Index) ||
646 Vector.getOpcode() == AMDGPUISD::BUILD_VERTICAL_VECTOR)
647 return Op;
648
649 Vector = vectorToVerticalVector(DAG, Vector);
650 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: Op.getValueType(),
651 N1: Vector, N2: Index);
652}
653
654SDValue R600TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
655 SelectionDAG &DAG) const {
656 SDLoc DL(Op);
657 SDValue Vector = Op.getOperand(i: 0);
658 SDValue Value = Op.getOperand(i: 1);
659 SDValue Index = Op.getOperand(i: 2);
660
661 if (isa<ConstantSDNode>(Val: Index) ||
662 Vector.getOpcode() == AMDGPUISD::BUILD_VERTICAL_VECTOR)
663 return Op;
664
665 Vector = vectorToVerticalVector(DAG, Vector);
666 SDValue Insert = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: Op.getValueType(),
667 N1: Vector, N2: Value, N3: Index);
668 return vectorToVerticalVector(DAG, Vector: Insert);
669}
670
671SDValue R600TargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
672 SDValue Op,
673 SelectionDAG &DAG) const {
674 GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Val&: Op);
675 if (GSD->getAddressSpace() != AMDGPUAS::CONSTANT_ADDRESS)
676 return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG);
677
678 const DataLayout &DL = DAG.getDataLayout();
679 const GlobalValue *GV = GSD->getGlobal();
680 MVT ConstPtrVT = getPointerTy(DL, AS: AMDGPUAS::CONSTANT_ADDRESS);
681
682 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: SDLoc(GSD), VT: ConstPtrVT);
683 return DAG.getNode(Opcode: AMDGPUISD::CONST_DATA_PTR, DL: SDLoc(GSD), VT: ConstPtrVT, Operand: GA);
684}
685
686SDValue R600TargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
687 // On hw >= R700, COS/SIN input must be between -1. and 1.
688 // Thus we lower them to TRIG ( FRACT ( x / 2Pi + 0.5) - 0.5)
689 EVT VT = Op.getValueType();
690 SDValue Arg = Op.getOperand(i: 0);
691 SDLoc DL(Op);
692
693 // TODO: Should this propagate fast-math-flags?
694 SDValue FractPart = DAG.getNode(Opcode: AMDGPUISD::FRACT, DL, VT,
695 Operand: DAG.getNode(Opcode: ISD::FADD, DL, VT,
696 N1: DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Arg,
697 N2: DAG.getConstantFP(Val: 0.15915494309, DL, VT: MVT::f32)),
698 N2: DAG.getConstantFP(Val: 0.5, DL, VT: MVT::f32)));
699 unsigned TrigNode;
700 switch (Op.getOpcode()) {
701 case ISD::FCOS:
702 TrigNode = AMDGPUISD::COS_HW;
703 break;
704 case ISD::FSIN:
705 TrigNode = AMDGPUISD::SIN_HW;
706 break;
707 default:
708 llvm_unreachable("Wrong trig opcode");
709 }
710 SDValue TrigVal = DAG.getNode(Opcode: TrigNode, DL, VT,
711 Operand: DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: FractPart,
712 N2: DAG.getConstantFP(Val: -0.5, DL, VT: MVT::f32)));
713 if (Gen >= AMDGPUSubtarget::R700)
714 return TrigVal;
715 // On R600 hw, COS/SIN input must be between -Pi and Pi.
716 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: TrigVal,
717 N2: DAG.getConstantFP(Val: numbers::pif, DL, VT: MVT::f32));
718}
719
720SDValue R600TargetLowering::LowerShiftParts(SDValue Op,
721 SelectionDAG &DAG) const {
722 SDValue Lo, Hi;
723 expandShiftParts(N: Op.getNode(), Lo, Hi, DAG);
724 return DAG.getMergeValues(Ops: {Lo, Hi}, dl: SDLoc(Op));
725}
726
727SDValue R600TargetLowering::LowerUADDSUBO(SDValue Op, SelectionDAG &DAG,
728 unsigned mainop, unsigned ovf) const {
729 SDLoc DL(Op);
730 EVT VT = Op.getValueType();
731
732 SDValue Lo = Op.getOperand(i: 0);
733 SDValue Hi = Op.getOperand(i: 1);
734
735 SDValue OVF = DAG.getNode(Opcode: ovf, DL, VT, N1: Lo, N2: Hi);
736 // Extend sign.
737 OVF = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: OVF,
738 N2: DAG.getValueType(MVT::i1));
739
740 SDValue Res = DAG.getNode(Opcode: mainop, DL, VT, N1: Lo, N2: Hi);
741
742 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: Res, N2: OVF);
743}
744
745SDValue R600TargetLowering::lowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) const {
746 SDLoc DL(Op);
747 return DAG.getNode(
748 Opcode: ISD::SETCC,
749 DL,
750 VT: MVT::i1,
751 N1: Op, N2: DAG.getConstantFP(Val: 1.0f, DL, VT: MVT::f32),
752 N3: DAG.getCondCode(Cond: ISD::SETEQ));
753}
754
755SDValue R600TargetLowering::lowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const {
756 SDLoc DL(Op);
757 return DAG.getNode(
758 Opcode: ISD::SETCC,
759 DL,
760 VT: MVT::i1,
761 N1: Op, N2: DAG.getConstantFP(Val: -1.0f, DL, VT: MVT::f32),
762 N3: DAG.getCondCode(Cond: ISD::SETEQ));
763}
764
765SDValue R600TargetLowering::LowerImplicitParameter(SelectionDAG &DAG, EVT VT,
766 const SDLoc &DL,
767 unsigned DwordOffset) const {
768 unsigned ByteOffset = DwordOffset * 4;
769 PointerType *PtrType =
770 PointerType::get(C&: *DAG.getContext(), AddressSpace: AMDGPUAS::PARAM_I_ADDRESS);
771
772 // We shouldn't be using an offset wider than 16-bits for implicit parameters.
773 assert(isInt<16>(ByteOffset));
774
775 return DAG.getLoad(VT, dl: DL, Chain: DAG.getEntryNode(),
776 Ptr: DAG.getConstant(Val: ByteOffset, DL, VT: MVT::i32), // PTR
777 PtrInfo: MachinePointerInfo(ConstantPointerNull::get(T: PtrType)));
778}
779
780bool R600TargetLowering::isZero(SDValue Op) const {
781 if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Val&: Op))
782 return Cst->isZero();
783 if (ConstantFPSDNode *CstFP = dyn_cast<ConstantFPSDNode>(Val&: Op))
784 return CstFP->isZero();
785 return false;
786}
787
788bool R600TargetLowering::isHWTrueValue(SDValue Op) const {
789 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
790 return CFP->isOne();
791 }
792 return isAllOnesConstant(V: Op);
793}
794
795bool R600TargetLowering::isHWFalseValue(SDValue Op) const {
796 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
797 return CFP->getValueAPF().isZero();
798 }
799 return isNullConstant(V: Op);
800}
801
802SDValue R600TargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
803 SDLoc DL(Op);
804 EVT VT = Op.getValueType();
805
806 SDValue LHS = Op.getOperand(i: 0);
807 SDValue RHS = Op.getOperand(i: 1);
808 SDValue True = Op.getOperand(i: 2);
809 SDValue False = Op.getOperand(i: 3);
810 SDValue CC = Op.getOperand(i: 4);
811 SDValue Temp;
812
813 if (VT == MVT::f32) {
814 DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr);
815 SDValue MinMax = combineFMinMaxLegacy(DL, VT, LHS, RHS, True, False, CC,
816 Flags: SDNodeFlags(), DCI);
817 if (MinMax)
818 return MinMax;
819 }
820
821 // LHS and RHS are guaranteed to be the same value type
822 EVT CompareVT = LHS.getValueType();
823
824 // Check if we can lower this to a native operation.
825
826 // Try to lower to a SET* instruction:
827 //
828 // SET* can match the following patterns:
829 //
830 // select_cc f32, f32, -1, 0, cc_supported
831 // select_cc f32, f32, 1.0f, 0.0f, cc_supported
832 // select_cc i32, i32, -1, 0, cc_supported
833 //
834
835 // Move hardware True/False values to the correct operand.
836 if (isHWTrueValue(Op: False) && isHWFalseValue(Op: True)) {
837 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val&: CC)->get();
838 ISD::CondCode InverseCC = ISD::getSetCCInverse(Operation: CCOpcode, Type: CompareVT);
839 if (isCondCodeLegal(CC: InverseCC, VT: CompareVT.getSimpleVT())) {
840 std::swap(a&: False, b&: True);
841 CC = DAG.getCondCode(Cond: InverseCC);
842 } else {
843 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(Operation: InverseCC);
844 if (isCondCodeLegal(CC: SwapInvCC, VT: CompareVT.getSimpleVT())) {
845 std::swap(a&: False, b&: True);
846 std::swap(a&: LHS, b&: RHS);
847 CC = DAG.getCondCode(Cond: SwapInvCC);
848 }
849 }
850 }
851
852 if (isHWTrueValue(Op: True) && isHWFalseValue(Op: False) &&
853 (CompareVT == VT || VT == MVT::i32)) {
854 // This can be matched by a SET* instruction.
855 return DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT, N1: LHS, N2: RHS, N3: True, N4: False, N5: CC);
856 }
857
858 // Try to lower to a CND* instruction:
859 //
860 // CND* can match the following patterns:
861 //
862 // select_cc f32, 0.0, f32, f32, cc_supported
863 // select_cc f32, 0.0, i32, i32, cc_supported
864 // select_cc i32, 0, f32, f32, cc_supported
865 // select_cc i32, 0, i32, i32, cc_supported
866 //
867
868 // Try to move the zero value to the RHS
869 if (isZero(Op: LHS)) {
870 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val&: CC)->get();
871 // Try swapping the operands
872 ISD::CondCode CCSwapped = ISD::getSetCCSwappedOperands(Operation: CCOpcode);
873 if (isCondCodeLegal(CC: CCSwapped, VT: CompareVT.getSimpleVT())) {
874 std::swap(a&: LHS, b&: RHS);
875 CC = DAG.getCondCode(Cond: CCSwapped);
876 } else {
877 // Try inverting the condition and then swapping the operands
878 ISD::CondCode CCInv = ISD::getSetCCInverse(Operation: CCOpcode, Type: CompareVT);
879 CCSwapped = ISD::getSetCCSwappedOperands(Operation: CCInv);
880 if (isCondCodeLegal(CC: CCSwapped, VT: CompareVT.getSimpleVT())) {
881 std::swap(a&: True, b&: False);
882 std::swap(a&: LHS, b&: RHS);
883 CC = DAG.getCondCode(Cond: CCSwapped);
884 }
885 }
886 }
887 if (isZero(Op: RHS)) {
888 SDValue Cond = LHS;
889 SDValue Zero = RHS;
890 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val&: CC)->get();
891 if (CompareVT != VT) {
892 // Bitcast True / False to the correct types. This will end up being
893 // a nop, but it allows us to define only a single pattern in the
894 // .TD files for each CND* instruction rather than having to have
895 // one pattern for integer True/False and one for fp True/False
896 True = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: CompareVT, Operand: True);
897 False = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: CompareVT, Operand: False);
898 }
899
900 switch (CCOpcode) {
901 case ISD::SETONE:
902 case ISD::SETUNE:
903 case ISD::SETNE:
904 CCOpcode = ISD::getSetCCInverse(Operation: CCOpcode, Type: CompareVT);
905 Temp = True;
906 True = False;
907 False = Temp;
908 break;
909 default:
910 break;
911 }
912 SDValue SelectNode = DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT: CompareVT,
913 N1: Cond, N2: Zero,
914 N3: True, N4: False,
915 N5: DAG.getCondCode(Cond: CCOpcode));
916 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: SelectNode);
917 }
918
919 // If we make it this for it means we have no native instructions to handle
920 // this SELECT_CC, so we must lower it.
921 SDValue HWTrue, HWFalse;
922
923 if (CompareVT == MVT::f32) {
924 HWTrue = DAG.getConstantFP(Val: 1.0f, DL, VT: CompareVT);
925 HWFalse = DAG.getConstantFP(Val: 0.0f, DL, VT: CompareVT);
926 } else if (CompareVT == MVT::i32) {
927 HWTrue = DAG.getAllOnesConstant(DL, VT: CompareVT);
928 HWFalse = DAG.getConstant(Val: 0, DL, VT: CompareVT);
929 }
930 else {
931 llvm_unreachable("Unhandled value type in LowerSELECT_CC");
932 }
933
934 // Lower this unsupported SELECT_CC into a combination of two supported
935 // SELECT_CC operations.
936 SDValue Cond = DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT: CompareVT, N1: LHS, N2: RHS, N3: HWTrue, N4: HWFalse, N5: CC);
937
938 return DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT,
939 N1: Cond, N2: HWFalse,
940 N3: True, N4: False,
941 N5: DAG.getCondCode(Cond: ISD::SETNE));
942}
943
944SDValue R600TargetLowering::lowerADDRSPACECAST(SDValue Op,
945 SelectionDAG &DAG) const {
946 SDLoc SL(Op);
947 EVT VT = Op.getValueType();
948
949 const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Val&: Op);
950 unsigned SrcAS = ASC->getSrcAddressSpace();
951 unsigned DestAS = ASC->getDestAddressSpace();
952
953 if (isNullConstant(V: Op.getOperand(i: 0)) && SrcAS == AMDGPUAS::FLAT_ADDRESS)
954 return DAG.getSignedConstant(Val: AMDGPU::getNullPointerValue(AS: DestAS), DL: SL, VT);
955
956 return Op;
957}
958
959/// LLVM generates byte-addressed pointers. For indirect addressing, we need to
960/// convert these pointers to a register index. Each register holds
961/// 16 bytes, (4 x 32bit sub-register), but we need to take into account the
962/// \p StackWidth, which tells us how many of the 4 sub-registers will be used
963/// for indirect addressing.
964SDValue R600TargetLowering::stackPtrToRegIndex(SDValue Ptr,
965 unsigned StackWidth,
966 SelectionDAG &DAG) const {
967 unsigned SRLPad;
968 switch(StackWidth) {
969 case 1:
970 SRLPad = 2;
971 break;
972 case 2:
973 SRLPad = 3;
974 break;
975 case 4:
976 SRLPad = 4;
977 break;
978 default: llvm_unreachable("Invalid stack width");
979 }
980
981 SDLoc DL(Ptr);
982 return DAG.getNode(Opcode: ISD::SRL, DL, VT: Ptr.getValueType(), N1: Ptr,
983 N2: DAG.getConstant(Val: SRLPad, DL, VT: MVT::i32));
984}
985
986void R600TargetLowering::getStackAddress(unsigned StackWidth,
987 unsigned ElemIdx,
988 unsigned &Channel,
989 unsigned &PtrIncr) const {
990 switch (StackWidth) {
991 default:
992 case 1:
993 Channel = 0;
994 if (ElemIdx > 0) {
995 PtrIncr = 1;
996 } else {
997 PtrIncr = 0;
998 }
999 break;
1000 case 2:
1001 Channel = ElemIdx % 2;
1002 if (ElemIdx == 2) {
1003 PtrIncr = 1;
1004 } else {
1005 PtrIncr = 0;
1006 }
1007 break;
1008 case 4:
1009 Channel = ElemIdx;
1010 PtrIncr = 0;
1011 break;
1012 }
1013}
1014
1015SDValue R600TargetLowering::lowerPrivateTruncStore(StoreSDNode *Store,
1016 SelectionDAG &DAG) const {
1017 SDLoc DL(Store);
1018 //TODO: Who creates the i8 stores?
1019 assert(Store->isTruncatingStore()
1020 || Store->getValue().getValueType() == MVT::i8);
1021 assert(Store->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS);
1022
1023 SDValue Mask;
1024 if (Store->getMemoryVT() == MVT::i8) {
1025 assert(Store->getAlign() >= 1);
1026 Mask = DAG.getConstant(Val: 0xff, DL, VT: MVT::i32);
1027 } else if (Store->getMemoryVT() == MVT::i16) {
1028 assert(Store->getAlign() >= 2);
1029 Mask = DAG.getConstant(Val: 0xffff, DL, VT: MVT::i32);
1030 } else {
1031 llvm_unreachable("Unsupported private trunc store");
1032 }
1033
1034 SDValue OldChain = Store->getChain();
1035 bool VectorTrunc = (OldChain.getOpcode() == AMDGPUISD::DUMMY_CHAIN);
1036 // Skip dummy
1037 SDValue Chain = VectorTrunc ? OldChain->getOperand(Num: 0) : OldChain;
1038 SDValue BasePtr = Store->getBasePtr();
1039 SDValue Offset = Store->getOffset();
1040 EVT MemVT = Store->getMemoryVT();
1041
1042 SDValue LoadPtr = BasePtr;
1043 if (!Offset.isUndef()) {
1044 LoadPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: BasePtr, N2: Offset);
1045 }
1046
1047 // Get dword location
1048 // TODO: this should be eliminated by the future SHR ptr, 2
1049 SDValue Ptr = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: LoadPtr,
1050 N2: DAG.getConstant(Val: 0xfffffffc, DL, VT: MVT::i32));
1051
1052 // Load dword
1053 // TODO: can we be smarter about machine pointer info?
1054 MachinePointerInfo PtrInfo(AMDGPUAS::PRIVATE_ADDRESS);
1055 SDValue Dst = DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr, PtrInfo);
1056
1057 Chain = Dst.getValue(R: 1);
1058
1059 // Get offset in dword
1060 SDValue ByteIdx = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: LoadPtr,
1061 N2: DAG.getConstant(Val: 0x3, DL, VT: MVT::i32));
1062
1063 // Convert byte offset to bit shift
1064 SDValue ShiftAmt = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: ByteIdx,
1065 N2: DAG.getConstant(Val: 3, DL, VT: MVT::i32));
1066
1067 // TODO: Contrary to the name of the function,
1068 // it also handles sub i32 non-truncating stores (like i1)
1069 SDValue SExtValue = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i32,
1070 Operand: Store->getValue());
1071
1072 // Mask the value to the right type
1073 SDValue MaskedValue = DAG.getZeroExtendInReg(Op: SExtValue, DL, VT: MemVT);
1074
1075 // Shift the value in place
1076 SDValue ShiftedValue = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32,
1077 N1: MaskedValue, N2: ShiftAmt);
1078
1079 // Shift the mask in place
1080 SDValue DstMask = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: Mask, N2: ShiftAmt);
1081
1082 // Invert the mask. NOTE: if we had native ROL instructions we could
1083 // use inverted mask
1084 DstMask = DAG.getNOT(DL, Val: DstMask, VT: MVT::i32);
1085
1086 // Cleanup the target bits
1087 Dst = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Dst, N2: DstMask);
1088
1089 // Add the new bits
1090 SDValue Value = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Dst, N2: ShiftedValue);
1091
1092 // Store dword
1093 // TODO: Can we be smarter about MachinePointerInfo?
1094 SDValue NewStore = DAG.getStore(Chain, dl: DL, Val: Value, Ptr, PtrInfo);
1095
1096 // If we are part of expanded vector, make our neighbors depend on this store
1097 if (VectorTrunc) {
1098 // Make all other vector elements depend on this store
1099 Chain = DAG.getNode(Opcode: AMDGPUISD::DUMMY_CHAIN, DL, VT: MVT::Other, Operand: NewStore);
1100 DAG.ReplaceAllUsesOfValueWith(From: OldChain, To: Chain);
1101 }
1102 return NewStore;
1103}
1104
1105SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
1106 StoreSDNode *StoreNode = cast<StoreSDNode>(Val&: Op);
1107 unsigned AS = StoreNode->getAddressSpace();
1108
1109 SDValue Chain = StoreNode->getChain();
1110 SDValue Ptr = StoreNode->getBasePtr();
1111 SDValue Value = StoreNode->getValue();
1112
1113 EVT VT = Value.getValueType();
1114 EVT MemVT = StoreNode->getMemoryVT();
1115 EVT PtrVT = Ptr.getValueType();
1116
1117 SDLoc DL(Op);
1118
1119 const bool TruncatingStore = StoreNode->isTruncatingStore();
1120
1121 // Neither LOCAL nor PRIVATE can do vectors at the moment
1122 if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS ||
1123 TruncatingStore) &&
1124 VT.isVector()) {
1125 if ((AS == AMDGPUAS::PRIVATE_ADDRESS) && TruncatingStore) {
1126 // Add an extra level of chain to isolate this vector
1127 SDValue NewChain = DAG.getNode(Opcode: AMDGPUISD::DUMMY_CHAIN, DL, VT: MVT::Other, Operand: Chain);
1128 SmallVector<SDValue, 4> NewOps(StoreNode->ops());
1129 NewOps[0] = NewChain;
1130 StoreNode = cast<StoreSDNode>(Val: DAG.UpdateNodeOperands(N: StoreNode, Ops: NewOps));
1131 }
1132
1133 return scalarizeVectorStore(ST: StoreNode, DAG);
1134 }
1135
1136 Align Alignment = StoreNode->getAlign();
1137 if (Alignment < MemVT.getStoreSize() &&
1138 !allowsMisalignedMemoryAccesses(VT: MemVT, AS, Alignment,
1139 Flags: StoreNode->getMemOperand()->getFlags(),
1140 IsFast: nullptr)) {
1141 return expandUnalignedStore(ST: StoreNode, DAG);
1142 }
1143
1144 SDValue DWordAddr = DAG.getNode(Opcode: ISD::SRL, DL, VT: PtrVT, N1: Ptr,
1145 N2: DAG.getConstant(Val: 2, DL, VT: PtrVT));
1146
1147 if (AS == AMDGPUAS::GLOBAL_ADDRESS) {
1148 // It is beneficial to create MSKOR here instead of combiner to avoid
1149 // artificial dependencies introduced by RMW
1150 if (TruncatingStore) {
1151 assert(VT.bitsLE(MVT::i32));
1152 SDValue MaskConstant;
1153 if (MemVT == MVT::i8) {
1154 MaskConstant = DAG.getConstant(Val: 0xFF, DL, VT: MVT::i32);
1155 } else {
1156 assert(MemVT == MVT::i16);
1157 assert(StoreNode->getAlign() >= 2);
1158 MaskConstant = DAG.getConstant(Val: 0xFFFF, DL, VT: MVT::i32);
1159 }
1160
1161 SDValue ByteIndex = DAG.getNode(Opcode: ISD::AND, DL, VT: PtrVT, N1: Ptr,
1162 N2: DAG.getConstant(Val: 0x00000003, DL, VT: PtrVT));
1163 SDValue BitShift = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ByteIndex,
1164 N2: DAG.getConstant(Val: 3, DL, VT));
1165
1166 // Put the mask in correct place
1167 SDValue Mask = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: MaskConstant, N2: BitShift);
1168
1169 // Put the value bits in correct place
1170 SDValue TruncValue = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Value, N2: MaskConstant);
1171 SDValue ShiftedValue = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: TruncValue, N2: BitShift);
1172
1173 // XXX: If we add a 64-bit ZW register class, then we could use a 2 x i32
1174 // vector instead.
1175 SDValue Src[4] = {
1176 ShiftedValue,
1177 DAG.getConstant(Val: 0, DL, VT: MVT::i32),
1178 DAG.getConstant(Val: 0, DL, VT: MVT::i32),
1179 Mask
1180 };
1181 SDValue Input = DAG.getBuildVector(VT: MVT::v4i32, DL, Ops: Src);
1182 SDValue Args[3] = { Chain, Input, DWordAddr };
1183 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::STORE_MSKOR, dl: DL,
1184 VTList: Op->getVTList(), Ops: Args, MemVT,
1185 MMO: StoreNode->getMemOperand());
1186 }
1187 if (Ptr->getOpcode() != AMDGPUISD::DWORDADDR && VT.bitsGE(VT: MVT::i32)) {
1188 // Convert pointer from byte address to dword address.
1189 Ptr = DAG.getNode(Opcode: AMDGPUISD::DWORDADDR, DL, VT: PtrVT, Operand: DWordAddr);
1190
1191 if (StoreNode->isIndexed()) {
1192 llvm_unreachable("Indexed stores not supported yet");
1193 } else {
1194 Chain = DAG.getStore(Chain, dl: DL, Val: Value, Ptr, MMO: StoreNode->getMemOperand());
1195 }
1196 return Chain;
1197 }
1198 }
1199
1200 // GLOBAL_ADDRESS has been handled above, LOCAL_ADDRESS allows all sizes
1201 if (AS != AMDGPUAS::PRIVATE_ADDRESS)
1202 return SDValue();
1203
1204 if (MemVT.bitsLT(VT: MVT::i32))
1205 return lowerPrivateTruncStore(Store: StoreNode, DAG);
1206
1207 // Standard i32+ store, tag it with DWORDADDR to note that the address
1208 // has been shifted
1209 if (Ptr.getOpcode() != AMDGPUISD::DWORDADDR) {
1210 Ptr = DAG.getNode(Opcode: AMDGPUISD::DWORDADDR, DL, VT: PtrVT, Operand: DWordAddr);
1211 return DAG.getStore(Chain, dl: DL, Val: Value, Ptr, MMO: StoreNode->getMemOperand());
1212 }
1213
1214 // Tagged i32+ stores will be matched by patterns
1215 return SDValue();
1216}
1217
1218// return (512 + (kc_bank << 12)
1219static int
1220ConstantAddressBlock(unsigned AddressSpace) {
1221 switch (AddressSpace) {
1222 case AMDGPUAS::CONSTANT_BUFFER_0:
1223 return 512;
1224 case AMDGPUAS::CONSTANT_BUFFER_1:
1225 return 512 + 4096;
1226 case AMDGPUAS::CONSTANT_BUFFER_2:
1227 return 512 + 4096 * 2;
1228 case AMDGPUAS::CONSTANT_BUFFER_3:
1229 return 512 + 4096 * 3;
1230 case AMDGPUAS::CONSTANT_BUFFER_4:
1231 return 512 + 4096 * 4;
1232 case AMDGPUAS::CONSTANT_BUFFER_5:
1233 return 512 + 4096 * 5;
1234 case AMDGPUAS::CONSTANT_BUFFER_6:
1235 return 512 + 4096 * 6;
1236 case AMDGPUAS::CONSTANT_BUFFER_7:
1237 return 512 + 4096 * 7;
1238 case AMDGPUAS::CONSTANT_BUFFER_8:
1239 return 512 + 4096 * 8;
1240 case AMDGPUAS::CONSTANT_BUFFER_9:
1241 return 512 + 4096 * 9;
1242 case AMDGPUAS::CONSTANT_BUFFER_10:
1243 return 512 + 4096 * 10;
1244 case AMDGPUAS::CONSTANT_BUFFER_11:
1245 return 512 + 4096 * 11;
1246 case AMDGPUAS::CONSTANT_BUFFER_12:
1247 return 512 + 4096 * 12;
1248 case AMDGPUAS::CONSTANT_BUFFER_13:
1249 return 512 + 4096 * 13;
1250 case AMDGPUAS::CONSTANT_BUFFER_14:
1251 return 512 + 4096 * 14;
1252 case AMDGPUAS::CONSTANT_BUFFER_15:
1253 return 512 + 4096 * 15;
1254 default:
1255 return -1;
1256 }
1257}
1258
1259SDValue R600TargetLowering::lowerPrivateExtLoad(SDValue Op,
1260 SelectionDAG &DAG) const {
1261 SDLoc DL(Op);
1262 LoadSDNode *Load = cast<LoadSDNode>(Val&: Op);
1263 ISD::LoadExtType ExtType = Load->getExtensionType();
1264 EVT MemVT = Load->getMemoryVT();
1265 assert(Load->getAlign() >= MemVT.getStoreSize());
1266
1267 SDValue BasePtr = Load->getBasePtr();
1268 SDValue Chain = Load->getChain();
1269 SDValue Offset = Load->getOffset();
1270
1271 SDValue LoadPtr = BasePtr;
1272 if (!Offset.isUndef()) {
1273 LoadPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: BasePtr, N2: Offset);
1274 }
1275
1276 // Get dword location
1277 // NOTE: this should be eliminated by the future SHR ptr, 2
1278 SDValue Ptr = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: LoadPtr,
1279 N2: DAG.getConstant(Val: 0xfffffffc, DL, VT: MVT::i32));
1280
1281 // Load dword
1282 // TODO: can we be smarter about machine pointer info?
1283 MachinePointerInfo PtrInfo(AMDGPUAS::PRIVATE_ADDRESS);
1284 SDValue Read = DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr, PtrInfo);
1285
1286 // Get offset within the register.
1287 SDValue ByteIdx = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32,
1288 N1: LoadPtr, N2: DAG.getConstant(Val: 0x3, DL, VT: MVT::i32));
1289
1290 // Bit offset of target byte (byteIdx * 8).
1291 SDValue ShiftAmt = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: ByteIdx,
1292 N2: DAG.getConstant(Val: 3, DL, VT: MVT::i32));
1293
1294 // Shift to the right.
1295 SDValue Ret = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Read, N2: ShiftAmt);
1296
1297 // Eliminate the upper bits by setting them to ...
1298 EVT MemEltVT = MemVT.getScalarType();
1299
1300 if (ExtType == ISD::SEXTLOAD) { // ... ones.
1301 SDValue MemEltVTNode = DAG.getValueType(MemEltVT);
1302 Ret = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i32, N1: Ret, N2: MemEltVTNode);
1303 } else { // ... or zeros.
1304 Ret = DAG.getZeroExtendInReg(Op: Ret, DL, VT: MemEltVT);
1305 }
1306
1307 SDValue Ops[] = {
1308 Ret,
1309 Read.getValue(R: 1) // This should be our output chain
1310 };
1311
1312 return DAG.getMergeValues(Ops, dl: DL);
1313}
1314
1315SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
1316 LoadSDNode *LoadNode = cast<LoadSDNode>(Val&: Op);
1317 unsigned AS = LoadNode->getAddressSpace();
1318 EVT MemVT = LoadNode->getMemoryVT();
1319 ISD::LoadExtType ExtType = LoadNode->getExtensionType();
1320
1321 if (AS == AMDGPUAS::PRIVATE_ADDRESS &&
1322 ExtType != ISD::NON_EXTLOAD && MemVT.bitsLT(VT: MVT::i32)) {
1323 return lowerPrivateExtLoad(Op, DAG);
1324 }
1325
1326 SDLoc DL(Op);
1327 EVT VT = Op.getValueType();
1328 SDValue Chain = LoadNode->getChain();
1329 SDValue Ptr = LoadNode->getBasePtr();
1330
1331 if ((LoadNode->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
1332 LoadNode->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) &&
1333 VT.isVector()) {
1334 SDValue Ops[2];
1335 std::tie(args&: Ops[0], args&: Ops[1]) = scalarizeVectorLoad(LD: LoadNode, DAG);
1336 return DAG.getMergeValues(Ops, dl: DL);
1337 }
1338
1339 // This is still used for explicit load from addrspace(8)
1340 int ConstantBlock = ConstantAddressBlock(AddressSpace: LoadNode->getAddressSpace());
1341 if (ConstantBlock > -1 &&
1342 ((LoadNode->getExtensionType() == ISD::NON_EXTLOAD) ||
1343 (LoadNode->getExtensionType() == ISD::ZEXTLOAD))) {
1344 SDValue Result;
1345 if (isa<Constant>(Val: LoadNode->getMemOperand()->getValue()) ||
1346 isa<ConstantSDNode>(Val: Ptr)) {
1347 return constBufferLoad(LoadNode, Block: LoadNode->getAddressSpace(), DAG);
1348 }
1349 // TODO: Does this even work?
1350 // non-constant ptr can't be folded, keeps it as a v4f32 load
1351 Result = DAG.getNode(Opcode: AMDGPUISD::CONST_ADDRESS, DL, VT: MVT::v4i32,
1352 N1: DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Ptr,
1353 N2: DAG.getConstant(Val: 4, DL, VT: MVT::i32)),
1354 N2: DAG.getConstant(Val: LoadNode->getAddressSpace() -
1355 AMDGPUAS::CONSTANT_BUFFER_0,
1356 DL, VT: MVT::i32));
1357
1358 if (!VT.isVector()) {
1359 Result = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Result,
1360 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
1361 }
1362
1363 SDValue MergedValues[2] = {
1364 Result,
1365 Chain
1366 };
1367 return DAG.getMergeValues(Ops: MergedValues, dl: DL);
1368 }
1369
1370 // For most operations returning SDValue() will result in the node being
1371 // expanded by the DAG Legalizer. This is not the case for ISD::LOAD, so we
1372 // need to manually expand loads that may be legal in some address spaces and
1373 // illegal in others. SEXT loads from CONSTANT_BUFFER_0 are supported for
1374 // compute shaders, since the data is sign extended when it is uploaded to the
1375 // buffer. However SEXT loads from other address spaces are not supported, so
1376 // we need to expand them here.
1377 if (LoadNode->getExtensionType() == ISD::SEXTLOAD) {
1378 assert(!MemVT.isVector() && (MemVT == MVT::i16 || MemVT == MVT::i8));
1379 SDValue NewLoad = DAG.getExtLoad(
1380 ExtType: ISD::EXTLOAD, dl: DL, VT, Chain, Ptr, PtrInfo: LoadNode->getPointerInfo(), MemVT,
1381 Alignment: LoadNode->getAlign(), MMOFlags: LoadNode->getMemOperand()->getFlags());
1382 SDValue Res = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: NewLoad,
1383 N2: DAG.getValueType(MemVT));
1384
1385 SDValue MergedValues[2] = { Res, Chain };
1386 return DAG.getMergeValues(Ops: MergedValues, dl: DL);
1387 }
1388
1389 if (LoadNode->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) {
1390 return SDValue();
1391 }
1392
1393 // DWORDADDR ISD marks already shifted address
1394 if (Ptr.getOpcode() != AMDGPUISD::DWORDADDR) {
1395 assert(VT == MVT::i32);
1396 Ptr = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Ptr, N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32));
1397 Ptr = DAG.getNode(Opcode: AMDGPUISD::DWORDADDR, DL, VT: MVT::i32, Operand: Ptr);
1398 return DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr, MMO: LoadNode->getMemOperand());
1399 }
1400 return SDValue();
1401}
1402
1403SDValue R600TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
1404 SDValue Chain = Op.getOperand(i: 0);
1405 SDValue Cond = Op.getOperand(i: 1);
1406 SDValue Jump = Op.getOperand(i: 2);
1407
1408 return DAG.getNode(Opcode: AMDGPUISD::BRANCH_COND, DL: SDLoc(Op), VT: Op.getValueType(),
1409 N1: Chain, N2: Jump, N3: Cond);
1410}
1411
1412SDValue R600TargetLowering::lowerFrameIndex(SDValue Op,
1413 SelectionDAG &DAG) const {
1414 MachineFunction &MF = DAG.getMachineFunction();
1415 const R600FrameLowering *TFL = Subtarget->getFrameLowering();
1416
1417 FrameIndexSDNode *FIN = cast<FrameIndexSDNode>(Val&: Op);
1418
1419 unsigned FrameIndex = FIN->getIndex();
1420 Register IgnoredFrameReg;
1421 StackOffset Offset =
1422 TFL->getFrameIndexReference(MF, FI: FrameIndex, FrameReg&: IgnoredFrameReg);
1423 return DAG.getConstant(Val: Offset.getFixed() * 4 * TFL->getStackWidth(MF),
1424 DL: SDLoc(Op), VT: Op.getValueType());
1425}
1426
1427CCAssignFn *R600TargetLowering::CCAssignFnForCall(CallingConv::ID CC,
1428 bool IsVarArg) const {
1429 switch (CC) {
1430 case CallingConv::AMDGPU_KERNEL:
1431 case CallingConv::SPIR_KERNEL:
1432 case CallingConv::C:
1433 case CallingConv::Fast:
1434 case CallingConv::Cold:
1435 llvm_unreachable("kernels should not be handled here");
1436 case CallingConv::AMDGPU_VS:
1437 case CallingConv::AMDGPU_GS:
1438 case CallingConv::AMDGPU_PS:
1439 case CallingConv::AMDGPU_CS:
1440 case CallingConv::AMDGPU_HS:
1441 case CallingConv::AMDGPU_ES:
1442 case CallingConv::AMDGPU_LS:
1443 return CC_R600;
1444 default:
1445 reportFatalUsageError(reason: "unsupported calling convention");
1446 }
1447}
1448
1449/// XXX Only kernel functions are supported, so we can assume for now that
1450/// every function is a kernel function, but in the future we should use
1451/// separate calling conventions for kernel and non-kernel functions.
1452SDValue R600TargetLowering::LowerFormalArguments(
1453 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1454 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1455 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1456 SmallVector<CCValAssign, 16> ArgLocs;
1457 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
1458 *DAG.getContext());
1459 MachineFunction &MF = DAG.getMachineFunction();
1460
1461 if (AMDGPU::isShader(CC: CallConv)) {
1462 CCInfo.AnalyzeFormalArguments(Ins, Fn: CCAssignFnForCall(CC: CallConv, IsVarArg: isVarArg));
1463 } else {
1464 analyzeFormalArgumentsCompute(State&: CCInfo, Ins);
1465 }
1466
1467 for (unsigned i = 0, e = Ins.size(); i < e; ++i) {
1468 CCValAssign &VA = ArgLocs[i];
1469 const ISD::InputArg &In = Ins[i];
1470 EVT VT = In.VT;
1471 EVT MemVT = VA.getLocVT();
1472 if (!VT.isVector() && MemVT.isVector()) {
1473 // Get load source type if scalarized.
1474 MemVT = MemVT.getVectorElementType();
1475 }
1476
1477 if (VT.isInteger() && !MemVT.isInteger())
1478 MemVT = MemVT.changeTypeToInteger();
1479
1480 if (AMDGPU::isShader(CC: CallConv)) {
1481 Register Reg = MF.addLiveIn(PReg: VA.getLocReg(), RC: &R600::R600_Reg128RegClass);
1482 SDValue Register = DAG.getCopyFromReg(Chain, dl: DL, Reg, VT);
1483 InVals.push_back(Elt: Register);
1484 continue;
1485 }
1486
1487 // i64 isn't a legal type, so the register type used ends up as i32, which
1488 // isn't expected here. It attempts to create this sextload, but it ends up
1489 // being invalid. Somehow this seems to work with i64 arguments, but breaks
1490 // for <1 x i64>.
1491
1492 // The first 36 bytes of the input buffer contains information about
1493 // thread group and global sizes.
1494 ISD::LoadExtType Ext = ISD::NON_EXTLOAD;
1495 if (MemVT.getScalarSizeInBits() != VT.getScalarSizeInBits()) {
1496 if (VT.isFloatingPoint()) {
1497 Ext = ISD::EXTLOAD;
1498 } else {
1499 // FIXME: This should really check the extload type, but the handling of
1500 // extload vector parameters seems to be broken.
1501
1502 // Ext = In.Flags.isSExt() ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
1503 Ext = ISD::SEXTLOAD;
1504 }
1505 }
1506
1507 // Compute the offset from the value.
1508 // XXX - I think PartOffset should give you this, but it seems to give the
1509 // size of the register which isn't useful.
1510
1511 unsigned PartOffset = VA.getLocMemOffset();
1512 Align Alignment = commonAlignment(A: Align(VT.getStoreSize()), Offset: PartOffset);
1513
1514 MachinePointerInfo PtrInfo(AMDGPUAS::PARAM_I_ADDRESS);
1515 SDValue Arg =
1516 DAG.getLoad(AM: ISD::UNINDEXED, ExtType: Ext, VT, dl: DL, Chain,
1517 Ptr: DAG.getConstant(Val: PartOffset, DL, VT: MVT::i32),
1518 Offset: DAG.getPOISON(VT: MVT::i32), PtrInfo, MemVT, Alignment,
1519 MMOFlags: MachineMemOperand::MONonTemporal |
1520 MachineMemOperand::MODereferenceable |
1521 MachineMemOperand::MOInvariant);
1522
1523 InVals.push_back(Elt: Arg);
1524 }
1525 return Chain;
1526}
1527
1528EVT R600TargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &,
1529 EVT VT) const {
1530 if (!VT.isVector())
1531 return MVT::i32;
1532 return VT.changeVectorElementTypeToInteger();
1533}
1534
1535bool R600TargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
1536 const MachineFunction &MF) const {
1537 // Local and Private addresses do not handle vectors. Limit to i32
1538 if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS)) {
1539 return (MemVT.getSizeInBits() <= 32);
1540 }
1541 return true;
1542}
1543
1544bool R600TargetLowering::allowsMisalignedMemoryAccesses(
1545 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
1546 unsigned *IsFast) const {
1547 if (IsFast)
1548 *IsFast = 0;
1549
1550 if (!VT.isSimple() || VT == MVT::Other)
1551 return false;
1552
1553 if (VT.bitsLT(VT: MVT::i32))
1554 return false;
1555
1556 // TODO: This is a rough estimate.
1557 if (IsFast)
1558 *IsFast = 1;
1559
1560 return VT.bitsGT(VT: MVT::i32) && Alignment >= Align(4);
1561}
1562
1563static SDValue CompactSwizzlableVector(
1564 SelectionDAG &DAG, SDValue VectorEntry,
1565 DenseMap<unsigned, unsigned> &RemapSwizzle) {
1566 assert(RemapSwizzle.empty());
1567
1568 SDLoc DL(VectorEntry);
1569 EVT EltTy = VectorEntry.getValueType().getVectorElementType();
1570
1571 SDValue NewBldVec[4];
1572 for (unsigned i = 0; i < 4; i++)
1573 NewBldVec[i] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltTy, N1: VectorEntry,
1574 N2: DAG.getIntPtrConstant(Val: i, DL));
1575
1576 for (unsigned i = 0; i < 4; i++) {
1577 if (NewBldVec[i].isUndef())
1578 // We mask write here to teach later passes that the ith element of this
1579 // vector is undef. Thus we can use it to reduce 128 bits reg usage,
1580 // break false dependencies and additionally make assembly easier to read.
1581 RemapSwizzle[i] = 7; // SEL_MASK_WRITE
1582 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: NewBldVec[i])) {
1583 if (C->isZero()) {
1584 RemapSwizzle[i] = 4; // SEL_0
1585 NewBldVec[i] = DAG.getUNDEF(VT: MVT::f32);
1586 } else if (C->isOne()) {
1587 RemapSwizzle[i] = 5; // SEL_1
1588 NewBldVec[i] = DAG.getUNDEF(VT: MVT::f32);
1589 }
1590 }
1591
1592 if (NewBldVec[i].isUndef())
1593 continue;
1594
1595 for (unsigned j = 0; j < i; j++) {
1596 if (NewBldVec[i] == NewBldVec[j]) {
1597 NewBldVec[i] = DAG.getUNDEF(VT: NewBldVec[i].getValueType());
1598 RemapSwizzle[i] = j;
1599 break;
1600 }
1601 }
1602 }
1603
1604 return DAG.getBuildVector(VT: VectorEntry.getValueType(), DL: SDLoc(VectorEntry),
1605 Ops: NewBldVec);
1606}
1607
1608static SDValue ReorganizeVector(SelectionDAG &DAG, SDValue VectorEntry,
1609 DenseMap<unsigned, unsigned> &RemapSwizzle) {
1610 assert(RemapSwizzle.empty());
1611
1612 SDLoc DL(VectorEntry);
1613 EVT EltTy = VectorEntry.getValueType().getVectorElementType();
1614
1615 SDValue NewBldVec[4];
1616 bool isUnmovable[4] = {false, false, false, false};
1617 for (unsigned i = 0; i < 4; i++)
1618 NewBldVec[i] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltTy, N1: VectorEntry,
1619 N2: DAG.getIntPtrConstant(Val: i, DL));
1620
1621 for (unsigned i = 0; i < 4; i++) {
1622 RemapSwizzle[i] = i;
1623 if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1624 unsigned Idx = NewBldVec[i].getConstantOperandVal(i: 1);
1625 if (i == Idx)
1626 isUnmovable[Idx] = true;
1627 }
1628 }
1629
1630 for (unsigned i = 0; i < 4; i++) {
1631 if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1632 unsigned Idx = NewBldVec[i].getConstantOperandVal(i: 1);
1633 if (isUnmovable[Idx])
1634 continue;
1635 // Swap i and Idx
1636 std::swap(a&: NewBldVec[Idx], b&: NewBldVec[i]);
1637 std::swap(a&: RemapSwizzle[i], b&: RemapSwizzle[Idx]);
1638 break;
1639 }
1640 }
1641
1642 return DAG.getBuildVector(VT: VectorEntry.getValueType(), DL: SDLoc(VectorEntry),
1643 Ops: NewBldVec);
1644}
1645
1646SDValue R600TargetLowering::OptimizeSwizzle(SDValue BuildVector, SDValue Swz[],
1647 SelectionDAG &DAG,
1648 const SDLoc &DL) const {
1649 // Old -> New swizzle values
1650 DenseMap<unsigned, unsigned> SwizzleRemap;
1651
1652 BuildVector = CompactSwizzlableVector(DAG, VectorEntry: BuildVector, RemapSwizzle&: SwizzleRemap);
1653 for (unsigned i = 0; i < 4; i++) {
1654 unsigned Idx = Swz[i]->getAsZExtVal();
1655 auto It = SwizzleRemap.find(Val: Idx);
1656 if (It != SwizzleRemap.end())
1657 Swz[i] = DAG.getConstant(Val: It->second, DL, VT: MVT::i32);
1658 }
1659
1660 SwizzleRemap.clear();
1661 BuildVector = ReorganizeVector(DAG, VectorEntry: BuildVector, RemapSwizzle&: SwizzleRemap);
1662 for (unsigned i = 0; i < 4; i++) {
1663 unsigned Idx = Swz[i]->getAsZExtVal();
1664 auto It = SwizzleRemap.find(Val: Idx);
1665 if (It != SwizzleRemap.end())
1666 Swz[i] = DAG.getConstant(Val: It->second, DL, VT: MVT::i32);
1667 }
1668
1669 return BuildVector;
1670}
1671
1672SDValue R600TargetLowering::constBufferLoad(LoadSDNode *LoadNode, int Block,
1673 SelectionDAG &DAG) const {
1674 SDLoc DL(LoadNode);
1675 EVT VT = LoadNode->getValueType(ResNo: 0);
1676 SDValue Chain = LoadNode->getChain();
1677 SDValue Ptr = LoadNode->getBasePtr();
1678 assert (isa<ConstantSDNode>(Ptr));
1679
1680 //TODO: Support smaller loads
1681 if (LoadNode->getMemoryVT().getScalarType() != MVT::i32 || !ISD::isNON_EXTLoad(N: LoadNode))
1682 return SDValue();
1683
1684 if (LoadNode->getAlign() < Align(4))
1685 return SDValue();
1686
1687 int ConstantBlock = ConstantAddressBlock(AddressSpace: Block);
1688
1689 SDValue Slots[4];
1690 for (unsigned i = 0; i < 4; i++) {
1691 // We want Const position encoded with the following formula :
1692 // (((512 + (kc_bank << 12) + const_index) << 2) + chan)
1693 // const_index is Ptr computed by llvm using an alignment of 16.
1694 // Thus we add (((512 + (kc_bank << 12)) + chan ) * 4 here and
1695 // then div by 4 at the ISel step
1696 SDValue NewPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: Ptr,
1697 N2: DAG.getConstant(Val: 4 * i + ConstantBlock * 16, DL, VT: MVT::i32));
1698 Slots[i] = DAG.getNode(Opcode: AMDGPUISD::CONST_ADDRESS, DL, VT: MVT::i32, Operand: NewPtr);
1699 }
1700 EVT NewVT = MVT::v4i32;
1701 unsigned NumElements = 4;
1702 if (VT.isVector()) {
1703 NewVT = VT;
1704 NumElements = VT.getVectorNumElements();
1705 }
1706 SDValue Result = DAG.getBuildVector(VT: NewVT, DL, Ops: ArrayRef(Slots, NumElements));
1707 if (!VT.isVector()) {
1708 Result = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Result,
1709 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
1710 }
1711 SDValue MergedValues[2] = {
1712 Result,
1713 Chain
1714 };
1715 return DAG.getMergeValues(Ops: MergedValues, dl: DL);
1716}
1717
1718//===----------------------------------------------------------------------===//
1719// Custom DAG Optimizations
1720//===----------------------------------------------------------------------===//
1721
1722SDValue R600TargetLowering::PerformDAGCombine(SDNode *N,
1723 DAGCombinerInfo &DCI) const {
1724 SelectionDAG &DAG = DCI.DAG;
1725 SDLoc DL(N);
1726
1727 switch (N->getOpcode()) {
1728 // (f32 fp_round (f64 uint_to_fp a)) -> (f32 uint_to_fp a)
1729 case ISD::FP_ROUND: {
1730 SDValue Arg = N->getOperand(Num: 0);
1731 if (Arg.getOpcode() == ISD::UINT_TO_FP && Arg.getValueType() == MVT::f64) {
1732 return DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: N->getValueType(ResNo: 0),
1733 Operand: Arg.getOperand(i: 0));
1734 }
1735 break;
1736 }
1737
1738 // (i32 fp_to_sint (fneg (select_cc f32, f32, 1.0, 0.0 cc))) ->
1739 // (i32 select_cc f32, f32, -1, 0 cc)
1740 //
1741 // Mesa's GLSL frontend generates the above pattern a lot and we can lower
1742 // this to one of the SET*_DX10 instructions.
1743 case ISD::FP_TO_SINT: {
1744 SDValue FNeg = N->getOperand(Num: 0);
1745 if (FNeg.getOpcode() != ISD::FNEG) {
1746 return SDValue();
1747 }
1748 SDValue SelectCC = FNeg.getOperand(i: 0);
1749 if (SelectCC.getOpcode() != ISD::SELECT_CC ||
1750 SelectCC.getOperand(i: 0).getValueType() != MVT::f32 || // LHS
1751 SelectCC.getOperand(i: 2).getValueType() != MVT::f32 || // True
1752 !isHWTrueValue(Op: SelectCC.getOperand(i: 2)) ||
1753 !isHWFalseValue(Op: SelectCC.getOperand(i: 3))) {
1754 return SDValue();
1755 }
1756
1757 return DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT: N->getValueType(ResNo: 0),
1758 N1: SelectCC.getOperand(i: 0), // LHS
1759 N2: SelectCC.getOperand(i: 1), // RHS
1760 N3: DAG.getAllOnesConstant(DL, VT: MVT::i32), // True
1761 N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32), // False
1762 N5: SelectCC.getOperand(i: 4)); // CC
1763 }
1764
1765 // insert_vector_elt (build_vector elt0, ... , eltN), NewEltIdx, idx
1766 // => build_vector elt0, ... , NewEltIdx, ... , eltN
1767 case ISD::INSERT_VECTOR_ELT: {
1768 SDValue InVec = N->getOperand(Num: 0);
1769 SDValue InVal = N->getOperand(Num: 1);
1770 SDValue EltNo = N->getOperand(Num: 2);
1771
1772 // If the inserted element is an UNDEF, just use the input vector.
1773 if (InVal.isUndef())
1774 return InVec;
1775
1776 EVT VT = InVec.getValueType();
1777
1778 // If we can't generate a legal BUILD_VECTOR, exit
1779 if (!isOperationLegal(Op: ISD::BUILD_VECTOR, VT))
1780 return SDValue();
1781
1782 // Check that we know which element is being inserted
1783 if (!isa<ConstantSDNode>(Val: EltNo))
1784 return SDValue();
1785 unsigned Elt = EltNo->getAsZExtVal();
1786
1787 // Check that the operand is a BUILD_VECTOR (or UNDEF, which can essentially
1788 // be converted to a BUILD_VECTOR). Fill in the Ops vector with the
1789 // vector elements.
1790 SmallVector<SDValue, 8> Ops;
1791 if (InVec.getOpcode() == ISD::BUILD_VECTOR) {
1792 Ops.append(in_start: InVec.getNode()->op_begin(),
1793 in_end: InVec.getNode()->op_end());
1794 } else if (InVec.isUndef()) {
1795 unsigned NElts = VT.getVectorNumElements();
1796 Ops.append(NumInputs: NElts, Elt: DAG.getUNDEF(VT: InVal.getValueType()));
1797 } else {
1798 return SDValue();
1799 }
1800
1801 // Insert the element
1802 if (Elt < Ops.size()) {
1803 // All the operands of BUILD_VECTOR must have the same type;
1804 // we enforce that here.
1805 EVT OpVT = Ops[0].getValueType();
1806 if (InVal.getValueType() != OpVT)
1807 InVal = OpVT.bitsGT(VT: InVal.getValueType()) ?
1808 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: OpVT, Operand: InVal) :
1809 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OpVT, Operand: InVal);
1810 Ops[Elt] = InVal;
1811 }
1812
1813 // Return the new vector
1814 return DAG.getBuildVector(VT, DL, Ops);
1815 }
1816
1817 // Extract_vec (Build_vector) generated by custom lowering
1818 // also needs to be customly combined
1819 case ISD::EXTRACT_VECTOR_ELT: {
1820 SDValue Arg = N->getOperand(Num: 0);
1821 if (Arg.getOpcode() == ISD::BUILD_VECTOR) {
1822 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
1823 unsigned Element = Const->getZExtValue();
1824 return Arg->getOperand(Num: Element);
1825 }
1826 }
1827 if (Arg.getOpcode() == ISD::BITCAST &&
1828 Arg.getOperand(i: 0).getOpcode() == ISD::BUILD_VECTOR &&
1829 (Arg.getOperand(i: 0).getValueType().getVectorNumElements() ==
1830 Arg.getValueType().getVectorNumElements())) {
1831 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
1832 unsigned Element = Const->getZExtValue();
1833 return DAG.getNode(Opcode: ISD::BITCAST, DL, VTList: N->getVTList(),
1834 N: Arg->getOperand(Num: 0).getOperand(i: Element));
1835 }
1836 }
1837 break;
1838 }
1839
1840 case ISD::SELECT_CC: {
1841 // Try common optimizations
1842 if (SDValue Ret = AMDGPUTargetLowering::PerformDAGCombine(N, DCI))
1843 return Ret;
1844
1845 // fold selectcc (selectcc x, y, a, b, cc), b, a, b, seteq ->
1846 // selectcc x, y, a, b, inv(cc)
1847 //
1848 // fold selectcc (selectcc x, y, a, b, cc), b, a, b, setne ->
1849 // selectcc x, y, a, b, cc
1850 SDValue LHS = N->getOperand(Num: 0);
1851 if (LHS.getOpcode() != ISD::SELECT_CC) {
1852 return SDValue();
1853 }
1854
1855 SDValue RHS = N->getOperand(Num: 1);
1856 SDValue True = N->getOperand(Num: 2);
1857 SDValue False = N->getOperand(Num: 3);
1858 ISD::CondCode NCC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
1859
1860 if (LHS.getOperand(i: 2).getNode() != True.getNode() ||
1861 LHS.getOperand(i: 3).getNode() != False.getNode() ||
1862 RHS.getNode() != False.getNode()) {
1863 return SDValue();
1864 }
1865
1866 switch (NCC) {
1867 default: return SDValue();
1868 case ISD::SETNE: return LHS;
1869 case ISD::SETEQ: {
1870 ISD::CondCode LHSCC = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 4))->get();
1871 LHSCC = ISD::getSetCCInverse(Operation: LHSCC, Type: LHS.getOperand(i: 0).getValueType());
1872 if (DCI.isBeforeLegalizeOps() ||
1873 isCondCodeLegal(CC: LHSCC, VT: LHS.getOperand(i: 0).getSimpleValueType()))
1874 return DAG.getSelectCC(DL,
1875 LHS: LHS.getOperand(i: 0),
1876 RHS: LHS.getOperand(i: 1),
1877 True: LHS.getOperand(i: 2),
1878 False: LHS.getOperand(i: 3),
1879 Cond: LHSCC);
1880 break;
1881 }
1882 }
1883 return SDValue();
1884 }
1885
1886 case AMDGPUISD::R600_EXPORT: {
1887 SDValue Arg = N->getOperand(Num: 1);
1888 if (Arg.getOpcode() != ISD::BUILD_VECTOR)
1889 break;
1890
1891 SDValue NewArgs[8] = {
1892 N->getOperand(Num: 0), // Chain
1893 SDValue(),
1894 N->getOperand(Num: 2), // ArrayBase
1895 N->getOperand(Num: 3), // Type
1896 N->getOperand(Num: 4), // SWZ_X
1897 N->getOperand(Num: 5), // SWZ_Y
1898 N->getOperand(Num: 6), // SWZ_Z
1899 N->getOperand(Num: 7) // SWZ_W
1900 };
1901 NewArgs[1] = OptimizeSwizzle(BuildVector: N->getOperand(Num: 1), Swz: &NewArgs[4], DAG, DL);
1902 return DAG.getNode(Opcode: AMDGPUISD::R600_EXPORT, DL, VTList: N->getVTList(), Ops: NewArgs);
1903 }
1904 case AMDGPUISD::TEXTURE_FETCH: {
1905 SDValue Arg = N->getOperand(Num: 1);
1906 if (Arg.getOpcode() != ISD::BUILD_VECTOR)
1907 break;
1908
1909 SDValue NewArgs[19] = {
1910 N->getOperand(Num: 0),
1911 N->getOperand(Num: 1),
1912 N->getOperand(Num: 2),
1913 N->getOperand(Num: 3),
1914 N->getOperand(Num: 4),
1915 N->getOperand(Num: 5),
1916 N->getOperand(Num: 6),
1917 N->getOperand(Num: 7),
1918 N->getOperand(Num: 8),
1919 N->getOperand(Num: 9),
1920 N->getOperand(Num: 10),
1921 N->getOperand(Num: 11),
1922 N->getOperand(Num: 12),
1923 N->getOperand(Num: 13),
1924 N->getOperand(Num: 14),
1925 N->getOperand(Num: 15),
1926 N->getOperand(Num: 16),
1927 N->getOperand(Num: 17),
1928 N->getOperand(Num: 18),
1929 };
1930 NewArgs[1] = OptimizeSwizzle(BuildVector: N->getOperand(Num: 1), Swz: &NewArgs[2], DAG, DL);
1931 return DAG.getNode(Opcode: AMDGPUISD::TEXTURE_FETCH, DL, VTList: N->getVTList(), Ops: NewArgs);
1932 }
1933
1934 case ISD::LOAD: {
1935 LoadSDNode *LoadNode = cast<LoadSDNode>(Val: N);
1936 SDValue Ptr = LoadNode->getBasePtr();
1937 if (LoadNode->getAddressSpace() == AMDGPUAS::PARAM_I_ADDRESS &&
1938 isa<ConstantSDNode>(Val: Ptr))
1939 return constBufferLoad(LoadNode, Block: AMDGPUAS::CONSTANT_BUFFER_0, DAG);
1940 break;
1941 }
1942
1943 default: break;
1944 }
1945
1946 return AMDGPUTargetLowering::PerformDAGCombine(N, DCI);
1947}
1948
1949bool R600TargetLowering::FoldOperand(SDNode *ParentNode, unsigned SrcIdx,
1950 SDValue &Src, SDValue &Neg, SDValue &Abs,
1951 SDValue &Sel, SDValue &Imm,
1952 SelectionDAG &DAG) const {
1953 const R600InstrInfo *TII = Subtarget->getInstrInfo();
1954 if (!Src.isMachineOpcode())
1955 return false;
1956
1957 switch (Src.getMachineOpcode()) {
1958 case R600::FNEG_R600:
1959 if (!Neg.getNode())
1960 return false;
1961 Src = Src.getOperand(i: 0);
1962 Neg = DAG.getTargetConstant(Val: 1, DL: SDLoc(ParentNode), VT: MVT::i32);
1963 return true;
1964 case R600::FABS_R600:
1965 if (!Abs.getNode())
1966 return false;
1967 Src = Src.getOperand(i: 0);
1968 Abs = DAG.getTargetConstant(Val: 1, DL: SDLoc(ParentNode), VT: MVT::i32);
1969 return true;
1970 case R600::CONST_COPY: {
1971 unsigned Opcode = ParentNode->getMachineOpcode();
1972 bool HasDst = TII->getOperandIdx(Opcode, Op: R600::OpName::dst) > -1;
1973
1974 if (!Sel.getNode())
1975 return false;
1976
1977 SDValue CstOffset = Src.getOperand(i: 0);
1978 if (ParentNode->getValueType(ResNo: 0).isVector())
1979 return false;
1980
1981 // Gather constants values
1982 int SrcIndices[] = {
1983 TII->getOperandIdx(Opcode, Op: R600::OpName::src0),
1984 TII->getOperandIdx(Opcode, Op: R600::OpName::src1),
1985 TII->getOperandIdx(Opcode, Op: R600::OpName::src2),
1986 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_X),
1987 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_Y),
1988 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_Z),
1989 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_W),
1990 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_X),
1991 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_Y),
1992 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_Z),
1993 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_W)
1994 };
1995 std::vector<unsigned> Consts;
1996 for (int OtherSrcIdx : SrcIndices) {
1997 int OtherSelIdx = TII->getSelIdx(Opcode, SrcIdx: OtherSrcIdx);
1998 if (OtherSrcIdx < 0 || OtherSelIdx < 0)
1999 continue;
2000 if (HasDst) {
2001 OtherSrcIdx--;
2002 OtherSelIdx--;
2003 }
2004 if (RegisterSDNode *Reg =
2005 dyn_cast<RegisterSDNode>(Val: ParentNode->getOperand(Num: OtherSrcIdx))) {
2006 if (Reg->getReg() == R600::ALU_CONST) {
2007 Consts.push_back(x: ParentNode->getConstantOperandVal(Num: OtherSelIdx));
2008 }
2009 }
2010 }
2011
2012 ConstantSDNode *Cst = cast<ConstantSDNode>(Val&: CstOffset);
2013 Consts.push_back(x: Cst->getZExtValue());
2014 if (!TII->fitsConstReadLimitations(Consts)) {
2015 return false;
2016 }
2017
2018 Sel = CstOffset;
2019 Src = DAG.getRegister(Reg: R600::ALU_CONST, VT: MVT::f32);
2020 return true;
2021 }
2022 case R600::MOV_IMM_GLOBAL_ADDR:
2023 // Check if the Imm slot is used. Taken from below.
2024 if (Imm->getAsZExtVal())
2025 return false;
2026 Imm = Src.getOperand(i: 0);
2027 Src = DAG.getRegister(Reg: R600::ALU_LITERAL_X, VT: MVT::i32);
2028 return true;
2029 case R600::MOV_IMM_I32:
2030 case R600::MOV_IMM_F32: {
2031 unsigned ImmReg = R600::ALU_LITERAL_X;
2032 uint64_t ImmValue = 0;
2033
2034 if (Src.getMachineOpcode() == R600::MOV_IMM_F32) {
2035 ConstantFPSDNode *FPC = cast<ConstantFPSDNode>(Val: Src.getOperand(i: 0));
2036 float FloatValue = FPC->getValueAPF().convertToFloat();
2037 if (FloatValue == 0.0) {
2038 ImmReg = R600::ZERO;
2039 } else if (FloatValue == 0.5) {
2040 ImmReg = R600::HALF;
2041 } else if (FloatValue == 1.0) {
2042 ImmReg = R600::ONE;
2043 } else {
2044 ImmValue = FPC->getValueAPF().bitcastToAPInt().getZExtValue();
2045 }
2046 } else {
2047 uint64_t Value = Src.getConstantOperandVal(i: 0);
2048 if (Value == 0) {
2049 ImmReg = R600::ZERO;
2050 } else if (Value == 1) {
2051 ImmReg = R600::ONE_INT;
2052 } else {
2053 ImmValue = Value;
2054 }
2055 }
2056
2057 // Check that we aren't already using an immediate.
2058 // XXX: It's possible for an instruction to have more than one
2059 // immediate operand, but this is not supported yet.
2060 if (ImmReg == R600::ALU_LITERAL_X) {
2061 if (!Imm.getNode())
2062 return false;
2063 ConstantSDNode *C = cast<ConstantSDNode>(Val&: Imm);
2064 if (C->getZExtValue())
2065 return false;
2066 Imm = DAG.getTargetConstant(Val: ImmValue, DL: SDLoc(ParentNode), VT: MVT::i32);
2067 }
2068 Src = DAG.getRegister(Reg: ImmReg, VT: MVT::i32);
2069 return true;
2070 }
2071 default:
2072 return false;
2073 }
2074}
2075
2076/// Fold the instructions after selecting them
2077SDNode *R600TargetLowering::PostISelFolding(MachineSDNode *Node,
2078 SelectionDAG &DAG) const {
2079 const R600InstrInfo *TII = Subtarget->getInstrInfo();
2080 if (!Node->isMachineOpcode())
2081 return Node;
2082
2083 unsigned Opcode = Node->getMachineOpcode();
2084 SDValue FakeOp;
2085
2086 std::vector<SDValue> Ops(Node->op_begin(), Node->op_end());
2087
2088 if (Opcode == R600::DOT_4) {
2089 int OperandIdx[] = {
2090 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_X),
2091 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_Y),
2092 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_Z),
2093 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_W),
2094 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_X),
2095 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_Y),
2096 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_Z),
2097 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_W)
2098 };
2099 int NegIdx[] = {
2100 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_neg_X),
2101 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_neg_Y),
2102 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_neg_Z),
2103 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_neg_W),
2104 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_neg_X),
2105 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_neg_Y),
2106 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_neg_Z),
2107 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_neg_W)
2108 };
2109 int AbsIdx[] = {
2110 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_abs_X),
2111 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_abs_Y),
2112 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_abs_Z),
2113 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_abs_W),
2114 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_abs_X),
2115 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_abs_Y),
2116 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_abs_Z),
2117 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_abs_W)
2118 };
2119 for (unsigned i = 0; i < 8; i++) {
2120 if (OperandIdx[i] < 0)
2121 return Node;
2122 SDValue &Src = Ops[OperandIdx[i] - 1];
2123 SDValue &Neg = Ops[NegIdx[i] - 1];
2124 SDValue &Abs = Ops[AbsIdx[i] - 1];
2125 bool HasDst = TII->getOperandIdx(Opcode, Op: R600::OpName::dst) > -1;
2126 int SelIdx = TII->getSelIdx(Opcode, SrcIdx: OperandIdx[i]);
2127 if (HasDst)
2128 SelIdx--;
2129 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp;
2130 if (FoldOperand(ParentNode: Node, SrcIdx: i, Src, Neg, Abs, Sel, Imm&: FakeOp, DAG))
2131 return DAG.getMachineNode(Opcode, dl: SDLoc(Node), VTs: Node->getVTList(), Ops);
2132 }
2133 } else if (Opcode == R600::REG_SEQUENCE) {
2134 for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2) {
2135 SDValue &Src = Ops[i];
2136 if (FoldOperand(ParentNode: Node, SrcIdx: i, Src, Neg&: FakeOp, Abs&: FakeOp, Sel&: FakeOp, Imm&: FakeOp, DAG))
2137 return DAG.getMachineNode(Opcode, dl: SDLoc(Node), VTs: Node->getVTList(), Ops);
2138 }
2139 } else {
2140 if (!TII->hasInstrModifiers(Opcode))
2141 return Node;
2142 int OperandIdx[] = {
2143 TII->getOperandIdx(Opcode, Op: R600::OpName::src0),
2144 TII->getOperandIdx(Opcode, Op: R600::OpName::src1),
2145 TII->getOperandIdx(Opcode, Op: R600::OpName::src2)
2146 };
2147 int NegIdx[] = {
2148 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_neg),
2149 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_neg),
2150 TII->getOperandIdx(Opcode, Op: R600::OpName::src2_neg)
2151 };
2152 int AbsIdx[] = {
2153 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_abs),
2154 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_abs),
2155 -1
2156 };
2157 for (unsigned i = 0; i < 3; i++) {
2158 if (OperandIdx[i] < 0)
2159 return Node;
2160 SDValue &Src = Ops[OperandIdx[i] - 1];
2161 SDValue &Neg = Ops[NegIdx[i] - 1];
2162 SDValue FakeAbs;
2163 SDValue &Abs = (AbsIdx[i] > -1) ? Ops[AbsIdx[i] - 1] : FakeAbs;
2164 bool HasDst = TII->getOperandIdx(Opcode, Op: R600::OpName::dst) > -1;
2165 int SelIdx = TII->getSelIdx(Opcode, SrcIdx: OperandIdx[i]);
2166 int ImmIdx = TII->getOperandIdx(Opcode, Op: R600::OpName::literal);
2167 if (HasDst) {
2168 SelIdx--;
2169 ImmIdx--;
2170 }
2171 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp;
2172 SDValue &Imm = Ops[ImmIdx];
2173 if (FoldOperand(ParentNode: Node, SrcIdx: i, Src, Neg, Abs, Sel, Imm, DAG))
2174 return DAG.getMachineNode(Opcode, dl: SDLoc(Node), VTs: Node->getVTList(), Ops);
2175 }
2176 }
2177
2178 return Node;
2179}
2180
2181TargetLowering::AtomicExpansionKind
2182R600TargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
2183 switch (RMW->getOperation()) {
2184 case AtomicRMWInst::Nand:
2185 case AtomicRMWInst::FAdd:
2186 case AtomicRMWInst::FSub:
2187 case AtomicRMWInst::FMax:
2188 case AtomicRMWInst::FMin:
2189 case AtomicRMWInst::USubCond:
2190 case AtomicRMWInst::USubSat:
2191 return AtomicExpansionKind::CmpXChg;
2192 case AtomicRMWInst::UIncWrap:
2193 case AtomicRMWInst::UDecWrap:
2194 // FIXME: Cayman at least appears to have instructions for this, but the
2195 // instruction definitions appear to be missing.
2196 return AtomicExpansionKind::CmpXChg;
2197 case AtomicRMWInst::Xchg: {
2198 const DataLayout &DL = RMW->getFunction()->getDataLayout();
2199 unsigned ValSize = DL.getTypeSizeInBits(Ty: RMW->getType());
2200 if (ValSize == 32 || ValSize == 64)
2201 return AtomicExpansionKind::None;
2202 return AtomicExpansionKind::CmpXChg;
2203 }
2204 default:
2205 if (auto *IntTy = dyn_cast<IntegerType>(Val: RMW->getType())) {
2206 unsigned Size = IntTy->getBitWidth();
2207 if (Size == 32 || Size == 64)
2208 return AtomicExpansionKind::None;
2209 }
2210
2211 return AtomicExpansionKind::CmpXChg;
2212 }
2213
2214 llvm_unreachable("covered atomicrmw op switch");
2215}
2216