1//===-- R600ISelLowering.cpp - R600 DAG Lowering Implementation -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Custom DAG lowering for R600
11//
12//===----------------------------------------------------------------------===//
13
14#include "R600ISelLowering.h"
15#include "AMDGPU.h"
16#include "AMDGPUSelectionDAGInfo.h"
17#include "MCTargetDesc/R600MCTargetDesc.h"
18#include "R600Defines.h"
19#include "R600MachineFunctionInfo.h"
20#include "R600Subtarget.h"
21#include "R600TargetMachine.h"
22#include "llvm/CodeGen/MachineFunction.h"
23#include "llvm/IR/IntrinsicsAMDGPU.h"
24#include "llvm/IR/IntrinsicsR600.h"
25#include "llvm/Passes/CodeGenPassBuilder.h"
26
27using namespace llvm;
28
29#include "R600GenCallingConv.inc"
30
31R600TargetLowering::R600TargetLowering(const TargetMachine &TM,
32 const R600Subtarget &STI)
33 : AMDGPUTargetLowering(TM, STI, STI), Subtarget(&STI),
34 Gen(STI.getGeneration()) {
35 addRegisterClass(VT: MVT::f32, RC: &R600::R600_Reg32RegClass);
36 addRegisterClass(VT: MVT::i32, RC: &R600::R600_Reg32RegClass);
37 addRegisterClass(VT: MVT::v2f32, RC: &R600::R600_Reg64RegClass);
38 addRegisterClass(VT: MVT::v2i32, RC: &R600::R600_Reg64RegClass);
39 addRegisterClass(VT: MVT::v4f32, RC: &R600::R600_Reg128RegClass);
40 addRegisterClass(VT: MVT::v4i32, RC: &R600::R600_Reg128RegClass);
41
42 setBooleanContents(ZeroOrNegativeOneBooleanContent);
43 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
44
45 computeRegisterProperties(TRI: Subtarget->getRegisterInfo());
46
47 // Legalize loads and stores to the private address space.
48 setOperationAction(Ops: ISD::LOAD, VTs: {MVT::i32, MVT::v2i32, MVT::v4i32}, Action: Custom);
49
50 // EXTLOAD should be the same as ZEXTLOAD. It is legal for some address
51 // spaces, so it is custom lowered to handle those where it isn't.
52 for (auto Op : {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD})
53 for (MVT VT : MVT::integer_valuetypes()) {
54 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i1, Action: Promote);
55 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i8, Action: Custom);
56 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i16, Action: Custom);
57 }
58
59 // Workaround for LegalizeDAG asserting on expansion of i1 vector loads.
60 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::v2i32,
61 MemVT: MVT::v2i1, Action: Expand);
62
63 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::v4i32,
64 MemVT: MVT::v4i1, Action: Expand);
65
66 setOperationAction(Ops: ISD::STORE, VTs: {MVT::i8, MVT::i32, MVT::v2i32, MVT::v4i32},
67 Action: Custom);
68
69 setTruncStoreAction(ValVT: MVT::i32, MemVT: MVT::i8, Action: Custom);
70 setTruncStoreAction(ValVT: MVT::i32, MemVT: MVT::i16, Action: Custom);
71 // We need to include these since trunc STORES to PRIVATE need
72 // special handling to accommodate RMW
73 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i16, Action: Custom);
74 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i16, Action: Custom);
75 setTruncStoreAction(ValVT: MVT::v8i32, MemVT: MVT::v8i16, Action: Custom);
76 setTruncStoreAction(ValVT: MVT::v16i32, MemVT: MVT::v16i16, Action: Custom);
77 setTruncStoreAction(ValVT: MVT::v32i32, MemVT: MVT::v32i16, Action: Custom);
78 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i8, Action: Custom);
79 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i8, Action: Custom);
80 setTruncStoreAction(ValVT: MVT::v8i32, MemVT: MVT::v8i8, Action: Custom);
81 setTruncStoreAction(ValVT: MVT::v16i32, MemVT: MVT::v16i8, Action: Custom);
82 setTruncStoreAction(ValVT: MVT::v32i32, MemVT: MVT::v32i8, Action: Custom);
83
84 // Workaround for LegalizeDAG asserting on expansion of i1 vector stores.
85 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i1, Action: Expand);
86 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i1, Action: Expand);
87
88 // Set condition code actions
89 setCondCodeAction(CCs: {ISD::SETO, ISD::SETUO, ISD::SETLT, ISD::SETLE, ISD::SETOLT,
90 ISD::SETOLE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGE,
91 ISD::SETUGT, ISD::SETULT, ISD::SETULE},
92 VT: MVT::f32, Action: Expand);
93
94 setCondCodeAction(CCs: {ISD::SETLE, ISD::SETLT, ISD::SETULE, ISD::SETULT},
95 VT: MVT::i32, Action: Expand);
96
97 setOperationAction(Ops: {ISD::FCOS, ISD::FSIN}, VT: MVT::f32, Action: Custom);
98
99 setOperationAction(Ops: ISD::SETCC, VTs: {MVT::v4i32, MVT::v2i32}, Action: Expand);
100
101 setOperationAction(Ops: ISD::BR_CC, VTs: {MVT::i32, MVT::f32}, Action: Expand);
102 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
103
104 setOperationAction(Op: ISD::FSUB, VT: MVT::f32, Action: Expand);
105
106 setOperationAction(Ops: ISD::IS_FPCLASS,
107 VTs: {MVT::f32, MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
108 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v16f32},
109 Action: Expand);
110
111 setOperationAction(Ops: {ISD::FCEIL, ISD::FTRUNC, ISD::FROUNDEVEN, ISD::FFLOOR},
112 VT: MVT::f64, Action: Custom);
113
114 setOperationAction(Ops: ISD::SELECT_CC, VTs: {MVT::f32, MVT::i32}, Action: Custom);
115
116 setOperationAction(Ops: ISD::SETCC, VTs: {MVT::i32, MVT::f32}, Action: Expand);
117 setOperationAction(Ops: {ISD::FP_TO_UINT, ISD::FP_TO_SINT}, VTs: {MVT::i1, MVT::i64},
118 Action: Custom);
119
120 setOperationAction(Ops: ISD::SELECT, VTs: {MVT::i32, MVT::f32, MVT::v2i32, MVT::v4i32},
121 Action: Expand);
122
123 // ADD, SUB overflow.
124 // TODO: turn these into Legal?
125 if (Subtarget->hasCARRY())
126 setOperationAction(Op: ISD::UADDO, VT: MVT::i32, Action: Custom);
127
128 if (Subtarget->hasBORROW())
129 setOperationAction(Op: ISD::USUBO, VT: MVT::i32, Action: Custom);
130
131 // Expand sign extension of vectors
132 if (!Subtarget->hasBFE())
133 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
134
135 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG, VTs: {MVT::v2i1, MVT::v4i1}, Action: Expand);
136
137 if (!Subtarget->hasBFE())
138 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i8, Action: Expand);
139 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG, VTs: {MVT::v2i8, MVT::v4i8}, Action: Expand);
140
141 if (!Subtarget->hasBFE())
142 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i16, Action: Expand);
143 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG, VTs: {MVT::v2i16, MVT::v4i16}, Action: Expand);
144
145 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i32, Action: Legal);
146 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG, VTs: {MVT::v2i32, MVT::v4i32}, Action: Expand);
147
148 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::Other, Action: Expand);
149
150 setOperationAction(Op: ISD::FrameIndex, VT: MVT::i32, Action: Custom);
151
152 setOperationAction(Ops: ISD::EXTRACT_VECTOR_ELT,
153 VTs: {MVT::v2i32, MVT::v2f32, MVT::v4i32, MVT::v4f32}, Action: Custom);
154
155 setOperationAction(Ops: ISD::INSERT_VECTOR_ELT,
156 VTs: {MVT::v2i32, MVT::v2f32, MVT::v4i32, MVT::v4f32}, Action: Custom);
157
158 // We don't have 64-bit shifts. Thus we need either SHX i64 or SHX_PARTS i32
159 // to be Legal/Custom in order to avoid library calls.
160 setOperationAction(Ops: {ISD::SHL_PARTS, ISD::SRL_PARTS, ISD::SRA_PARTS}, VT: MVT::i32,
161 Action: Custom);
162
163 if (!Subtarget->hasFMA())
164 setOperationAction(Ops: ISD::FMA, VTs: {MVT::f32, MVT::f64}, Action: Expand);
165
166 // FIXME: May need no denormals check
167 setOperationAction(Op: ISD::FMAD, VT: MVT::f32, Action: Legal);
168
169 if (!Subtarget->hasBFI())
170 // fcopysign can be done in a single instruction with BFI.
171 setOperationAction(Ops: ISD::FCOPYSIGN, VTs: {MVT::f32, MVT::f64}, Action: Expand);
172
173 if (!Subtarget->hasBCNT(Size: 32))
174 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Expand);
175
176 if (!Subtarget->hasBCNT(Size: 64))
177 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Expand);
178
179 if (Subtarget->hasFFBH())
180 setOperationAction(Op: ISD::CTLZ_ZERO_POISON, VT: MVT::i32, Action: Custom);
181
182 if (Subtarget->hasFFBL())
183 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::i32, Action: Custom);
184
185 // FIXME: This was moved from AMDGPUTargetLowering, I'm not sure if we
186 // need it for R600.
187 if (Subtarget->hasBFE())
188 setHasExtractBitsInsn(true);
189
190 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i32, Action: Custom);
191 setOperationAction(Op: ISD::ADDRSPACECAST, VT: MVT::i32, Action: Custom);
192
193 // LLVM will expand these to atomic_cmp_swap(0)
194 // and atomic_swap, respectively.
195 setOperationAction(Ops: {ISD::ATOMIC_LOAD, ISD::ATOMIC_STORE}, VT: MVT::i32, Action: Expand);
196
197 // We need to custom lower some of the intrinsics
198 setOperationAction(Ops: {ISD::INTRINSIC_VOID, ISD::INTRINSIC_WO_CHAIN}, VT: MVT::Other,
199 Action: Custom);
200
201 setSchedulingPreference(Sched::Source);
202
203 setTargetDAGCombine({ISD::FP_ROUND, ISD::FP_TO_SINT, ISD::EXTRACT_VECTOR_ELT,
204 ISD::SELECT_CC, ISD::INSERT_VECTOR_ELT, ISD::LOAD});
205}
206
207static inline bool isEOP(MachineBasicBlock::iterator I) {
208 if (std::next(x: I) == I->getParent()->end())
209 return false;
210 return std::next(x: I)->getOpcode() == R600::RETURN;
211}
212
213MachineBasicBlock *
214R600TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
215 MachineBasicBlock *BB) const {
216 MachineFunction *MF = BB->getParent();
217 MachineRegisterInfo &MRI = MF->getRegInfo();
218 MachineBasicBlock::iterator I = MI;
219 const R600InstrInfo *TII = Subtarget->getInstrInfo();
220
221 switch (MI.getOpcode()) {
222 default:
223 // Replace LDS_*_RET instruction that don't have any uses with the
224 // equivalent LDS_*_NORET instruction.
225 if (TII->isLDSRetInstr(Opcode: MI.getOpcode())) {
226 int DstIdx = TII->getOperandIdx(Opcode: MI.getOpcode(), Op: R600::OpName::dst);
227 assert(DstIdx != -1);
228 MachineInstrBuilder NewMI;
229 // FIXME: getLDSNoRetOp method only handles LDS_1A1D LDS ops. Add
230 // LDS_1A2D support and remove this special case.
231 if (!MRI.use_empty(RegNo: MI.getOperand(i: DstIdx).getReg()) ||
232 MI.getOpcode() == R600::LDS_CMPST_RET)
233 return BB;
234
235 NewMI = BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I),
236 MCID: TII->get(Opcode: R600::getLDSNoRetOp(Opcode: MI.getOpcode())));
237 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands()))
238 NewMI.add(MO);
239 } else {
240 return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, MBB: BB);
241 }
242 break;
243
244 case R600::FABS_R600: {
245 MachineInstr *NewMI = TII->buildDefaultInstruction(
246 MBB&: *BB, I, Opcode: R600::MOV, DstReg: MI.getOperand(i: 0).getReg(),
247 Src0Reg: MI.getOperand(i: 1).getReg());
248 TII->addFlag(MI&: *NewMI, SrcIdx: 0, MO_FLAG_ABS);
249 break;
250 }
251
252 case R600::FNEG_R600: {
253 MachineInstr *NewMI = TII->buildDefaultInstruction(
254 MBB&: *BB, I, Opcode: R600::MOV, DstReg: MI.getOperand(i: 0).getReg(),
255 Src0Reg: MI.getOperand(i: 1).getReg());
256 TII->addFlag(MI&: *NewMI, SrcIdx: 0, MO_FLAG_NEG);
257 break;
258 }
259
260 case R600::MASK_WRITE: {
261 Register maskedRegister = MI.getOperand(i: 0).getReg();
262 assert(maskedRegister.isVirtual());
263 MachineInstr * defInstr = MRI.getVRegDef(Reg: maskedRegister);
264 TII->addFlag(MI&: *defInstr, SrcIdx: 0, MO_FLAG_MASK);
265 break;
266 }
267
268 case R600::MOV_IMM_F32:
269 TII->buildMovImm(BB&: *BB, I, DstReg: MI.getOperand(i: 0).getReg(), Imm: MI.getOperand(i: 1)
270 .getFPImm()
271 ->getValueAPF()
272 .bitcastToAPInt()
273 .getZExtValue());
274 break;
275
276 case R600::MOV_IMM_I32:
277 TII->buildMovImm(BB&: *BB, I, DstReg: MI.getOperand(i: 0).getReg(),
278 Imm: MI.getOperand(i: 1).getImm());
279 break;
280
281 case R600::MOV_IMM_GLOBAL_ADDR: {
282 //TODO: Perhaps combine this instruction with the next if possible
283 auto MIB = TII->buildDefaultInstruction(
284 MBB&: *BB, I: MI, Opcode: R600::MOV, DstReg: MI.getOperand(i: 0).getReg(), Src0Reg: R600::ALU_LITERAL_X);
285 int Idx = TII->getOperandIdx(MI: *MIB, Op: R600::OpName::literal);
286 //TODO: Ugh this is rather ugly
287 const MachineOperand &MO = MI.getOperand(i: 1);
288 MIB->getOperand(i: Idx).ChangeToGA(GV: MO.getGlobal(), Offset: MO.getOffset(),
289 TargetFlags: MO.getTargetFlags());
290 break;
291 }
292
293 case R600::CONST_COPY: {
294 MachineInstr *NewMI = TII->buildDefaultInstruction(
295 MBB&: *BB, I: MI, Opcode: R600::MOV, DstReg: MI.getOperand(i: 0).getReg(), Src0Reg: R600::ALU_CONST);
296 TII->setImmOperand(MI&: *NewMI, Op: R600::OpName::src0_sel,
297 Imm: MI.getOperand(i: 1).getImm());
298 break;
299 }
300
301 case R600::RAT_WRITE_CACHELESS_32_eg:
302 case R600::RAT_WRITE_CACHELESS_64_eg:
303 case R600::RAT_WRITE_CACHELESS_128_eg:
304 BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I), MCID: TII->get(Opcode: MI.getOpcode()))
305 .add(MO: MI.getOperand(i: 0))
306 .add(MO: MI.getOperand(i: 1))
307 .addImm(Val: isEOP(I)); // Set End of program bit
308 break;
309
310 case R600::RAT_STORE_TYPED_eg:
311 BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I), MCID: TII->get(Opcode: MI.getOpcode()))
312 .add(MO: MI.getOperand(i: 0))
313 .add(MO: MI.getOperand(i: 1))
314 .add(MO: MI.getOperand(i: 2))
315 .addImm(Val: isEOP(I)); // Set End of program bit
316 break;
317
318 case R600::BRANCH:
319 BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I), MCID: TII->get(Opcode: R600::JUMP))
320 .add(MO: MI.getOperand(i: 0));
321 break;
322
323 case R600::BRANCH_COND_f32: {
324 MachineInstr *NewMI =
325 BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I), MCID: TII->get(Opcode: R600::PRED_X),
326 DestReg: R600::PREDICATE_BIT)
327 .add(MO: MI.getOperand(i: 1))
328 .addImm(Val: R600::PRED_SETNE)
329 .addImm(Val: 0); // Flags
330 TII->addFlag(MI&: *NewMI, SrcIdx: 0, MO_FLAG_PUSH);
331 BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I), MCID: TII->get(Opcode: R600::JUMP_COND))
332 .add(MO: MI.getOperand(i: 0))
333 .addReg(RegNo: R600::PREDICATE_BIT, Flags: RegState::Kill);
334 break;
335 }
336
337 case R600::BRANCH_COND_i32: {
338 MachineInstr *NewMI =
339 BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I), MCID: TII->get(Opcode: R600::PRED_X),
340 DestReg: R600::PREDICATE_BIT)
341 .add(MO: MI.getOperand(i: 1))
342 .addImm(Val: R600::PRED_SETNE_INT)
343 .addImm(Val: 0); // Flags
344 TII->addFlag(MI&: *NewMI, SrcIdx: 0, MO_FLAG_PUSH);
345 BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I), MCID: TII->get(Opcode: R600::JUMP_COND))
346 .add(MO: MI.getOperand(i: 0))
347 .addReg(RegNo: R600::PREDICATE_BIT, Flags: RegState::Kill);
348 break;
349 }
350
351 case R600::EG_ExportSwz:
352 case R600::R600_ExportSwz: {
353 // Instruction is left unmodified if its not the last one of its type
354 bool isLastInstructionOfItsType = true;
355 unsigned InstExportType = MI.getOperand(i: 1).getImm();
356 for (MachineBasicBlock::iterator NextExportInst = std::next(x: I),
357 EndBlock = BB->end(); NextExportInst != EndBlock;
358 NextExportInst = std::next(x: NextExportInst)) {
359 if (NextExportInst->getOpcode() == R600::EG_ExportSwz ||
360 NextExportInst->getOpcode() == R600::R600_ExportSwz) {
361 unsigned CurrentInstExportType = NextExportInst->getOperand(i: 1)
362 .getImm();
363 if (CurrentInstExportType == InstExportType) {
364 isLastInstructionOfItsType = false;
365 break;
366 }
367 }
368 }
369 bool EOP = isEOP(I);
370 if (!EOP && !isLastInstructionOfItsType)
371 return BB;
372 unsigned CfInst = (MI.getOpcode() == R600::EG_ExportSwz) ? 84 : 40;
373 BuildMI(BB&: *BB, I, MIMD: BB->findDebugLoc(MBBI: I), MCID: TII->get(Opcode: MI.getOpcode()))
374 .add(MO: MI.getOperand(i: 0))
375 .add(MO: MI.getOperand(i: 1))
376 .add(MO: MI.getOperand(i: 2))
377 .add(MO: MI.getOperand(i: 3))
378 .add(MO: MI.getOperand(i: 4))
379 .add(MO: MI.getOperand(i: 5))
380 .add(MO: MI.getOperand(i: 6))
381 .addImm(Val: CfInst)
382 .addImm(Val: EOP);
383 break;
384 }
385 case R600::RETURN: {
386 return BB;
387 }
388 }
389
390 MI.eraseFromParent();
391 return BB;
392}
393
394//===----------------------------------------------------------------------===//
395// Custom DAG Lowering Operations
396//===----------------------------------------------------------------------===//
397
398SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
399 MachineFunction &MF = DAG.getMachineFunction();
400 R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
401 switch (Op.getOpcode()) {
402 default: return AMDGPUTargetLowering::LowerOperation(Op, DAG);
403 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
404 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
405 case ISD::SHL_PARTS:
406 case ISD::SRA_PARTS:
407 case ISD::SRL_PARTS: return LowerShiftParts(Op, DAG);
408 case ISD::UADDO: return LowerUADDSUBO(Op, DAG, mainop: ISD::ADD, ovf: AMDGPUISD::CARRY);
409 case ISD::USUBO: return LowerUADDSUBO(Op, DAG, mainop: ISD::SUB, ovf: AMDGPUISD::BORROW);
410 case ISD::FCOS:
411 case ISD::FSIN: return LowerTrig(Op, DAG);
412 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
413 case ISD::STORE: return LowerSTORE(Op, DAG);
414 case ISD::LOAD: {
415 SDValue Result = LowerLOAD(Op, DAG);
416 assert((!Result.getNode() ||
417 Result.getNode()->getNumValues() == 2) &&
418 "Load should return a value and a chain");
419 return Result;
420 }
421
422 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
423 case ISD::GlobalAddress: return LowerGlobalAddress(MFI, Op, DAG);
424 case ISD::FrameIndex: return lowerFrameIndex(Op, DAG);
425 case ISD::ADDRSPACECAST:
426 return lowerADDRSPACECAST(Op, DAG);
427 case ISD::INTRINSIC_VOID: {
428 SDValue Chain = Op.getOperand(i: 0);
429 unsigned IntrinsicID = Op.getConstantOperandVal(i: 1);
430 switch (IntrinsicID) {
431 case Intrinsic::r600_store_swizzle: {
432 SDLoc DL(Op);
433 const SDValue Args[8] = {
434 Chain,
435 Op.getOperand(i: 2), // Export Value
436 Op.getOperand(i: 3), // ArrayBase
437 Op.getOperand(i: 4), // Type
438 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // SWZ_X
439 DAG.getConstant(Val: 1, DL, VT: MVT::i32), // SWZ_Y
440 DAG.getConstant(Val: 2, DL, VT: MVT::i32), // SWZ_Z
441 DAG.getConstant(Val: 3, DL, VT: MVT::i32) // SWZ_W
442 };
443 return DAG.getNode(Opcode: AMDGPUISD::R600_EXPORT, DL, VT: Op.getValueType(), Ops: Args);
444 }
445
446 // default for switch(IntrinsicID)
447 default: break;
448 }
449 // break out of case ISD::INTRINSIC_VOID in switch(Op.getOpcode())
450 break;
451 }
452 case ISD::INTRINSIC_WO_CHAIN: {
453 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
454 EVT VT = Op.getValueType();
455 SDLoc DL(Op);
456 switch (IntrinsicID) {
457 case Intrinsic::r600_tex:
458 case Intrinsic::r600_texc: {
459 unsigned TextureOp;
460 switch (IntrinsicID) {
461 case Intrinsic::r600_tex:
462 TextureOp = 0;
463 break;
464 case Intrinsic::r600_texc:
465 TextureOp = 1;
466 break;
467 default:
468 llvm_unreachable("unhandled texture operation");
469 }
470
471 SDValue TexArgs[19] = {
472 DAG.getConstant(Val: TextureOp, DL, VT: MVT::i32),
473 Op.getOperand(i: 1),
474 DAG.getConstant(Val: 0, DL, VT: MVT::i32),
475 DAG.getConstant(Val: 1, DL, VT: MVT::i32),
476 DAG.getConstant(Val: 2, DL, VT: MVT::i32),
477 DAG.getConstant(Val: 3, DL, VT: MVT::i32),
478 Op.getOperand(i: 2),
479 Op.getOperand(i: 3),
480 Op.getOperand(i: 4),
481 DAG.getConstant(Val: 0, DL, VT: MVT::i32),
482 DAG.getConstant(Val: 1, DL, VT: MVT::i32),
483 DAG.getConstant(Val: 2, DL, VT: MVT::i32),
484 DAG.getConstant(Val: 3, DL, VT: MVT::i32),
485 Op.getOperand(i: 5),
486 Op.getOperand(i: 6),
487 Op.getOperand(i: 7),
488 Op.getOperand(i: 8),
489 Op.getOperand(i: 9),
490 Op.getOperand(i: 10)
491 };
492 return DAG.getNode(Opcode: AMDGPUISD::TEXTURE_FETCH, DL, VT: MVT::v4f32, Ops: TexArgs);
493 }
494 case Intrinsic::r600_dot4: {
495 SDValue Args[8] = {
496 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: Op.getOperand(i: 1),
497 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32)),
498 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: Op.getOperand(i: 2),
499 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32)),
500 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: Op.getOperand(i: 1),
501 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32)),
502 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: Op.getOperand(i: 2),
503 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32)),
504 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: Op.getOperand(i: 1),
505 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32)),
506 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: Op.getOperand(i: 2),
507 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32)),
508 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: Op.getOperand(i: 1),
509 N2: DAG.getConstant(Val: 3, DL, VT: MVT::i32)),
510 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: Op.getOperand(i: 2),
511 N2: DAG.getConstant(Val: 3, DL, VT: MVT::i32))
512 };
513 return DAG.getNode(Opcode: AMDGPUISD::DOT4, DL, VT: MVT::f32, Ops: Args);
514 }
515
516 case Intrinsic::r600_implicitarg_ptr: {
517 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout(), AS: AMDGPUAS::PARAM_I_ADDRESS);
518 uint32_t ByteOffset = getImplicitParameterOffset(MF, Param: FIRST_IMPLICIT);
519 return DAG.getConstant(Val: ByteOffset, DL, VT: PtrVT);
520 }
521 case Intrinsic::r600_read_ngroups_x:
522 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 0);
523 case Intrinsic::r600_read_ngroups_y:
524 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 1);
525 case Intrinsic::r600_read_ngroups_z:
526 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 2);
527 case Intrinsic::r600_read_global_size_x:
528 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 3);
529 case Intrinsic::r600_read_global_size_y:
530 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 4);
531 case Intrinsic::r600_read_global_size_z:
532 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 5);
533 case Intrinsic::r600_read_local_size_x:
534 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 6);
535 case Intrinsic::r600_read_local_size_y:
536 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 7);
537 case Intrinsic::r600_read_local_size_z:
538 return LowerImplicitParameter(DAG, VT, DL, DwordOffset: 8);
539
540 case Intrinsic::r600_read_tgid_x:
541 case Intrinsic::amdgcn_workgroup_id_x:
542 return CreateLiveInRegisterRaw(DAG, RC: &R600::R600_TReg32RegClass,
543 Reg: R600::T1_X, VT);
544 case Intrinsic::r600_read_tgid_y:
545 case Intrinsic::amdgcn_workgroup_id_y:
546 return CreateLiveInRegisterRaw(DAG, RC: &R600::R600_TReg32RegClass,
547 Reg: R600::T1_Y, VT);
548 case Intrinsic::r600_read_tgid_z:
549 case Intrinsic::amdgcn_workgroup_id_z:
550 return CreateLiveInRegisterRaw(DAG, RC: &R600::R600_TReg32RegClass,
551 Reg: R600::T1_Z, VT);
552 case Intrinsic::r600_read_tidig_x:
553 case Intrinsic::amdgcn_workitem_id_x:
554 return CreateLiveInRegisterRaw(DAG, RC: &R600::R600_TReg32RegClass,
555 Reg: R600::T0_X, VT);
556 case Intrinsic::r600_read_tidig_y:
557 case Intrinsic::amdgcn_workitem_id_y:
558 return CreateLiveInRegisterRaw(DAG, RC: &R600::R600_TReg32RegClass,
559 Reg: R600::T0_Y, VT);
560 case Intrinsic::r600_read_tidig_z:
561 case Intrinsic::amdgcn_workitem_id_z:
562 return CreateLiveInRegisterRaw(DAG, RC: &R600::R600_TReg32RegClass,
563 Reg: R600::T0_Z, VT);
564
565 case Intrinsic::r600_recipsqrt_ieee:
566 return DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: Op.getOperand(i: 1));
567
568 case Intrinsic::r600_recipsqrt_clamped:
569 return DAG.getNode(Opcode: AMDGPUISD::RSQ_CLAMP, DL, VT, Operand: Op.getOperand(i: 1));
570 default:
571 return Op;
572 }
573
574 // break out of case ISD::INTRINSIC_WO_CHAIN in switch(Op.getOpcode())
575 break;
576 }
577 } // end switch(Op.getOpcode())
578 return SDValue();
579}
580
581void R600TargetLowering::ReplaceNodeResults(SDNode *N,
582 SmallVectorImpl<SDValue> &Results,
583 SelectionDAG &DAG) const {
584 switch (N->getOpcode()) {
585 default:
586 AMDGPUTargetLowering::ReplaceNodeResults(N, Results, DAG);
587 return;
588 case ISD::FP_TO_UINT:
589 if (N->getValueType(ResNo: 0) == MVT::i1) {
590 Results.push_back(Elt: lowerFP_TO_UINT(Op: N->getOperand(Num: 0), DAG));
591 return;
592 }
593 // Since we don't care about out of bounds values we can use FP_TO_SINT for
594 // uints too. The DAGLegalizer code for uint considers some extra cases
595 // which are not necessary here.
596 [[fallthrough]];
597 case ISD::FP_TO_SINT: {
598 if (N->getValueType(ResNo: 0) == MVT::i1) {
599 Results.push_back(Elt: lowerFP_TO_SINT(Op: N->getOperand(Num: 0), DAG));
600 return;
601 }
602
603 SDValue Result;
604 if (expandFP_TO_SINT(N, Result, DAG))
605 Results.push_back(Elt: Result);
606 return;
607 }
608 case ISD::SDIVREM: {
609 SDValue Op = SDValue(N, 1);
610 SDValue RES = LowerSDIVREM(Op, DAG);
611 Results.push_back(Elt: RES);
612 Results.push_back(Elt: RES.getValue(R: 1));
613 break;
614 }
615 case ISD::UDIVREM: {
616 SDValue Op = SDValue(N, 0);
617 LowerUDIVREM64(Op, DAG, Results);
618 break;
619 }
620 }
621}
622
623SDValue R600TargetLowering::vectorToVerticalVector(SelectionDAG &DAG,
624 SDValue Vector) const {
625 SDLoc DL(Vector);
626 EVT VecVT = Vector.getValueType();
627 EVT EltVT = VecVT.getVectorElementType();
628 SmallVector<SDValue, 8> Args;
629
630 for (unsigned i = 0, e = VecVT.getVectorNumElements(); i != e; ++i) {
631 Args.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Vector,
632 N2: DAG.getVectorIdxConstant(Val: i, DL)));
633 }
634
635 return DAG.getNode(Opcode: AMDGPUISD::BUILD_VERTICAL_VECTOR, DL, VT: VecVT, Ops: Args);
636}
637
638SDValue R600TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
639 SelectionDAG &DAG) const {
640 SDLoc DL(Op);
641 SDValue Vector = Op.getOperand(i: 0);
642 SDValue Index = Op.getOperand(i: 1);
643
644 if (isa<ConstantSDNode>(Val: Index) ||
645 Vector.getOpcode() == AMDGPUISD::BUILD_VERTICAL_VECTOR)
646 return Op;
647
648 Vector = vectorToVerticalVector(DAG, Vector);
649 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: Op.getValueType(),
650 N1: Vector, N2: Index);
651}
652
653SDValue R600TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
654 SelectionDAG &DAG) const {
655 SDLoc DL(Op);
656 SDValue Vector = Op.getOperand(i: 0);
657 SDValue Value = Op.getOperand(i: 1);
658 SDValue Index = Op.getOperand(i: 2);
659
660 if (isa<ConstantSDNode>(Val: Index) ||
661 Vector.getOpcode() == AMDGPUISD::BUILD_VERTICAL_VECTOR)
662 return Op;
663
664 Vector = vectorToVerticalVector(DAG, Vector);
665 SDValue Insert = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: Op.getValueType(),
666 N1: Vector, N2: Value, N3: Index);
667 return vectorToVerticalVector(DAG, Vector: Insert);
668}
669
670SDValue R600TargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
671 SDValue Op,
672 SelectionDAG &DAG) const {
673 GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Val&: Op);
674 if (GSD->getAddressSpace() != AMDGPUAS::CONSTANT_ADDRESS)
675 return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG);
676
677 const DataLayout &DL = DAG.getDataLayout();
678 const GlobalValue *GV = GSD->getGlobal();
679 MVT ConstPtrVT = getPointerTy(DL, AS: AMDGPUAS::CONSTANT_ADDRESS);
680
681 SDValue GA = DAG.getTargetGlobalAddress(GV, DL: SDLoc(GSD), VT: ConstPtrVT);
682 return DAG.getNode(Opcode: AMDGPUISD::CONST_DATA_PTR, DL: SDLoc(GSD), VT: ConstPtrVT, Operand: GA);
683}
684
685SDValue R600TargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
686 // On hw >= R700, COS/SIN input must be between -1. and 1.
687 // Thus we lower them to TRIG ( FRACT ( x / 2Pi + 0.5) - 0.5)
688 EVT VT = Op.getValueType();
689 SDValue Arg = Op.getOperand(i: 0);
690 SDLoc DL(Op);
691
692 // TODO: Should this propagate fast-math-flags?
693 SDValue FractPart = DAG.getNode(Opcode: AMDGPUISD::FRACT, DL, VT,
694 Operand: DAG.getNode(Opcode: ISD::FADD, DL, VT,
695 N1: DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Arg,
696 N2: DAG.getConstantFP(Val: 0.15915494309, DL, VT: MVT::f32)),
697 N2: DAG.getConstantFP(Val: 0.5, DL, VT: MVT::f32)));
698 unsigned TrigNode;
699 switch (Op.getOpcode()) {
700 case ISD::FCOS:
701 TrigNode = AMDGPUISD::COS_HW;
702 break;
703 case ISD::FSIN:
704 TrigNode = AMDGPUISD::SIN_HW;
705 break;
706 default:
707 llvm_unreachable("Wrong trig opcode");
708 }
709 SDValue TrigVal = DAG.getNode(Opcode: TrigNode, DL, VT,
710 Operand: DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: FractPart,
711 N2: DAG.getConstantFP(Val: -0.5, DL, VT: MVT::f32)));
712 if (Gen >= AMDGPUSubtarget::R700)
713 return TrigVal;
714 // On R600 hw, COS/SIN input must be between -Pi and Pi.
715 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: TrigVal,
716 N2: DAG.getConstantFP(Val: numbers::pif, DL, VT: MVT::f32));
717}
718
719SDValue R600TargetLowering::LowerShiftParts(SDValue Op,
720 SelectionDAG &DAG) const {
721 SDValue Lo, Hi;
722 expandShiftParts(N: Op.getNode(), Lo, Hi, DAG);
723 return DAG.getMergeValues(Ops: {Lo, Hi}, dl: SDLoc(Op));
724}
725
726SDValue R600TargetLowering::LowerUADDSUBO(SDValue Op, SelectionDAG &DAG,
727 unsigned mainop, unsigned ovf) const {
728 SDLoc DL(Op);
729 EVT VT = Op.getValueType();
730
731 SDValue Lo = Op.getOperand(i: 0);
732 SDValue Hi = Op.getOperand(i: 1);
733
734 SDValue OVF = DAG.getNode(Opcode: ovf, DL, VT, N1: Lo, N2: Hi);
735 // Extend sign.
736 OVF = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: OVF,
737 N2: DAG.getValueType(MVT::i1));
738
739 SDValue Res = DAG.getNode(Opcode: mainop, DL, VT, N1: Lo, N2: Hi);
740
741 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: Res, N2: OVF);
742}
743
744SDValue R600TargetLowering::lowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) const {
745 SDLoc DL(Op);
746 return DAG.getNode(
747 Opcode: ISD::SETCC,
748 DL,
749 VT: MVT::i1,
750 N1: Op, N2: DAG.getConstantFP(Val: 1.0f, DL, VT: MVT::f32),
751 N3: DAG.getCondCode(Cond: ISD::SETEQ));
752}
753
754SDValue R600TargetLowering::lowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const {
755 SDLoc DL(Op);
756 return DAG.getNode(
757 Opcode: ISD::SETCC,
758 DL,
759 VT: MVT::i1,
760 N1: Op, N2: DAG.getConstantFP(Val: -1.0f, DL, VT: MVT::f32),
761 N3: DAG.getCondCode(Cond: ISD::SETEQ));
762}
763
764SDValue R600TargetLowering::LowerImplicitParameter(SelectionDAG &DAG, EVT VT,
765 const SDLoc &DL,
766 unsigned DwordOffset) const {
767 unsigned ByteOffset = DwordOffset * 4;
768 PointerType *PtrType =
769 PointerType::get(C&: *DAG.getContext(), AddressSpace: AMDGPUAS::PARAM_I_ADDRESS);
770
771 // We shouldn't be using an offset wider than 16-bits for implicit parameters.
772 assert(isInt<16>(ByteOffset));
773
774 return DAG.getLoad(VT, dl: DL, Chain: DAG.getEntryNode(),
775 Ptr: DAG.getConstant(Val: ByteOffset, DL, VT: MVT::i32), // PTR
776 PtrInfo: MachinePointerInfo(ConstantPointerNull::get(T: PtrType)));
777}
778
779bool R600TargetLowering::isZero(SDValue Op) const {
780 if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Val&: Op))
781 return Cst->isZero();
782 if (ConstantFPSDNode *CstFP = dyn_cast<ConstantFPSDNode>(Val&: Op))
783 return CstFP->isZero();
784 return false;
785}
786
787bool R600TargetLowering::isHWTrueValue(SDValue Op) const {
788 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
789 return CFP->isOne();
790 }
791 return isAllOnesConstant(V: Op);
792}
793
794bool R600TargetLowering::isHWFalseValue(SDValue Op) const {
795 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
796 return CFP->getValueAPF().isZero();
797 }
798 return isNullConstant(V: Op);
799}
800
801SDValue R600TargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
802 SDLoc DL(Op);
803 EVT VT = Op.getValueType();
804
805 SDValue LHS = Op.getOperand(i: 0);
806 SDValue RHS = Op.getOperand(i: 1);
807 SDValue True = Op.getOperand(i: 2);
808 SDValue False = Op.getOperand(i: 3);
809 SDValue CC = Op.getOperand(i: 4);
810 SDValue Temp;
811
812 if (VT == MVT::f32) {
813 DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr);
814 SDValue MinMax = combineFMinMaxLegacy(DL, VT, LHS, RHS, True, False, CC, DCI);
815 if (MinMax)
816 return MinMax;
817 }
818
819 // LHS and RHS are guaranteed to be the same value type
820 EVT CompareVT = LHS.getValueType();
821
822 // Check if we can lower this to a native operation.
823
824 // Try to lower to a SET* instruction:
825 //
826 // SET* can match the following patterns:
827 //
828 // select_cc f32, f32, -1, 0, cc_supported
829 // select_cc f32, f32, 1.0f, 0.0f, cc_supported
830 // select_cc i32, i32, -1, 0, cc_supported
831 //
832
833 // Move hardware True/False values to the correct operand.
834 if (isHWTrueValue(Op: False) && isHWFalseValue(Op: True)) {
835 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val&: CC)->get();
836 ISD::CondCode InverseCC = ISD::getSetCCInverse(Operation: CCOpcode, Type: CompareVT);
837 if (isCondCodeLegal(CC: InverseCC, VT: CompareVT.getSimpleVT())) {
838 std::swap(a&: False, b&: True);
839 CC = DAG.getCondCode(Cond: InverseCC);
840 } else {
841 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(Operation: InverseCC);
842 if (isCondCodeLegal(CC: SwapInvCC, VT: CompareVT.getSimpleVT())) {
843 std::swap(a&: False, b&: True);
844 std::swap(a&: LHS, b&: RHS);
845 CC = DAG.getCondCode(Cond: SwapInvCC);
846 }
847 }
848 }
849
850 if (isHWTrueValue(Op: True) && isHWFalseValue(Op: False) &&
851 (CompareVT == VT || VT == MVT::i32)) {
852 // This can be matched by a SET* instruction.
853 return DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT, N1: LHS, N2: RHS, N3: True, N4: False, N5: CC);
854 }
855
856 // Try to lower to a CND* instruction:
857 //
858 // CND* can match the following patterns:
859 //
860 // select_cc f32, 0.0, f32, f32, cc_supported
861 // select_cc f32, 0.0, i32, i32, cc_supported
862 // select_cc i32, 0, f32, f32, cc_supported
863 // select_cc i32, 0, i32, i32, cc_supported
864 //
865
866 // Try to move the zero value to the RHS
867 if (isZero(Op: LHS)) {
868 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val&: CC)->get();
869 // Try swapping the operands
870 ISD::CondCode CCSwapped = ISD::getSetCCSwappedOperands(Operation: CCOpcode);
871 if (isCondCodeLegal(CC: CCSwapped, VT: CompareVT.getSimpleVT())) {
872 std::swap(a&: LHS, b&: RHS);
873 CC = DAG.getCondCode(Cond: CCSwapped);
874 } else {
875 // Try inverting the condition and then swapping the operands
876 ISD::CondCode CCInv = ISD::getSetCCInverse(Operation: CCOpcode, Type: CompareVT);
877 CCSwapped = ISD::getSetCCSwappedOperands(Operation: CCInv);
878 if (isCondCodeLegal(CC: CCSwapped, VT: CompareVT.getSimpleVT())) {
879 std::swap(a&: True, b&: False);
880 std::swap(a&: LHS, b&: RHS);
881 CC = DAG.getCondCode(Cond: CCSwapped);
882 }
883 }
884 }
885 if (isZero(Op: RHS)) {
886 SDValue Cond = LHS;
887 SDValue Zero = RHS;
888 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val&: CC)->get();
889 if (CompareVT != VT) {
890 // Bitcast True / False to the correct types. This will end up being
891 // a nop, but it allows us to define only a single pattern in the
892 // .TD files for each CND* instruction rather than having to have
893 // one pattern for integer True/False and one for fp True/False
894 True = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: CompareVT, Operand: True);
895 False = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: CompareVT, Operand: False);
896 }
897
898 switch (CCOpcode) {
899 case ISD::SETONE:
900 case ISD::SETUNE:
901 case ISD::SETNE:
902 CCOpcode = ISD::getSetCCInverse(Operation: CCOpcode, Type: CompareVT);
903 Temp = True;
904 True = False;
905 False = Temp;
906 break;
907 default:
908 break;
909 }
910 SDValue SelectNode = DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT: CompareVT,
911 N1: Cond, N2: Zero,
912 N3: True, N4: False,
913 N5: DAG.getCondCode(Cond: CCOpcode));
914 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: SelectNode);
915 }
916
917 // If we make it this for it means we have no native instructions to handle
918 // this SELECT_CC, so we must lower it.
919 SDValue HWTrue, HWFalse;
920
921 if (CompareVT == MVT::f32) {
922 HWTrue = DAG.getConstantFP(Val: 1.0f, DL, VT: CompareVT);
923 HWFalse = DAG.getConstantFP(Val: 0.0f, DL, VT: CompareVT);
924 } else if (CompareVT == MVT::i32) {
925 HWTrue = DAG.getAllOnesConstant(DL, VT: CompareVT);
926 HWFalse = DAG.getConstant(Val: 0, DL, VT: CompareVT);
927 }
928 else {
929 llvm_unreachable("Unhandled value type in LowerSELECT_CC");
930 }
931
932 // Lower this unsupported SELECT_CC into a combination of two supported
933 // SELECT_CC operations.
934 SDValue Cond = DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT: CompareVT, N1: LHS, N2: RHS, N3: HWTrue, N4: HWFalse, N5: CC);
935
936 return DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT,
937 N1: Cond, N2: HWFalse,
938 N3: True, N4: False,
939 N5: DAG.getCondCode(Cond: ISD::SETNE));
940}
941
942SDValue R600TargetLowering::lowerADDRSPACECAST(SDValue Op,
943 SelectionDAG &DAG) const {
944 SDLoc SL(Op);
945 EVT VT = Op.getValueType();
946
947 const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Val&: Op);
948 unsigned SrcAS = ASC->getSrcAddressSpace();
949 unsigned DestAS = ASC->getDestAddressSpace();
950
951 if (isNullConstant(V: Op.getOperand(i: 0)) && SrcAS == AMDGPUAS::FLAT_ADDRESS)
952 return DAG.getSignedConstant(Val: AMDGPU::getNullPointerValue(AS: DestAS), DL: SL, VT);
953
954 return Op;
955}
956
957/// LLVM generates byte-addressed pointers. For indirect addressing, we need to
958/// convert these pointers to a register index. Each register holds
959/// 16 bytes, (4 x 32bit sub-register), but we need to take into account the
960/// \p StackWidth, which tells us how many of the 4 sub-registers will be used
961/// for indirect addressing.
962SDValue R600TargetLowering::stackPtrToRegIndex(SDValue Ptr,
963 unsigned StackWidth,
964 SelectionDAG &DAG) const {
965 unsigned SRLPad;
966 switch(StackWidth) {
967 case 1:
968 SRLPad = 2;
969 break;
970 case 2:
971 SRLPad = 3;
972 break;
973 case 4:
974 SRLPad = 4;
975 break;
976 default: llvm_unreachable("Invalid stack width");
977 }
978
979 SDLoc DL(Ptr);
980 return DAG.getNode(Opcode: ISD::SRL, DL, VT: Ptr.getValueType(), N1: Ptr,
981 N2: DAG.getConstant(Val: SRLPad, DL, VT: MVT::i32));
982}
983
984void R600TargetLowering::getStackAddress(unsigned StackWidth,
985 unsigned ElemIdx,
986 unsigned &Channel,
987 unsigned &PtrIncr) const {
988 switch (StackWidth) {
989 default:
990 case 1:
991 Channel = 0;
992 if (ElemIdx > 0) {
993 PtrIncr = 1;
994 } else {
995 PtrIncr = 0;
996 }
997 break;
998 case 2:
999 Channel = ElemIdx % 2;
1000 if (ElemIdx == 2) {
1001 PtrIncr = 1;
1002 } else {
1003 PtrIncr = 0;
1004 }
1005 break;
1006 case 4:
1007 Channel = ElemIdx;
1008 PtrIncr = 0;
1009 break;
1010 }
1011}
1012
1013SDValue R600TargetLowering::lowerPrivateTruncStore(StoreSDNode *Store,
1014 SelectionDAG &DAG) const {
1015 SDLoc DL(Store);
1016 //TODO: Who creates the i8 stores?
1017 assert(Store->isTruncatingStore()
1018 || Store->getValue().getValueType() == MVT::i8);
1019 assert(Store->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS);
1020
1021 SDValue Mask;
1022 if (Store->getMemoryVT() == MVT::i8) {
1023 assert(Store->getAlign() >= 1);
1024 Mask = DAG.getConstant(Val: 0xff, DL, VT: MVT::i32);
1025 } else if (Store->getMemoryVT() == MVT::i16) {
1026 assert(Store->getAlign() >= 2);
1027 Mask = DAG.getConstant(Val: 0xffff, DL, VT: MVT::i32);
1028 } else {
1029 llvm_unreachable("Unsupported private trunc store");
1030 }
1031
1032 SDValue OldChain = Store->getChain();
1033 bool VectorTrunc = (OldChain.getOpcode() == AMDGPUISD::DUMMY_CHAIN);
1034 // Skip dummy
1035 SDValue Chain = VectorTrunc ? OldChain->getOperand(Num: 0) : OldChain;
1036 SDValue BasePtr = Store->getBasePtr();
1037 SDValue Offset = Store->getOffset();
1038 EVT MemVT = Store->getMemoryVT();
1039
1040 SDValue LoadPtr = BasePtr;
1041 if (!Offset.isUndef()) {
1042 LoadPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: BasePtr, N2: Offset);
1043 }
1044
1045 // Get dword location
1046 // TODO: this should be eliminated by the future SHR ptr, 2
1047 SDValue Ptr = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: LoadPtr,
1048 N2: DAG.getConstant(Val: 0xfffffffc, DL, VT: MVT::i32));
1049
1050 // Load dword
1051 // TODO: can we be smarter about machine pointer info?
1052 MachinePointerInfo PtrInfo(AMDGPUAS::PRIVATE_ADDRESS);
1053 SDValue Dst = DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr, PtrInfo);
1054
1055 Chain = Dst.getValue(R: 1);
1056
1057 // Get offset in dword
1058 SDValue ByteIdx = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: LoadPtr,
1059 N2: DAG.getConstant(Val: 0x3, DL, VT: MVT::i32));
1060
1061 // Convert byte offset to bit shift
1062 SDValue ShiftAmt = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: ByteIdx,
1063 N2: DAG.getConstant(Val: 3, DL, VT: MVT::i32));
1064
1065 // TODO: Contrary to the name of the function,
1066 // it also handles sub i32 non-truncating stores (like i1)
1067 SDValue SExtValue = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i32,
1068 Operand: Store->getValue());
1069
1070 // Mask the value to the right type
1071 SDValue MaskedValue = DAG.getZeroExtendInReg(Op: SExtValue, DL, VT: MemVT);
1072
1073 // Shift the value in place
1074 SDValue ShiftedValue = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32,
1075 N1: MaskedValue, N2: ShiftAmt);
1076
1077 // Shift the mask in place
1078 SDValue DstMask = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: Mask, N2: ShiftAmt);
1079
1080 // Invert the mask. NOTE: if we had native ROL instructions we could
1081 // use inverted mask
1082 DstMask = DAG.getNOT(DL, Val: DstMask, VT: MVT::i32);
1083
1084 // Cleanup the target bits
1085 Dst = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Dst, N2: DstMask);
1086
1087 // Add the new bits
1088 SDValue Value = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Dst, N2: ShiftedValue);
1089
1090 // Store dword
1091 // TODO: Can we be smarter about MachinePointerInfo?
1092 SDValue NewStore = DAG.getStore(Chain, dl: DL, Val: Value, Ptr, PtrInfo);
1093
1094 // If we are part of expanded vector, make our neighbors depend on this store
1095 if (VectorTrunc) {
1096 // Make all other vector elements depend on this store
1097 Chain = DAG.getNode(Opcode: AMDGPUISD::DUMMY_CHAIN, DL, VT: MVT::Other, Operand: NewStore);
1098 DAG.ReplaceAllUsesOfValueWith(From: OldChain, To: Chain);
1099 }
1100 return NewStore;
1101}
1102
1103SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
1104 StoreSDNode *StoreNode = cast<StoreSDNode>(Val&: Op);
1105 unsigned AS = StoreNode->getAddressSpace();
1106
1107 SDValue Chain = StoreNode->getChain();
1108 SDValue Ptr = StoreNode->getBasePtr();
1109 SDValue Value = StoreNode->getValue();
1110
1111 EVT VT = Value.getValueType();
1112 EVT MemVT = StoreNode->getMemoryVT();
1113 EVT PtrVT = Ptr.getValueType();
1114
1115 SDLoc DL(Op);
1116
1117 const bool TruncatingStore = StoreNode->isTruncatingStore();
1118
1119 // Neither LOCAL nor PRIVATE can do vectors at the moment
1120 if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS ||
1121 TruncatingStore) &&
1122 VT.isVector()) {
1123 if ((AS == AMDGPUAS::PRIVATE_ADDRESS) && TruncatingStore) {
1124 // Add an extra level of chain to isolate this vector
1125 SDValue NewChain = DAG.getNode(Opcode: AMDGPUISD::DUMMY_CHAIN, DL, VT: MVT::Other, Operand: Chain);
1126 SmallVector<SDValue, 4> NewOps(StoreNode->ops());
1127 NewOps[0] = NewChain;
1128 StoreNode = cast<StoreSDNode>(Val: DAG.UpdateNodeOperands(N: StoreNode, Ops: NewOps));
1129 }
1130
1131 return scalarizeVectorStore(ST: StoreNode, DAG);
1132 }
1133
1134 Align Alignment = StoreNode->getAlign();
1135 if (Alignment < MemVT.getStoreSize() &&
1136 !allowsMisalignedMemoryAccesses(VT: MemVT, AS, Alignment,
1137 Flags: StoreNode->getMemOperand()->getFlags(),
1138 IsFast: nullptr)) {
1139 return expandUnalignedStore(ST: StoreNode, DAG);
1140 }
1141
1142 SDValue DWordAddr = DAG.getNode(Opcode: ISD::SRL, DL, VT: PtrVT, N1: Ptr,
1143 N2: DAG.getConstant(Val: 2, DL, VT: PtrVT));
1144
1145 if (AS == AMDGPUAS::GLOBAL_ADDRESS) {
1146 // It is beneficial to create MSKOR here instead of combiner to avoid
1147 // artificial dependencies introduced by RMW
1148 if (TruncatingStore) {
1149 assert(VT.bitsLE(MVT::i32));
1150 SDValue MaskConstant;
1151 if (MemVT == MVT::i8) {
1152 MaskConstant = DAG.getConstant(Val: 0xFF, DL, VT: MVT::i32);
1153 } else {
1154 assert(MemVT == MVT::i16);
1155 assert(StoreNode->getAlign() >= 2);
1156 MaskConstant = DAG.getConstant(Val: 0xFFFF, DL, VT: MVT::i32);
1157 }
1158
1159 SDValue ByteIndex = DAG.getNode(Opcode: ISD::AND, DL, VT: PtrVT, N1: Ptr,
1160 N2: DAG.getConstant(Val: 0x00000003, DL, VT: PtrVT));
1161 SDValue BitShift = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ByteIndex,
1162 N2: DAG.getConstant(Val: 3, DL, VT));
1163
1164 // Put the mask in correct place
1165 SDValue Mask = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: MaskConstant, N2: BitShift);
1166
1167 // Put the value bits in correct place
1168 SDValue TruncValue = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Value, N2: MaskConstant);
1169 SDValue ShiftedValue = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: TruncValue, N2: BitShift);
1170
1171 // XXX: If we add a 64-bit ZW register class, then we could use a 2 x i32
1172 // vector instead.
1173 SDValue Src[4] = {
1174 ShiftedValue,
1175 DAG.getConstant(Val: 0, DL, VT: MVT::i32),
1176 DAG.getConstant(Val: 0, DL, VT: MVT::i32),
1177 Mask
1178 };
1179 SDValue Input = DAG.getBuildVector(VT: MVT::v4i32, DL, Ops: Src);
1180 SDValue Args[3] = { Chain, Input, DWordAddr };
1181 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::STORE_MSKOR, dl: DL,
1182 VTList: Op->getVTList(), Ops: Args, MemVT,
1183 MMO: StoreNode->getMemOperand());
1184 }
1185 if (Ptr->getOpcode() != AMDGPUISD::DWORDADDR && VT.bitsGE(VT: MVT::i32)) {
1186 // Convert pointer from byte address to dword address.
1187 Ptr = DAG.getNode(Opcode: AMDGPUISD::DWORDADDR, DL, VT: PtrVT, Operand: DWordAddr);
1188
1189 if (StoreNode->isIndexed()) {
1190 llvm_unreachable("Indexed stores not supported yet");
1191 } else {
1192 Chain = DAG.getStore(Chain, dl: DL, Val: Value, Ptr, MMO: StoreNode->getMemOperand());
1193 }
1194 return Chain;
1195 }
1196 }
1197
1198 // GLOBAL_ADDRESS has been handled above, LOCAL_ADDRESS allows all sizes
1199 if (AS != AMDGPUAS::PRIVATE_ADDRESS)
1200 return SDValue();
1201
1202 if (MemVT.bitsLT(VT: MVT::i32))
1203 return lowerPrivateTruncStore(Store: StoreNode, DAG);
1204
1205 // Standard i32+ store, tag it with DWORDADDR to note that the address
1206 // has been shifted
1207 if (Ptr.getOpcode() != AMDGPUISD::DWORDADDR) {
1208 Ptr = DAG.getNode(Opcode: AMDGPUISD::DWORDADDR, DL, VT: PtrVT, Operand: DWordAddr);
1209 return DAG.getStore(Chain, dl: DL, Val: Value, Ptr, MMO: StoreNode->getMemOperand());
1210 }
1211
1212 // Tagged i32+ stores will be matched by patterns
1213 return SDValue();
1214}
1215
1216// return (512 + (kc_bank << 12)
1217static int
1218ConstantAddressBlock(unsigned AddressSpace) {
1219 switch (AddressSpace) {
1220 case AMDGPUAS::CONSTANT_BUFFER_0:
1221 return 512;
1222 case AMDGPUAS::CONSTANT_BUFFER_1:
1223 return 512 + 4096;
1224 case AMDGPUAS::CONSTANT_BUFFER_2:
1225 return 512 + 4096 * 2;
1226 case AMDGPUAS::CONSTANT_BUFFER_3:
1227 return 512 + 4096 * 3;
1228 case AMDGPUAS::CONSTANT_BUFFER_4:
1229 return 512 + 4096 * 4;
1230 case AMDGPUAS::CONSTANT_BUFFER_5:
1231 return 512 + 4096 * 5;
1232 case AMDGPUAS::CONSTANT_BUFFER_6:
1233 return 512 + 4096 * 6;
1234 case AMDGPUAS::CONSTANT_BUFFER_7:
1235 return 512 + 4096 * 7;
1236 case AMDGPUAS::CONSTANT_BUFFER_8:
1237 return 512 + 4096 * 8;
1238 case AMDGPUAS::CONSTANT_BUFFER_9:
1239 return 512 + 4096 * 9;
1240 case AMDGPUAS::CONSTANT_BUFFER_10:
1241 return 512 + 4096 * 10;
1242 case AMDGPUAS::CONSTANT_BUFFER_11:
1243 return 512 + 4096 * 11;
1244 case AMDGPUAS::CONSTANT_BUFFER_12:
1245 return 512 + 4096 * 12;
1246 case AMDGPUAS::CONSTANT_BUFFER_13:
1247 return 512 + 4096 * 13;
1248 case AMDGPUAS::CONSTANT_BUFFER_14:
1249 return 512 + 4096 * 14;
1250 case AMDGPUAS::CONSTANT_BUFFER_15:
1251 return 512 + 4096 * 15;
1252 default:
1253 return -1;
1254 }
1255}
1256
1257SDValue R600TargetLowering::lowerPrivateExtLoad(SDValue Op,
1258 SelectionDAG &DAG) const {
1259 SDLoc DL(Op);
1260 LoadSDNode *Load = cast<LoadSDNode>(Val&: Op);
1261 ISD::LoadExtType ExtType = Load->getExtensionType();
1262 EVT MemVT = Load->getMemoryVT();
1263 assert(Load->getAlign() >= MemVT.getStoreSize());
1264
1265 SDValue BasePtr = Load->getBasePtr();
1266 SDValue Chain = Load->getChain();
1267 SDValue Offset = Load->getOffset();
1268
1269 SDValue LoadPtr = BasePtr;
1270 if (!Offset.isUndef()) {
1271 LoadPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: BasePtr, N2: Offset);
1272 }
1273
1274 // Get dword location
1275 // NOTE: this should be eliminated by the future SHR ptr, 2
1276 SDValue Ptr = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: LoadPtr,
1277 N2: DAG.getConstant(Val: 0xfffffffc, DL, VT: MVT::i32));
1278
1279 // Load dword
1280 // TODO: can we be smarter about machine pointer info?
1281 MachinePointerInfo PtrInfo(AMDGPUAS::PRIVATE_ADDRESS);
1282 SDValue Read = DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr, PtrInfo);
1283
1284 // Get offset within the register.
1285 SDValue ByteIdx = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32,
1286 N1: LoadPtr, N2: DAG.getConstant(Val: 0x3, DL, VT: MVT::i32));
1287
1288 // Bit offset of target byte (byteIdx * 8).
1289 SDValue ShiftAmt = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: ByteIdx,
1290 N2: DAG.getConstant(Val: 3, DL, VT: MVT::i32));
1291
1292 // Shift to the right.
1293 SDValue Ret = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Read, N2: ShiftAmt);
1294
1295 // Eliminate the upper bits by setting them to ...
1296 EVT MemEltVT = MemVT.getScalarType();
1297
1298 if (ExtType == ISD::SEXTLOAD) { // ... ones.
1299 SDValue MemEltVTNode = DAG.getValueType(MemEltVT);
1300 Ret = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i32, N1: Ret, N2: MemEltVTNode);
1301 } else { // ... or zeros.
1302 Ret = DAG.getZeroExtendInReg(Op: Ret, DL, VT: MemEltVT);
1303 }
1304
1305 SDValue Ops[] = {
1306 Ret,
1307 Read.getValue(R: 1) // This should be our output chain
1308 };
1309
1310 return DAG.getMergeValues(Ops, dl: DL);
1311}
1312
1313SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
1314 LoadSDNode *LoadNode = cast<LoadSDNode>(Val&: Op);
1315 unsigned AS = LoadNode->getAddressSpace();
1316 EVT MemVT = LoadNode->getMemoryVT();
1317 ISD::LoadExtType ExtType = LoadNode->getExtensionType();
1318
1319 if (AS == AMDGPUAS::PRIVATE_ADDRESS &&
1320 ExtType != ISD::NON_EXTLOAD && MemVT.bitsLT(VT: MVT::i32)) {
1321 return lowerPrivateExtLoad(Op, DAG);
1322 }
1323
1324 SDLoc DL(Op);
1325 EVT VT = Op.getValueType();
1326 SDValue Chain = LoadNode->getChain();
1327 SDValue Ptr = LoadNode->getBasePtr();
1328
1329 if ((LoadNode->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
1330 LoadNode->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) &&
1331 VT.isVector()) {
1332 SDValue Ops[2];
1333 std::tie(args&: Ops[0], args&: Ops[1]) = scalarizeVectorLoad(LD: LoadNode, DAG);
1334 return DAG.getMergeValues(Ops, dl: DL);
1335 }
1336
1337 // This is still used for explicit load from addrspace(8)
1338 int ConstantBlock = ConstantAddressBlock(AddressSpace: LoadNode->getAddressSpace());
1339 if (ConstantBlock > -1 &&
1340 ((LoadNode->getExtensionType() == ISD::NON_EXTLOAD) ||
1341 (LoadNode->getExtensionType() == ISD::ZEXTLOAD))) {
1342 SDValue Result;
1343 if (isa<Constant>(Val: LoadNode->getMemOperand()->getValue()) ||
1344 isa<ConstantSDNode>(Val: Ptr)) {
1345 return constBufferLoad(LoadNode, Block: LoadNode->getAddressSpace(), DAG);
1346 }
1347 // TODO: Does this even work?
1348 // non-constant ptr can't be folded, keeps it as a v4f32 load
1349 Result = DAG.getNode(Opcode: AMDGPUISD::CONST_ADDRESS, DL, VT: MVT::v4i32,
1350 N1: DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Ptr,
1351 N2: DAG.getConstant(Val: 4, DL, VT: MVT::i32)),
1352 N2: DAG.getConstant(Val: LoadNode->getAddressSpace() -
1353 AMDGPUAS::CONSTANT_BUFFER_0,
1354 DL, VT: MVT::i32));
1355
1356 if (!VT.isVector()) {
1357 Result = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Result,
1358 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
1359 }
1360
1361 SDValue MergedValues[2] = {
1362 Result,
1363 Chain
1364 };
1365 return DAG.getMergeValues(Ops: MergedValues, dl: DL);
1366 }
1367
1368 // For most operations returning SDValue() will result in the node being
1369 // expanded by the DAG Legalizer. This is not the case for ISD::LOAD, so we
1370 // need to manually expand loads that may be legal in some address spaces and
1371 // illegal in others. SEXT loads from CONSTANT_BUFFER_0 are supported for
1372 // compute shaders, since the data is sign extended when it is uploaded to the
1373 // buffer. However SEXT loads from other address spaces are not supported, so
1374 // we need to expand them here.
1375 if (LoadNode->getExtensionType() == ISD::SEXTLOAD) {
1376 assert(!MemVT.isVector() && (MemVT == MVT::i16 || MemVT == MVT::i8));
1377 SDValue NewLoad = DAG.getExtLoad(
1378 ExtType: ISD::EXTLOAD, dl: DL, VT, Chain, Ptr, PtrInfo: LoadNode->getPointerInfo(), MemVT,
1379 Alignment: LoadNode->getAlign(), MMOFlags: LoadNode->getMemOperand()->getFlags());
1380 SDValue Res = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: NewLoad,
1381 N2: DAG.getValueType(MemVT));
1382
1383 SDValue MergedValues[2] = { Res, Chain };
1384 return DAG.getMergeValues(Ops: MergedValues, dl: DL);
1385 }
1386
1387 if (LoadNode->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) {
1388 return SDValue();
1389 }
1390
1391 // DWORDADDR ISD marks already shifted address
1392 if (Ptr.getOpcode() != AMDGPUISD::DWORDADDR) {
1393 assert(VT == MVT::i32);
1394 Ptr = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Ptr, N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32));
1395 Ptr = DAG.getNode(Opcode: AMDGPUISD::DWORDADDR, DL, VT: MVT::i32, Operand: Ptr);
1396 return DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr, MMO: LoadNode->getMemOperand());
1397 }
1398 return SDValue();
1399}
1400
1401SDValue R600TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
1402 SDValue Chain = Op.getOperand(i: 0);
1403 SDValue Cond = Op.getOperand(i: 1);
1404 SDValue Jump = Op.getOperand(i: 2);
1405
1406 return DAG.getNode(Opcode: AMDGPUISD::BRANCH_COND, DL: SDLoc(Op), VT: Op.getValueType(),
1407 N1: Chain, N2: Jump, N3: Cond);
1408}
1409
1410SDValue R600TargetLowering::lowerFrameIndex(SDValue Op,
1411 SelectionDAG &DAG) const {
1412 MachineFunction &MF = DAG.getMachineFunction();
1413 const R600FrameLowering *TFL = Subtarget->getFrameLowering();
1414
1415 FrameIndexSDNode *FIN = cast<FrameIndexSDNode>(Val&: Op);
1416
1417 unsigned FrameIndex = FIN->getIndex();
1418 Register IgnoredFrameReg;
1419 StackOffset Offset =
1420 TFL->getFrameIndexReference(MF, FI: FrameIndex, FrameReg&: IgnoredFrameReg);
1421 return DAG.getConstant(Val: Offset.getFixed() * 4 * TFL->getStackWidth(MF),
1422 DL: SDLoc(Op), VT: Op.getValueType());
1423}
1424
1425CCAssignFn *R600TargetLowering::CCAssignFnForCall(CallingConv::ID CC,
1426 bool IsVarArg) const {
1427 switch (CC) {
1428 case CallingConv::AMDGPU_KERNEL:
1429 case CallingConv::SPIR_KERNEL:
1430 case CallingConv::C:
1431 case CallingConv::Fast:
1432 case CallingConv::Cold:
1433 llvm_unreachable("kernels should not be handled here");
1434 case CallingConv::AMDGPU_VS:
1435 case CallingConv::AMDGPU_GS:
1436 case CallingConv::AMDGPU_PS:
1437 case CallingConv::AMDGPU_CS:
1438 case CallingConv::AMDGPU_HS:
1439 case CallingConv::AMDGPU_ES:
1440 case CallingConv::AMDGPU_LS:
1441 return CC_R600;
1442 default:
1443 reportFatalUsageError(reason: "unsupported calling convention");
1444 }
1445}
1446
1447/// XXX Only kernel functions are supported, so we can assume for now that
1448/// every function is a kernel function, but in the future we should use
1449/// separate calling conventions for kernel and non-kernel functions.
1450SDValue R600TargetLowering::LowerFormalArguments(
1451 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1452 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1453 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1454 SmallVector<CCValAssign, 16> ArgLocs;
1455 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
1456 *DAG.getContext());
1457 MachineFunction &MF = DAG.getMachineFunction();
1458
1459 if (AMDGPU::isShader(CC: CallConv)) {
1460 CCInfo.AnalyzeFormalArguments(Ins, Fn: CCAssignFnForCall(CC: CallConv, IsVarArg: isVarArg));
1461 } else {
1462 analyzeFormalArgumentsCompute(State&: CCInfo, Ins);
1463 }
1464
1465 for (unsigned i = 0, e = Ins.size(); i < e; ++i) {
1466 CCValAssign &VA = ArgLocs[i];
1467 const ISD::InputArg &In = Ins[i];
1468 EVT VT = In.VT;
1469 EVT MemVT = VA.getLocVT();
1470 if (!VT.isVector() && MemVT.isVector()) {
1471 // Get load source type if scalarized.
1472 MemVT = MemVT.getVectorElementType();
1473 }
1474
1475 if (VT.isInteger() && !MemVT.isInteger())
1476 MemVT = MemVT.changeTypeToInteger();
1477
1478 if (AMDGPU::isShader(CC: CallConv)) {
1479 Register Reg = MF.addLiveIn(PReg: VA.getLocReg(), RC: &R600::R600_Reg128RegClass);
1480 SDValue Register = DAG.getCopyFromReg(Chain, dl: DL, Reg, VT);
1481 InVals.push_back(Elt: Register);
1482 continue;
1483 }
1484
1485 // i64 isn't a legal type, so the register type used ends up as i32, which
1486 // isn't expected here. It attempts to create this sextload, but it ends up
1487 // being invalid. Somehow this seems to work with i64 arguments, but breaks
1488 // for <1 x i64>.
1489
1490 // The first 36 bytes of the input buffer contains information about
1491 // thread group and global sizes.
1492 ISD::LoadExtType Ext = ISD::NON_EXTLOAD;
1493 if (MemVT.getScalarSizeInBits() != VT.getScalarSizeInBits()) {
1494 if (VT.isFloatingPoint()) {
1495 Ext = ISD::EXTLOAD;
1496 } else {
1497 // FIXME: This should really check the extload type, but the handling of
1498 // extload vector parameters seems to be broken.
1499
1500 // Ext = In.Flags.isSExt() ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
1501 Ext = ISD::SEXTLOAD;
1502 }
1503 }
1504
1505 // Compute the offset from the value.
1506 // XXX - I think PartOffset should give you this, but it seems to give the
1507 // size of the register which isn't useful.
1508
1509 unsigned PartOffset = VA.getLocMemOffset();
1510 Align Alignment = commonAlignment(A: Align(VT.getStoreSize()), Offset: PartOffset);
1511
1512 MachinePointerInfo PtrInfo(AMDGPUAS::PARAM_I_ADDRESS);
1513 SDValue Arg = DAG.getLoad(
1514 AM: ISD::UNINDEXED, ExtType: Ext, VT, dl: DL, Chain,
1515 Ptr: DAG.getConstant(Val: PartOffset, DL, VT: MVT::i32), Offset: DAG.getUNDEF(VT: MVT::i32),
1516 PtrInfo,
1517 MemVT, Alignment, MMOFlags: MachineMemOperand::MONonTemporal |
1518 MachineMemOperand::MODereferenceable |
1519 MachineMemOperand::MOInvariant);
1520
1521 InVals.push_back(Elt: Arg);
1522 }
1523 return Chain;
1524}
1525
1526EVT R600TargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &,
1527 EVT VT) const {
1528 if (!VT.isVector())
1529 return MVT::i32;
1530 return VT.changeVectorElementTypeToInteger();
1531}
1532
1533bool R600TargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
1534 const MachineFunction &MF) const {
1535 // Local and Private addresses do not handle vectors. Limit to i32
1536 if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS)) {
1537 return (MemVT.getSizeInBits() <= 32);
1538 }
1539 return true;
1540}
1541
1542bool R600TargetLowering::allowsMisalignedMemoryAccesses(
1543 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
1544 unsigned *IsFast) const {
1545 if (IsFast)
1546 *IsFast = 0;
1547
1548 if (!VT.isSimple() || VT == MVT::Other)
1549 return false;
1550
1551 if (VT.bitsLT(VT: MVT::i32))
1552 return false;
1553
1554 // TODO: This is a rough estimate.
1555 if (IsFast)
1556 *IsFast = 1;
1557
1558 return VT.bitsGT(VT: MVT::i32) && Alignment >= Align(4);
1559}
1560
1561static SDValue CompactSwizzlableVector(
1562 SelectionDAG &DAG, SDValue VectorEntry,
1563 DenseMap<unsigned, unsigned> &RemapSwizzle) {
1564 assert(RemapSwizzle.empty());
1565
1566 SDLoc DL(VectorEntry);
1567 EVT EltTy = VectorEntry.getValueType().getVectorElementType();
1568
1569 SDValue NewBldVec[4];
1570 for (unsigned i = 0; i < 4; i++)
1571 NewBldVec[i] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltTy, N1: VectorEntry,
1572 N2: DAG.getIntPtrConstant(Val: i, DL));
1573
1574 for (unsigned i = 0; i < 4; i++) {
1575 if (NewBldVec[i].isUndef())
1576 // We mask write here to teach later passes that the ith element of this
1577 // vector is undef. Thus we can use it to reduce 128 bits reg usage,
1578 // break false dependencies and additionally make assembly easier to read.
1579 RemapSwizzle[i] = 7; // SEL_MASK_WRITE
1580 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: NewBldVec[i])) {
1581 if (C->isZero()) {
1582 RemapSwizzle[i] = 4; // SEL_0
1583 NewBldVec[i] = DAG.getUNDEF(VT: MVT::f32);
1584 } else if (C->isOne()) {
1585 RemapSwizzle[i] = 5; // SEL_1
1586 NewBldVec[i] = DAG.getUNDEF(VT: MVT::f32);
1587 }
1588 }
1589
1590 if (NewBldVec[i].isUndef())
1591 continue;
1592
1593 for (unsigned j = 0; j < i; j++) {
1594 if (NewBldVec[i] == NewBldVec[j]) {
1595 NewBldVec[i] = DAG.getUNDEF(VT: NewBldVec[i].getValueType());
1596 RemapSwizzle[i] = j;
1597 break;
1598 }
1599 }
1600 }
1601
1602 return DAG.getBuildVector(VT: VectorEntry.getValueType(), DL: SDLoc(VectorEntry),
1603 Ops: NewBldVec);
1604}
1605
1606static SDValue ReorganizeVector(SelectionDAG &DAG, SDValue VectorEntry,
1607 DenseMap<unsigned, unsigned> &RemapSwizzle) {
1608 assert(RemapSwizzle.empty());
1609
1610 SDLoc DL(VectorEntry);
1611 EVT EltTy = VectorEntry.getValueType().getVectorElementType();
1612
1613 SDValue NewBldVec[4];
1614 bool isUnmovable[4] = {false, false, false, false};
1615 for (unsigned i = 0; i < 4; i++)
1616 NewBldVec[i] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltTy, N1: VectorEntry,
1617 N2: DAG.getIntPtrConstant(Val: i, DL));
1618
1619 for (unsigned i = 0; i < 4; i++) {
1620 RemapSwizzle[i] = i;
1621 if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1622 unsigned Idx = NewBldVec[i].getConstantOperandVal(i: 1);
1623 if (i == Idx)
1624 isUnmovable[Idx] = true;
1625 }
1626 }
1627
1628 for (unsigned i = 0; i < 4; i++) {
1629 if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1630 unsigned Idx = NewBldVec[i].getConstantOperandVal(i: 1);
1631 if (isUnmovable[Idx])
1632 continue;
1633 // Swap i and Idx
1634 std::swap(a&: NewBldVec[Idx], b&: NewBldVec[i]);
1635 std::swap(a&: RemapSwizzle[i], b&: RemapSwizzle[Idx]);
1636 break;
1637 }
1638 }
1639
1640 return DAG.getBuildVector(VT: VectorEntry.getValueType(), DL: SDLoc(VectorEntry),
1641 Ops: NewBldVec);
1642}
1643
1644SDValue R600TargetLowering::OptimizeSwizzle(SDValue BuildVector, SDValue Swz[],
1645 SelectionDAG &DAG,
1646 const SDLoc &DL) const {
1647 // Old -> New swizzle values
1648 DenseMap<unsigned, unsigned> SwizzleRemap;
1649
1650 BuildVector = CompactSwizzlableVector(DAG, VectorEntry: BuildVector, RemapSwizzle&: SwizzleRemap);
1651 for (unsigned i = 0; i < 4; i++) {
1652 unsigned Idx = Swz[i]->getAsZExtVal();
1653 auto It = SwizzleRemap.find(Val: Idx);
1654 if (It != SwizzleRemap.end())
1655 Swz[i] = DAG.getConstant(Val: It->second, DL, VT: MVT::i32);
1656 }
1657
1658 SwizzleRemap.clear();
1659 BuildVector = ReorganizeVector(DAG, VectorEntry: BuildVector, RemapSwizzle&: SwizzleRemap);
1660 for (unsigned i = 0; i < 4; i++) {
1661 unsigned Idx = Swz[i]->getAsZExtVal();
1662 auto It = SwizzleRemap.find(Val: Idx);
1663 if (It != SwizzleRemap.end())
1664 Swz[i] = DAG.getConstant(Val: It->second, DL, VT: MVT::i32);
1665 }
1666
1667 return BuildVector;
1668}
1669
1670SDValue R600TargetLowering::constBufferLoad(LoadSDNode *LoadNode, int Block,
1671 SelectionDAG &DAG) const {
1672 SDLoc DL(LoadNode);
1673 EVT VT = LoadNode->getValueType(ResNo: 0);
1674 SDValue Chain = LoadNode->getChain();
1675 SDValue Ptr = LoadNode->getBasePtr();
1676 assert (isa<ConstantSDNode>(Ptr));
1677
1678 //TODO: Support smaller loads
1679 if (LoadNode->getMemoryVT().getScalarType() != MVT::i32 || !ISD::isNON_EXTLoad(N: LoadNode))
1680 return SDValue();
1681
1682 if (LoadNode->getAlign() < Align(4))
1683 return SDValue();
1684
1685 int ConstantBlock = ConstantAddressBlock(AddressSpace: Block);
1686
1687 SDValue Slots[4];
1688 for (unsigned i = 0; i < 4; i++) {
1689 // We want Const position encoded with the following formula :
1690 // (((512 + (kc_bank << 12) + const_index) << 2) + chan)
1691 // const_index is Ptr computed by llvm using an alignment of 16.
1692 // Thus we add (((512 + (kc_bank << 12)) + chan ) * 4 here and
1693 // then div by 4 at the ISel step
1694 SDValue NewPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: Ptr,
1695 N2: DAG.getConstant(Val: 4 * i + ConstantBlock * 16, DL, VT: MVT::i32));
1696 Slots[i] = DAG.getNode(Opcode: AMDGPUISD::CONST_ADDRESS, DL, VT: MVT::i32, Operand: NewPtr);
1697 }
1698 EVT NewVT = MVT::v4i32;
1699 unsigned NumElements = 4;
1700 if (VT.isVector()) {
1701 NewVT = VT;
1702 NumElements = VT.getVectorNumElements();
1703 }
1704 SDValue Result = DAG.getBuildVector(VT: NewVT, DL, Ops: ArrayRef(Slots, NumElements));
1705 if (!VT.isVector()) {
1706 Result = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Result,
1707 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
1708 }
1709 SDValue MergedValues[2] = {
1710 Result,
1711 Chain
1712 };
1713 return DAG.getMergeValues(Ops: MergedValues, dl: DL);
1714}
1715
1716//===----------------------------------------------------------------------===//
1717// Custom DAG Optimizations
1718//===----------------------------------------------------------------------===//
1719
1720SDValue R600TargetLowering::PerformDAGCombine(SDNode *N,
1721 DAGCombinerInfo &DCI) const {
1722 SelectionDAG &DAG = DCI.DAG;
1723 SDLoc DL(N);
1724
1725 switch (N->getOpcode()) {
1726 // (f32 fp_round (f64 uint_to_fp a)) -> (f32 uint_to_fp a)
1727 case ISD::FP_ROUND: {
1728 SDValue Arg = N->getOperand(Num: 0);
1729 if (Arg.getOpcode() == ISD::UINT_TO_FP && Arg.getValueType() == MVT::f64) {
1730 return DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: N->getValueType(ResNo: 0),
1731 Operand: Arg.getOperand(i: 0));
1732 }
1733 break;
1734 }
1735
1736 // (i32 fp_to_sint (fneg (select_cc f32, f32, 1.0, 0.0 cc))) ->
1737 // (i32 select_cc f32, f32, -1, 0 cc)
1738 //
1739 // Mesa's GLSL frontend generates the above pattern a lot and we can lower
1740 // this to one of the SET*_DX10 instructions.
1741 case ISD::FP_TO_SINT: {
1742 SDValue FNeg = N->getOperand(Num: 0);
1743 if (FNeg.getOpcode() != ISD::FNEG) {
1744 return SDValue();
1745 }
1746 SDValue SelectCC = FNeg.getOperand(i: 0);
1747 if (SelectCC.getOpcode() != ISD::SELECT_CC ||
1748 SelectCC.getOperand(i: 0).getValueType() != MVT::f32 || // LHS
1749 SelectCC.getOperand(i: 2).getValueType() != MVT::f32 || // True
1750 !isHWTrueValue(Op: SelectCC.getOperand(i: 2)) ||
1751 !isHWFalseValue(Op: SelectCC.getOperand(i: 3))) {
1752 return SDValue();
1753 }
1754
1755 return DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT: N->getValueType(ResNo: 0),
1756 N1: SelectCC.getOperand(i: 0), // LHS
1757 N2: SelectCC.getOperand(i: 1), // RHS
1758 N3: DAG.getAllOnesConstant(DL, VT: MVT::i32), // True
1759 N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32), // False
1760 N5: SelectCC.getOperand(i: 4)); // CC
1761 }
1762
1763 // insert_vector_elt (build_vector elt0, ... , eltN), NewEltIdx, idx
1764 // => build_vector elt0, ... , NewEltIdx, ... , eltN
1765 case ISD::INSERT_VECTOR_ELT: {
1766 SDValue InVec = N->getOperand(Num: 0);
1767 SDValue InVal = N->getOperand(Num: 1);
1768 SDValue EltNo = N->getOperand(Num: 2);
1769
1770 // If the inserted element is an UNDEF, just use the input vector.
1771 if (InVal.isUndef())
1772 return InVec;
1773
1774 EVT VT = InVec.getValueType();
1775
1776 // If we can't generate a legal BUILD_VECTOR, exit
1777 if (!isOperationLegal(Op: ISD::BUILD_VECTOR, VT))
1778 return SDValue();
1779
1780 // Check that we know which element is being inserted
1781 if (!isa<ConstantSDNode>(Val: EltNo))
1782 return SDValue();
1783 unsigned Elt = EltNo->getAsZExtVal();
1784
1785 // Check that the operand is a BUILD_VECTOR (or UNDEF, which can essentially
1786 // be converted to a BUILD_VECTOR). Fill in the Ops vector with the
1787 // vector elements.
1788 SmallVector<SDValue, 8> Ops;
1789 if (InVec.getOpcode() == ISD::BUILD_VECTOR) {
1790 Ops.append(in_start: InVec.getNode()->op_begin(),
1791 in_end: InVec.getNode()->op_end());
1792 } else if (InVec.isUndef()) {
1793 unsigned NElts = VT.getVectorNumElements();
1794 Ops.append(NumInputs: NElts, Elt: DAG.getUNDEF(VT: InVal.getValueType()));
1795 } else {
1796 return SDValue();
1797 }
1798
1799 // Insert the element
1800 if (Elt < Ops.size()) {
1801 // All the operands of BUILD_VECTOR must have the same type;
1802 // we enforce that here.
1803 EVT OpVT = Ops[0].getValueType();
1804 if (InVal.getValueType() != OpVT)
1805 InVal = OpVT.bitsGT(VT: InVal.getValueType()) ?
1806 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: OpVT, Operand: InVal) :
1807 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OpVT, Operand: InVal);
1808 Ops[Elt] = InVal;
1809 }
1810
1811 // Return the new vector
1812 return DAG.getBuildVector(VT, DL, Ops);
1813 }
1814
1815 // Extract_vec (Build_vector) generated by custom lowering
1816 // also needs to be customly combined
1817 case ISD::EXTRACT_VECTOR_ELT: {
1818 SDValue Arg = N->getOperand(Num: 0);
1819 if (Arg.getOpcode() == ISD::BUILD_VECTOR) {
1820 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
1821 unsigned Element = Const->getZExtValue();
1822 return Arg->getOperand(Num: Element);
1823 }
1824 }
1825 if (Arg.getOpcode() == ISD::BITCAST &&
1826 Arg.getOperand(i: 0).getOpcode() == ISD::BUILD_VECTOR &&
1827 (Arg.getOperand(i: 0).getValueType().getVectorNumElements() ==
1828 Arg.getValueType().getVectorNumElements())) {
1829 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
1830 unsigned Element = Const->getZExtValue();
1831 return DAG.getNode(Opcode: ISD::BITCAST, DL, VTList: N->getVTList(),
1832 N: Arg->getOperand(Num: 0).getOperand(i: Element));
1833 }
1834 }
1835 break;
1836 }
1837
1838 case ISD::SELECT_CC: {
1839 // Try common optimizations
1840 if (SDValue Ret = AMDGPUTargetLowering::PerformDAGCombine(N, DCI))
1841 return Ret;
1842
1843 // fold selectcc (selectcc x, y, a, b, cc), b, a, b, seteq ->
1844 // selectcc x, y, a, b, inv(cc)
1845 //
1846 // fold selectcc (selectcc x, y, a, b, cc), b, a, b, setne ->
1847 // selectcc x, y, a, b, cc
1848 SDValue LHS = N->getOperand(Num: 0);
1849 if (LHS.getOpcode() != ISD::SELECT_CC) {
1850 return SDValue();
1851 }
1852
1853 SDValue RHS = N->getOperand(Num: 1);
1854 SDValue True = N->getOperand(Num: 2);
1855 SDValue False = N->getOperand(Num: 3);
1856 ISD::CondCode NCC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
1857
1858 if (LHS.getOperand(i: 2).getNode() != True.getNode() ||
1859 LHS.getOperand(i: 3).getNode() != False.getNode() ||
1860 RHS.getNode() != False.getNode()) {
1861 return SDValue();
1862 }
1863
1864 switch (NCC) {
1865 default: return SDValue();
1866 case ISD::SETNE: return LHS;
1867 case ISD::SETEQ: {
1868 ISD::CondCode LHSCC = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 4))->get();
1869 LHSCC = ISD::getSetCCInverse(Operation: LHSCC, Type: LHS.getOperand(i: 0).getValueType());
1870 if (DCI.isBeforeLegalizeOps() ||
1871 isCondCodeLegal(CC: LHSCC, VT: LHS.getOperand(i: 0).getSimpleValueType()))
1872 return DAG.getSelectCC(DL,
1873 LHS: LHS.getOperand(i: 0),
1874 RHS: LHS.getOperand(i: 1),
1875 True: LHS.getOperand(i: 2),
1876 False: LHS.getOperand(i: 3),
1877 Cond: LHSCC);
1878 break;
1879 }
1880 }
1881 return SDValue();
1882 }
1883
1884 case AMDGPUISD::R600_EXPORT: {
1885 SDValue Arg = N->getOperand(Num: 1);
1886 if (Arg.getOpcode() != ISD::BUILD_VECTOR)
1887 break;
1888
1889 SDValue NewArgs[8] = {
1890 N->getOperand(Num: 0), // Chain
1891 SDValue(),
1892 N->getOperand(Num: 2), // ArrayBase
1893 N->getOperand(Num: 3), // Type
1894 N->getOperand(Num: 4), // SWZ_X
1895 N->getOperand(Num: 5), // SWZ_Y
1896 N->getOperand(Num: 6), // SWZ_Z
1897 N->getOperand(Num: 7) // SWZ_W
1898 };
1899 NewArgs[1] = OptimizeSwizzle(BuildVector: N->getOperand(Num: 1), Swz: &NewArgs[4], DAG, DL);
1900 return DAG.getNode(Opcode: AMDGPUISD::R600_EXPORT, DL, VTList: N->getVTList(), Ops: NewArgs);
1901 }
1902 case AMDGPUISD::TEXTURE_FETCH: {
1903 SDValue Arg = N->getOperand(Num: 1);
1904 if (Arg.getOpcode() != ISD::BUILD_VECTOR)
1905 break;
1906
1907 SDValue NewArgs[19] = {
1908 N->getOperand(Num: 0),
1909 N->getOperand(Num: 1),
1910 N->getOperand(Num: 2),
1911 N->getOperand(Num: 3),
1912 N->getOperand(Num: 4),
1913 N->getOperand(Num: 5),
1914 N->getOperand(Num: 6),
1915 N->getOperand(Num: 7),
1916 N->getOperand(Num: 8),
1917 N->getOperand(Num: 9),
1918 N->getOperand(Num: 10),
1919 N->getOperand(Num: 11),
1920 N->getOperand(Num: 12),
1921 N->getOperand(Num: 13),
1922 N->getOperand(Num: 14),
1923 N->getOperand(Num: 15),
1924 N->getOperand(Num: 16),
1925 N->getOperand(Num: 17),
1926 N->getOperand(Num: 18),
1927 };
1928 NewArgs[1] = OptimizeSwizzle(BuildVector: N->getOperand(Num: 1), Swz: &NewArgs[2], DAG, DL);
1929 return DAG.getNode(Opcode: AMDGPUISD::TEXTURE_FETCH, DL, VTList: N->getVTList(), Ops: NewArgs);
1930 }
1931
1932 case ISD::LOAD: {
1933 LoadSDNode *LoadNode = cast<LoadSDNode>(Val: N);
1934 SDValue Ptr = LoadNode->getBasePtr();
1935 if (LoadNode->getAddressSpace() == AMDGPUAS::PARAM_I_ADDRESS &&
1936 isa<ConstantSDNode>(Val: Ptr))
1937 return constBufferLoad(LoadNode, Block: AMDGPUAS::CONSTANT_BUFFER_0, DAG);
1938 break;
1939 }
1940
1941 default: break;
1942 }
1943
1944 return AMDGPUTargetLowering::PerformDAGCombine(N, DCI);
1945}
1946
1947bool R600TargetLowering::FoldOperand(SDNode *ParentNode, unsigned SrcIdx,
1948 SDValue &Src, SDValue &Neg, SDValue &Abs,
1949 SDValue &Sel, SDValue &Imm,
1950 SelectionDAG &DAG) const {
1951 const R600InstrInfo *TII = Subtarget->getInstrInfo();
1952 if (!Src.isMachineOpcode())
1953 return false;
1954
1955 switch (Src.getMachineOpcode()) {
1956 case R600::FNEG_R600:
1957 if (!Neg.getNode())
1958 return false;
1959 Src = Src.getOperand(i: 0);
1960 Neg = DAG.getTargetConstant(Val: 1, DL: SDLoc(ParentNode), VT: MVT::i32);
1961 return true;
1962 case R600::FABS_R600:
1963 if (!Abs.getNode())
1964 return false;
1965 Src = Src.getOperand(i: 0);
1966 Abs = DAG.getTargetConstant(Val: 1, DL: SDLoc(ParentNode), VT: MVT::i32);
1967 return true;
1968 case R600::CONST_COPY: {
1969 unsigned Opcode = ParentNode->getMachineOpcode();
1970 bool HasDst = TII->getOperandIdx(Opcode, Op: R600::OpName::dst) > -1;
1971
1972 if (!Sel.getNode())
1973 return false;
1974
1975 SDValue CstOffset = Src.getOperand(i: 0);
1976 if (ParentNode->getValueType(ResNo: 0).isVector())
1977 return false;
1978
1979 // Gather constants values
1980 int SrcIndices[] = {
1981 TII->getOperandIdx(Opcode, Op: R600::OpName::src0),
1982 TII->getOperandIdx(Opcode, Op: R600::OpName::src1),
1983 TII->getOperandIdx(Opcode, Op: R600::OpName::src2),
1984 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_X),
1985 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_Y),
1986 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_Z),
1987 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_W),
1988 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_X),
1989 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_Y),
1990 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_Z),
1991 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_W)
1992 };
1993 std::vector<unsigned> Consts;
1994 for (int OtherSrcIdx : SrcIndices) {
1995 int OtherSelIdx = TII->getSelIdx(Opcode, SrcIdx: OtherSrcIdx);
1996 if (OtherSrcIdx < 0 || OtherSelIdx < 0)
1997 continue;
1998 if (HasDst) {
1999 OtherSrcIdx--;
2000 OtherSelIdx--;
2001 }
2002 if (RegisterSDNode *Reg =
2003 dyn_cast<RegisterSDNode>(Val: ParentNode->getOperand(Num: OtherSrcIdx))) {
2004 if (Reg->getReg() == R600::ALU_CONST) {
2005 Consts.push_back(x: ParentNode->getConstantOperandVal(Num: OtherSelIdx));
2006 }
2007 }
2008 }
2009
2010 ConstantSDNode *Cst = cast<ConstantSDNode>(Val&: CstOffset);
2011 Consts.push_back(x: Cst->getZExtValue());
2012 if (!TII->fitsConstReadLimitations(Consts)) {
2013 return false;
2014 }
2015
2016 Sel = CstOffset;
2017 Src = DAG.getRegister(Reg: R600::ALU_CONST, VT: MVT::f32);
2018 return true;
2019 }
2020 case R600::MOV_IMM_GLOBAL_ADDR:
2021 // Check if the Imm slot is used. Taken from below.
2022 if (Imm->getAsZExtVal())
2023 return false;
2024 Imm = Src.getOperand(i: 0);
2025 Src = DAG.getRegister(Reg: R600::ALU_LITERAL_X, VT: MVT::i32);
2026 return true;
2027 case R600::MOV_IMM_I32:
2028 case R600::MOV_IMM_F32: {
2029 unsigned ImmReg = R600::ALU_LITERAL_X;
2030 uint64_t ImmValue = 0;
2031
2032 if (Src.getMachineOpcode() == R600::MOV_IMM_F32) {
2033 ConstantFPSDNode *FPC = cast<ConstantFPSDNode>(Val: Src.getOperand(i: 0));
2034 float FloatValue = FPC->getValueAPF().convertToFloat();
2035 if (FloatValue == 0.0) {
2036 ImmReg = R600::ZERO;
2037 } else if (FloatValue == 0.5) {
2038 ImmReg = R600::HALF;
2039 } else if (FloatValue == 1.0) {
2040 ImmReg = R600::ONE;
2041 } else {
2042 ImmValue = FPC->getValueAPF().bitcastToAPInt().getZExtValue();
2043 }
2044 } else {
2045 uint64_t Value = Src.getConstantOperandVal(i: 0);
2046 if (Value == 0) {
2047 ImmReg = R600::ZERO;
2048 } else if (Value == 1) {
2049 ImmReg = R600::ONE_INT;
2050 } else {
2051 ImmValue = Value;
2052 }
2053 }
2054
2055 // Check that we aren't already using an immediate.
2056 // XXX: It's possible for an instruction to have more than one
2057 // immediate operand, but this is not supported yet.
2058 if (ImmReg == R600::ALU_LITERAL_X) {
2059 if (!Imm.getNode())
2060 return false;
2061 ConstantSDNode *C = cast<ConstantSDNode>(Val&: Imm);
2062 if (C->getZExtValue())
2063 return false;
2064 Imm = DAG.getTargetConstant(Val: ImmValue, DL: SDLoc(ParentNode), VT: MVT::i32);
2065 }
2066 Src = DAG.getRegister(Reg: ImmReg, VT: MVT::i32);
2067 return true;
2068 }
2069 default:
2070 return false;
2071 }
2072}
2073
2074/// Fold the instructions after selecting them
2075SDNode *R600TargetLowering::PostISelFolding(MachineSDNode *Node,
2076 SelectionDAG &DAG) const {
2077 const R600InstrInfo *TII = Subtarget->getInstrInfo();
2078 if (!Node->isMachineOpcode())
2079 return Node;
2080
2081 unsigned Opcode = Node->getMachineOpcode();
2082 SDValue FakeOp;
2083
2084 std::vector<SDValue> Ops(Node->op_begin(), Node->op_end());
2085
2086 if (Opcode == R600::DOT_4) {
2087 int OperandIdx[] = {
2088 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_X),
2089 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_Y),
2090 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_Z),
2091 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_W),
2092 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_X),
2093 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_Y),
2094 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_Z),
2095 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_W)
2096 };
2097 int NegIdx[] = {
2098 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_neg_X),
2099 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_neg_Y),
2100 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_neg_Z),
2101 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_neg_W),
2102 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_neg_X),
2103 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_neg_Y),
2104 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_neg_Z),
2105 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_neg_W)
2106 };
2107 int AbsIdx[] = {
2108 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_abs_X),
2109 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_abs_Y),
2110 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_abs_Z),
2111 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_abs_W),
2112 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_abs_X),
2113 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_abs_Y),
2114 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_abs_Z),
2115 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_abs_W)
2116 };
2117 for (unsigned i = 0; i < 8; i++) {
2118 if (OperandIdx[i] < 0)
2119 return Node;
2120 SDValue &Src = Ops[OperandIdx[i] - 1];
2121 SDValue &Neg = Ops[NegIdx[i] - 1];
2122 SDValue &Abs = Ops[AbsIdx[i] - 1];
2123 bool HasDst = TII->getOperandIdx(Opcode, Op: R600::OpName::dst) > -1;
2124 int SelIdx = TII->getSelIdx(Opcode, SrcIdx: OperandIdx[i]);
2125 if (HasDst)
2126 SelIdx--;
2127 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp;
2128 if (FoldOperand(ParentNode: Node, SrcIdx: i, Src, Neg, Abs, Sel, Imm&: FakeOp, DAG))
2129 return DAG.getMachineNode(Opcode, dl: SDLoc(Node), VTs: Node->getVTList(), Ops);
2130 }
2131 } else if (Opcode == R600::REG_SEQUENCE) {
2132 for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2) {
2133 SDValue &Src = Ops[i];
2134 if (FoldOperand(ParentNode: Node, SrcIdx: i, Src, Neg&: FakeOp, Abs&: FakeOp, Sel&: FakeOp, Imm&: FakeOp, DAG))
2135 return DAG.getMachineNode(Opcode, dl: SDLoc(Node), VTs: Node->getVTList(), Ops);
2136 }
2137 } else {
2138 if (!TII->hasInstrModifiers(Opcode))
2139 return Node;
2140 int OperandIdx[] = {
2141 TII->getOperandIdx(Opcode, Op: R600::OpName::src0),
2142 TII->getOperandIdx(Opcode, Op: R600::OpName::src1),
2143 TII->getOperandIdx(Opcode, Op: R600::OpName::src2)
2144 };
2145 int NegIdx[] = {
2146 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_neg),
2147 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_neg),
2148 TII->getOperandIdx(Opcode, Op: R600::OpName::src2_neg)
2149 };
2150 int AbsIdx[] = {
2151 TII->getOperandIdx(Opcode, Op: R600::OpName::src0_abs),
2152 TII->getOperandIdx(Opcode, Op: R600::OpName::src1_abs),
2153 -1
2154 };
2155 for (unsigned i = 0; i < 3; i++) {
2156 if (OperandIdx[i] < 0)
2157 return Node;
2158 SDValue &Src = Ops[OperandIdx[i] - 1];
2159 SDValue &Neg = Ops[NegIdx[i] - 1];
2160 SDValue FakeAbs;
2161 SDValue &Abs = (AbsIdx[i] > -1) ? Ops[AbsIdx[i] - 1] : FakeAbs;
2162 bool HasDst = TII->getOperandIdx(Opcode, Op: R600::OpName::dst) > -1;
2163 int SelIdx = TII->getSelIdx(Opcode, SrcIdx: OperandIdx[i]);
2164 int ImmIdx = TII->getOperandIdx(Opcode, Op: R600::OpName::literal);
2165 if (HasDst) {
2166 SelIdx--;
2167 ImmIdx--;
2168 }
2169 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp;
2170 SDValue &Imm = Ops[ImmIdx];
2171 if (FoldOperand(ParentNode: Node, SrcIdx: i, Src, Neg, Abs, Sel, Imm, DAG))
2172 return DAG.getMachineNode(Opcode, dl: SDLoc(Node), VTs: Node->getVTList(), Ops);
2173 }
2174 }
2175
2176 return Node;
2177}
2178
2179TargetLowering::AtomicExpansionKind
2180R600TargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
2181 switch (RMW->getOperation()) {
2182 case AtomicRMWInst::Nand:
2183 case AtomicRMWInst::FAdd:
2184 case AtomicRMWInst::FSub:
2185 case AtomicRMWInst::FMax:
2186 case AtomicRMWInst::FMin:
2187 case AtomicRMWInst::USubCond:
2188 case AtomicRMWInst::USubSat:
2189 return AtomicExpansionKind::CmpXChg;
2190 case AtomicRMWInst::UIncWrap:
2191 case AtomicRMWInst::UDecWrap:
2192 // FIXME: Cayman at least appears to have instructions for this, but the
2193 // instruction definitions appear to be missing.
2194 return AtomicExpansionKind::CmpXChg;
2195 case AtomicRMWInst::Xchg: {
2196 const DataLayout &DL = RMW->getFunction()->getDataLayout();
2197 unsigned ValSize = DL.getTypeSizeInBits(Ty: RMW->getType());
2198 if (ValSize == 32 || ValSize == 64)
2199 return AtomicExpansionKind::None;
2200 return AtomicExpansionKind::CmpXChg;
2201 }
2202 default:
2203 if (auto *IntTy = dyn_cast<IntegerType>(Val: RMW->getType())) {
2204 unsigned Size = IntTy->getBitWidth();
2205 if (Size == 32 || Size == 64)
2206 return AtomicExpansionKind::None;
2207 }
2208
2209 return AtomicExpansionKind::CmpXChg;
2210 }
2211
2212 llvm_unreachable("covered atomicrmw op switch");
2213}
2214