1//===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Custom DAG lowering for SI
11//
12//===----------------------------------------------------------------------===//
13
14#include "SIISelLowering.h"
15#include "AMDGPU.h"
16#include "AMDGPUInstrInfo.h"
17#include "AMDGPULaneMaskUtils.h"
18#include "AMDGPUSelectionDAGInfo.h"
19#include "AMDGPUTargetMachine.h"
20#include "GCNSubtarget.h"
21#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
22#include "SIMachineFunctionInfo.h"
23#include "SIRegisterInfo.h"
24#include "llvm/ADT/APInt.h"
25#include "llvm/ADT/FloatingPointMode.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/Analysis/OptimizationRemarkEmitter.h"
28#include "llvm/Analysis/UniformityAnalysis.h"
29#include "llvm/CodeGen/Analysis.h"
30#include "llvm/CodeGen/ByteProvider.h"
31#include "llvm/CodeGen/FunctionLoweringInfo.h"
32#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
33#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
34#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
35#include "llvm/CodeGen/MachineFrameInfo.h"
36#include "llvm/CodeGen/MachineFunction.h"
37#include "llvm/CodeGen/MachineLoopInfo.h"
38#include "llvm/CodeGen/PseudoSourceValueManager.h"
39#include "llvm/CodeGen/SDPatternMatch.h"
40#include "llvm/IR/DiagnosticInfo.h"
41#include "llvm/IR/IRBuilder.h"
42#include "llvm/IR/IntrinsicInst.h"
43#include "llvm/IR/IntrinsicsAMDGPU.h"
44#include "llvm/IR/IntrinsicsR600.h"
45#include "llvm/IR/MDBuilder.h"
46#include "llvm/Support/CommandLine.h"
47#include "llvm/Support/KnownBits.h"
48#include "llvm/Support/ModRef.h"
49#include "llvm/Transforms/Utils/LowerAtomic.h"
50#include <optional>
51
52using namespace llvm;
53using namespace llvm::SDPatternMatch;
54
55#define DEBUG_TYPE "si-lower"
56
57STATISTIC(NumTailCalls, "Number of tail calls");
58
59static cl::opt<bool>
60 DisableLoopAlignment("amdgpu-disable-loop-alignment",
61 cl::desc("Do not align and prefetch loops"),
62 cl::init(Val: false));
63
64static cl::opt<bool> UseDivergentRegisterIndexing(
65 "amdgpu-use-divergent-register-indexing", cl::Hidden,
66 cl::desc("Use indirect register addressing for divergent indexes"),
67 cl::init(Val: false));
68
69static bool denormalModeIsFlushAllF32(const MachineFunction &MF) {
70 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
71 return Info->getMode().FP32Denormals == DenormalMode::getPreserveSign();
72}
73
74static bool denormalModeIsFlushAllF64F16(const MachineFunction &MF) {
75 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
76 return Info->getMode().FP64FP16Denormals == DenormalMode::getPreserveSign();
77}
78
79static unsigned findFirstFreeSGPR(CCState &CCInfo) {
80 unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
81 for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) {
82 if (!CCInfo.isAllocated(Reg: AMDGPU::SGPR0 + Reg)) {
83 return AMDGPU::SGPR0 + Reg;
84 }
85 }
86 llvm_unreachable("Cannot allocate sgpr");
87}
88
89SITargetLowering::SITargetLowering(const TargetMachine &TM,
90 const GCNSubtarget &STI)
91 : AMDGPUTargetLowering(TM, STI, STI), Subtarget(&STI) {
92 addRegisterClass(VT: MVT::i1, RC: &AMDGPU::VReg_1RegClass);
93 addRegisterClass(VT: MVT::i64, RC: &AMDGPU::SReg_64RegClass);
94
95 addRegisterClass(VT: MVT::i32, RC: &AMDGPU::SReg_32RegClass);
96
97 const SIRegisterInfo *TRI = STI.getRegisterInfo();
98 const TargetRegisterClass *V32RegClass =
99 TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 32);
100 addRegisterClass(VT: MVT::f32, RC: V32RegClass);
101
102 addRegisterClass(VT: MVT::v2i32, RC: &AMDGPU::SReg_64RegClass);
103
104 const TargetRegisterClass *V64RegClass =
105 TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 64);
106
107 addRegisterClass(VT: MVT::f64, RC: V64RegClass);
108 addRegisterClass(VT: MVT::v2f32, RC: V64RegClass);
109 addRegisterClass(VT: MVT::Untyped, RC: V64RegClass);
110
111 addRegisterClass(VT: MVT::v3i32, RC: &AMDGPU::SGPR_96RegClass);
112 addRegisterClass(VT: MVT::v3f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 96));
113
114 addRegisterClass(VT: MVT::v2i64, RC: &AMDGPU::SGPR_128RegClass);
115 addRegisterClass(VT: MVT::v2f64, RC: &AMDGPU::SGPR_128RegClass);
116
117 addRegisterClass(VT: MVT::v4i32, RC: &AMDGPU::SGPR_128RegClass);
118 addRegisterClass(VT: MVT::v4f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 128));
119
120 addRegisterClass(VT: MVT::v5i32, RC: &AMDGPU::SGPR_160RegClass);
121 addRegisterClass(VT: MVT::v5f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 160));
122
123 addRegisterClass(VT: MVT::v6i32, RC: &AMDGPU::SGPR_192RegClass);
124 addRegisterClass(VT: MVT::v6f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 192));
125
126 addRegisterClass(VT: MVT::v3i64, RC: &AMDGPU::SGPR_192RegClass);
127 addRegisterClass(VT: MVT::v3f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 192));
128
129 addRegisterClass(VT: MVT::v7i32, RC: &AMDGPU::SGPR_224RegClass);
130 addRegisterClass(VT: MVT::v7f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 224));
131
132 addRegisterClass(VT: MVT::v8i32, RC: &AMDGPU::SGPR_256RegClass);
133 addRegisterClass(VT: MVT::v8f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 256));
134
135 addRegisterClass(VT: MVT::v4i64, RC: &AMDGPU::SGPR_256RegClass);
136 addRegisterClass(VT: MVT::v4f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 256));
137
138 addRegisterClass(VT: MVT::v9i32, RC: &AMDGPU::SGPR_288RegClass);
139 addRegisterClass(VT: MVT::v9f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 288));
140
141 addRegisterClass(VT: MVT::v10i32, RC: &AMDGPU::SGPR_320RegClass);
142 addRegisterClass(VT: MVT::v10f32,
143 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 320));
144
145 addRegisterClass(VT: MVT::v11i32, RC: &AMDGPU::SGPR_352RegClass);
146 addRegisterClass(VT: MVT::v11f32,
147 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 352));
148
149 addRegisterClass(VT: MVT::v12i32, RC: &AMDGPU::SGPR_384RegClass);
150 addRegisterClass(VT: MVT::v12f32,
151 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 384));
152
153 addRegisterClass(VT: MVT::v16i32, RC: &AMDGPU::SGPR_512RegClass);
154 addRegisterClass(VT: MVT::v16f32,
155 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 512));
156
157 addRegisterClass(VT: MVT::v8i64, RC: &AMDGPU::SGPR_512RegClass);
158 addRegisterClass(VT: MVT::v8f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 512));
159
160 addRegisterClass(VT: MVT::v16i64, RC: &AMDGPU::SGPR_1024RegClass);
161 addRegisterClass(VT: MVT::v16f64,
162 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 1024));
163
164 if (Subtarget->has16BitInsts()) {
165 if (Subtarget->useRealTrue16Insts()) {
166 addRegisterClass(VT: MVT::i16, RC: &AMDGPU::VGPR_16RegClass);
167 addRegisterClass(VT: MVT::f16, RC: &AMDGPU::VGPR_16RegClass);
168 addRegisterClass(VT: MVT::bf16, RC: &AMDGPU::VGPR_16RegClass);
169 } else {
170 addRegisterClass(VT: MVT::i16, RC: &AMDGPU::SReg_32RegClass);
171 addRegisterClass(VT: MVT::f16, RC: &AMDGPU::SReg_32RegClass);
172 addRegisterClass(VT: MVT::bf16, RC: &AMDGPU::SReg_32RegClass);
173 }
174
175 // Unless there are also VOP3P operations, not operations are really legal.
176 addRegisterClass(VT: MVT::v2i16, RC: &AMDGPU::SReg_32RegClass);
177 addRegisterClass(VT: MVT::v2f16, RC: &AMDGPU::SReg_32RegClass);
178 addRegisterClass(VT: MVT::v2bf16, RC: &AMDGPU::SReg_32RegClass);
179 addRegisterClass(VT: MVT::v4i16, RC: &AMDGPU::SReg_64RegClass);
180 addRegisterClass(VT: MVT::v4f16, RC: &AMDGPU::SReg_64RegClass);
181 addRegisterClass(VT: MVT::v4bf16, RC: &AMDGPU::SReg_64RegClass);
182 addRegisterClass(VT: MVT::v8i16, RC: &AMDGPU::SGPR_128RegClass);
183 addRegisterClass(VT: MVT::v8f16, RC: &AMDGPU::SGPR_128RegClass);
184 addRegisterClass(VT: MVT::v8bf16, RC: &AMDGPU::SGPR_128RegClass);
185 addRegisterClass(VT: MVT::v16i16, RC: &AMDGPU::SGPR_256RegClass);
186 addRegisterClass(VT: MVT::v16f16, RC: &AMDGPU::SGPR_256RegClass);
187 addRegisterClass(VT: MVT::v16bf16, RC: &AMDGPU::SGPR_256RegClass);
188 addRegisterClass(VT: MVT::v32i16, RC: &AMDGPU::SGPR_512RegClass);
189 addRegisterClass(VT: MVT::v32f16, RC: &AMDGPU::SGPR_512RegClass);
190 addRegisterClass(VT: MVT::v32bf16, RC: &AMDGPU::SGPR_512RegClass);
191 }
192
193 addRegisterClass(VT: MVT::v32i32, RC: &AMDGPU::VReg_1024RegClass);
194 addRegisterClass(VT: MVT::v32f32,
195 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 1024));
196
197 computeRegisterProperties(TRI: Subtarget->getRegisterInfo());
198
199 setMinFunctionAlignment(Align(4));
200 setPrefFunctionAlignment(Align(STI.getInstCacheLineSize()));
201
202 // The boolean content concept here is too inflexible. Compares only ever
203 // really produce a 1-bit result. Any copy/extend from these will turn into a
204 // select, and zext/1 or sext/-1 are equally cheap. Arbitrarily choose 0/1, as
205 // it's what most targets use.
206 setBooleanContents(ZeroOrOneBooleanContent);
207 setBooleanVectorContents(ZeroOrOneBooleanContent);
208
209 // We need to custom lower vector stores from local memory
210 setOperationAction(Ops: ISD::LOAD,
211 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
212 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
213 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
214 MVT::i1, MVT::v32i32},
215 Action: Custom);
216
217 setOperationAction(Ops: ISD::STORE,
218 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
219 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
220 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
221 MVT::i1, MVT::v32i32},
222 Action: Custom);
223
224 if (isTypeLegal(VT: MVT::bf16)) {
225 for (unsigned Opc :
226 {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
227 ISD::FREM, ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
228 ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FSQRT, ISD::FCBRT,
229 ISD::FSIN, ISD::FCOS, ISD::FPOW, ISD::FPOWI,
230 ISD::FLDEXP, ISD::FFREXP, ISD::FLOG, ISD::FLOG2,
231 ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FEXP10,
232 ISD::FCEIL, ISD::FTRUNC, ISD::FRINT, ISD::FNEARBYINT,
233 ISD::FROUND, ISD::FROUNDEVEN, ISD::FFLOOR, ISD::FCANONICALIZE,
234 ISD::SETCC}) {
235 setOperationAction(Op: Opc, VT: MVT::bf16, Action: Promote);
236 }
237
238 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::bf16, Action: Expand);
239
240 setOperationAction(Op: ISD::SELECT, VT: MVT::bf16, Action: Promote);
241 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::bf16, DestVT: MVT::i16);
242
243 setOperationAction(Op: ISD::FABS, VT: MVT::bf16, Action: Legal);
244 setOperationAction(Op: ISD::FNEG, VT: MVT::bf16, Action: Legal);
245 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::bf16, Action: Legal);
246
247 // We only need to custom lower because we can't specify an action for bf16
248 // sources.
249 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
250 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
251 }
252
253 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i16, Action: Expand);
254 setTruncStoreAction(ValVT: MVT::v3i32, MemVT: MVT::v3i16, Action: Expand);
255 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i16, Action: Expand);
256 setTruncStoreAction(ValVT: MVT::v8i32, MemVT: MVT::v8i16, Action: Expand);
257 setTruncStoreAction(ValVT: MVT::v16i32, MemVT: MVT::v16i16, Action: Expand);
258 setTruncStoreAction(ValVT: MVT::v32i32, MemVT: MVT::v32i16, Action: Expand);
259 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i8, Action: Expand);
260 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i8, Action: Expand);
261 setTruncStoreAction(ValVT: MVT::v8i32, MemVT: MVT::v8i8, Action: Expand);
262 setTruncStoreAction(ValVT: MVT::v16i32, MemVT: MVT::v16i8, Action: Expand);
263 setTruncStoreAction(ValVT: MVT::v32i32, MemVT: MVT::v32i8, Action: Expand);
264 setTruncStoreAction(ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Expand);
265 setTruncStoreAction(ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Expand);
266 setTruncStoreAction(ValVT: MVT::v8i16, MemVT: MVT::v8i8, Action: Expand);
267 setTruncStoreAction(ValVT: MVT::v16i16, MemVT: MVT::v16i8, Action: Expand);
268 setTruncStoreAction(ValVT: MVT::v32i16, MemVT: MVT::v32i8, Action: Expand);
269
270 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i16, Action: Expand);
271 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i32, Action: Expand);
272 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i8, Action: Expand);
273 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i8, Action: Expand);
274 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i16, Action: Expand);
275 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i32, Action: Expand);
276 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i32, Action: Expand);
277
278 setOperationAction(Ops: ISD::GlobalAddress, VTs: {MVT::i32, MVT::i64}, Action: Custom);
279 setOperationAction(Ops: ISD::ExternalSymbol, VTs: {MVT::i32, MVT::i64}, Action: Custom);
280
281 setOperationAction(Op: ISD::SELECT, VT: MVT::i1, Action: Promote);
282 setOperationAction(Op: ISD::SELECT, VT: MVT::i64, Action: Custom);
283 setOperationAction(Op: ISD::SELECT, VT: MVT::f64, Action: Promote);
284 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::f64, DestVT: MVT::i64);
285
286 setOperationAction(Ops: ISD::FSQRT, VTs: {MVT::f32, MVT::f64}, Action: Custom);
287
288 setOperationAction(Ops: ISD::SELECT_CC,
289 VTs: {MVT::f32, MVT::i32, MVT::i64, MVT::f64, MVT::i1}, Action: Expand);
290
291 setOperationAction(Op: ISD::SETCC, VT: MVT::i1, Action: Promote);
292 setOperationAction(Ops: ISD::SETCC, VTs: {MVT::v2i1, MVT::v4i1}, Action: Expand);
293 AddPromotedToType(Opc: ISD::SETCC, OrigVT: MVT::i1, DestVT: MVT::i32);
294
295 setOperationAction(Ops: ISD::TRUNCATE,
296 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
297 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
298 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32},
299 Action: Expand);
300 setOperationAction(Ops: ISD::FP_ROUND,
301 VTs: {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
302 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v9f32,
303 MVT::v10f32, MVT::v11f32, MVT::v12f32, MVT::v16f32},
304 Action: Expand);
305
306 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG,
307 VTs: {MVT::v2i1, MVT::v4i1, MVT::v2i8, MVT::v4i8, MVT::v2i16,
308 MVT::v3i16, MVT::v4i16, MVT::Other},
309 Action: Custom);
310
311 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
312 setOperationAction(Ops: ISD::BR_CC,
313 VTs: {MVT::i1, MVT::i32, MVT::i64, MVT::f32, MVT::f64}, Action: Expand);
314
315 setOperationAction(Ops: {ISD::ABS, ISD::UADDO, ISD::USUBO}, VT: MVT::i32, Action: Legal);
316
317 setOperationAction(Ops: {ISD::UADDO_CARRY, ISD::USUBO_CARRY}, VT: MVT::i32, Action: Legal);
318
319 setOperationAction(Ops: {ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS}, VT: MVT::i64,
320 Action: Expand);
321
322#if 0
323 setOperationAction({ISD::UADDO_CARRY, ISD::USUBO_CARRY}, MVT::i64, Legal);
324#endif
325
326 // We only support LOAD/STORE and vector manipulation ops for vectors
327 // with > 4 elements.
328 for (MVT VT :
329 {MVT::v8i32, MVT::v8f32, MVT::v9i32, MVT::v9f32, MVT::v10i32,
330 MVT::v10f32, MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32,
331 MVT::v16i32, MVT::v16f32, MVT::v2i64, MVT::v2f64, MVT::v4i16,
332 MVT::v4f16, MVT::v4bf16, MVT::v3i64, MVT::v3f64, MVT::v6i32,
333 MVT::v6f32, MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64,
334 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
335 MVT::v16bf16, MVT::v16i64, MVT::v16f64, MVT::v32i32, MVT::v32f32,
336 MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
337 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
338 switch (Op) {
339 case ISD::LOAD:
340 case ISD::STORE:
341 case ISD::BUILD_VECTOR:
342 case ISD::BITCAST:
343 case ISD::UNDEF:
344 case ISD::EXTRACT_VECTOR_ELT:
345 case ISD::INSERT_VECTOR_ELT:
346 case ISD::SCALAR_TO_VECTOR:
347 case ISD::IS_FPCLASS:
348 break;
349 case ISD::EXTRACT_SUBVECTOR:
350 case ISD::INSERT_SUBVECTOR:
351 case ISD::CONCAT_VECTORS:
352 setOperationAction(Op, VT, Action: Custom);
353 break;
354 default:
355 setOperationAction(Op, VT, Action: Expand);
356 break;
357 }
358 }
359 }
360
361 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v4f32, Action: Expand);
362
363 // TODO: For dynamic 64-bit vector inserts/extracts, should emit a pseudo that
364 // is expanded to avoid having two separate loops in case the index is a VGPR.
365
366 // Most operations are naturally 32-bit vector operations. We only support
367 // load and store of i64 vectors, so promote v2i64 vector operations to v4i32.
368 for (MVT Vec64 : {MVT::v2i64, MVT::v2f64}) {
369 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
370 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v4i32);
371
372 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
373 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v4i32);
374
375 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
376 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v4i32);
377
378 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
379 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v4i32);
380 }
381
382 for (MVT Vec64 : {MVT::v3i64, MVT::v3f64}) {
383 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
384 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v6i32);
385
386 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
387 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v6i32);
388
389 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
390 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v6i32);
391
392 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
393 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v6i32);
394 }
395
396 for (MVT Vec64 : {MVT::v4i64, MVT::v4f64}) {
397 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
398 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v8i32);
399
400 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
401 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v8i32);
402
403 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
404 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v8i32);
405
406 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
407 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v8i32);
408 }
409
410 for (MVT Vec64 : {MVT::v8i64, MVT::v8f64}) {
411 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
412 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v16i32);
413
414 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
415 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v16i32);
416
417 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
418 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v16i32);
419
420 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
421 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v16i32);
422 }
423
424 for (MVT Vec64 : {MVT::v16i64, MVT::v16f64}) {
425 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
426 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v32i32);
427
428 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
429 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v32i32);
430
431 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
432 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v32i32);
433
434 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
435 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v32i32);
436 }
437
438 setOperationAction(Ops: ISD::VECTOR_SHUFFLE,
439 VTs: {MVT::v4i32, MVT::v4f32, MVT::v8i32, MVT::v8f32,
440 MVT::v16i32, MVT::v16f32, MVT::v32i32, MVT::v32f32},
441 Action: Custom);
442
443 if (Subtarget->hasPkMovB32()) {
444 // TODO: 16-bit element vectors should be legal with even aligned elements.
445 // TODO: Can be legal with wider source types than the result with
446 // subregister extracts.
447 setOperationAction(Ops: ISD::VECTOR_SHUFFLE, VTs: {MVT::v2i32, MVT::v2f32}, Action: Legal);
448 }
449
450 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VT: MVT::v2i32, Action: Legal);
451 // Prevent SELECT v2i32 from being implemented with the above bitwise ops and
452 // instead lower to cndmask in SITargetLowering::LowerSELECT().
453 setOperationAction(Op: ISD::SELECT, VT: MVT::v2i32, Action: Custom);
454 // Enable MatchRotate to produce ISD::ROTR, which is later transformed to
455 // alignbit.
456 setOperationAction(Op: ISD::ROTR, VT: MVT::v2i32, Action: Custom);
457
458 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs: {MVT::v4f16, MVT::v4i16, MVT::v4bf16},
459 Action: Custom);
460
461 // Avoid stack access for these.
462 // TODO: Generalize to more vector types.
463 setOperationAction(Ops: {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
464 VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v2i8, MVT::v4i8,
465 MVT::v8i8, MVT::v4i16, MVT::v4f16, MVT::v4bf16},
466 Action: Custom);
467
468 // Deal with vec3 vector operations when widened to vec4.
469 setOperationAction(Ops: ISD::INSERT_SUBVECTOR,
470 VTs: {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32}, Action: Custom);
471
472 // Deal with vec5/6/7 vector operations when widened to vec8.
473 setOperationAction(Ops: ISD::INSERT_SUBVECTOR,
474 VTs: {MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
475 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
476 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
477 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
478 Action: Custom);
479
480 // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling,
481 // and output demarshalling
482 setOperationAction(Ops: ISD::ATOMIC_CMP_SWAP, VTs: {MVT::i32, MVT::i64}, Action: Custom);
483
484 // We can't return success/failure, only the old value,
485 // let LLVM add the comparison
486 setOperationAction(Ops: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VTs: {MVT::i32, MVT::i64},
487 Action: Expand);
488
489 setOperationAction(Ops: ISD::ADDRSPACECAST, VTs: {MVT::i32, MVT::i64}, Action: Custom);
490
491 setOperationAction(Ops: ISD::BITREVERSE, VTs: {MVT::i32, MVT::i64}, Action: Legal);
492
493 // FIXME: This should be narrowed to i32, but that only happens if i64 is
494 // illegal.
495 // FIXME: Should lower sub-i32 bswaps to bit-ops without v_perm_b32.
496 setOperationAction(Ops: ISD::BSWAP, VTs: {MVT::i64, MVT::i32}, Action: Legal);
497
498 // On SI this is s_memtime and s_memrealtime on VI.
499 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Legal);
500
501 if (Subtarget->hasSMemRealTime() ||
502 Subtarget->getGeneration() >= AMDGPUSubtarget::GFX11)
503 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64, Action: Legal);
504 setOperationAction(Ops: {ISD::TRAP, ISD::DEBUGTRAP}, VT: MVT::Other, Action: Custom);
505
506 if (Subtarget->has16BitInsts()) {
507 setOperationAction(Ops: {ISD::FPOW, ISD::FPOWI}, VT: MVT::f16, Action: Promote);
508 setOperationAction(Ops: {ISD::FLOG, ISD::FEXP, ISD::FLOG10}, VT: MVT::f16, Action: Custom);
509 setOperationAction(Ops: ISD::IS_FPCLASS, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Legal);
510 setOperationAction(Ops: {ISD::FLOG2, ISD::FEXP2}, VT: MVT::f16, Action: Legal);
511 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f16, Action: Legal);
512 } else {
513 setOperationAction(Op: ISD::FSQRT, VT: MVT::f16, Action: Custom);
514 }
515
516 if (Subtarget->hasMadMacF32Insts())
517 setOperationAction(Op: ISD::FMAD, VT: MVT::f32, Action: Legal);
518
519 setOperationAction(Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, VT: MVT::i32, Action: Custom);
520 setOperationAction(Ops: {ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF}, VT: MVT::i32, Action: Custom);
521 setOperationAction(Op: ISD::CTLS, VT: MVT::i32, Action: Custom);
522
523 // We only really have 32-bit BFE instructions (and 16-bit on VI).
524 //
525 // On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any
526 // effort to match them now. We want this to be false for i64 cases when the
527 // extraction isn't restricted to the upper or lower half. Ideally we would
528 // have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that
529 // span the midpoint are probably relatively rare, so don't worry about them
530 // for now.
531 setHasExtractBitsInsn(true);
532
533 // Clamp modifier on add/sub
534 if (Subtarget->hasIntClamp())
535 setOperationAction(Ops: {ISD::UADDSAT, ISD::USUBSAT}, VT: MVT::i32, Action: Legal);
536
537 if (Subtarget->hasAddNoCarryInsts())
538 setOperationAction(Ops: {ISD::SADDSAT, ISD::SSUBSAT}, VTs: {MVT::i16, MVT::i32},
539 Action: Legal);
540
541 setOperationAction(
542 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
543 VTs: {MVT::f32, MVT::f64}, Action: Custom);
544
545 // These are really only legal for ieee_mode functions. We should be avoiding
546 // them for functions that don't have ieee_mode enabled, so just say they are
547 // legal.
548 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE},
549 VTs: {MVT::f32, MVT::f64}, Action: Legal);
550
551 if (Subtarget->haveRoundOpsF64())
552 setOperationAction(Ops: {ISD::FTRUNC, ISD::FCEIL, ISD::FROUNDEVEN}, VT: MVT::f64,
553 Action: Legal);
554 else
555 setOperationAction(Ops: {ISD::FCEIL, ISD::FTRUNC, ISD::FROUNDEVEN, ISD::FFLOOR},
556 VT: MVT::f64, Action: Custom);
557
558 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f64, Action: Legal);
559 setOperationAction(Ops: {ISD::FLDEXP, ISD::STRICT_FLDEXP}, VTs: {MVT::f32, MVT::f64},
560 Action: Legal);
561 setOperationAction(Ops: ISD::FFREXP, VTs: {MVT::f32, MVT::f64}, Action: Custom);
562
563 setOperationAction(Ops: {ISD::FSIN, ISD::FCOS, ISD::FDIV}, VT: MVT::f32, Action: Custom);
564 setOperationAction(Op: ISD::FDIV, VT: MVT::f64, Action: Custom);
565
566 setOperationAction(Ops: ISD::BF16_TO_FP, VTs: {MVT::i16, MVT::f32, MVT::f64}, Action: Expand);
567 setOperationAction(Ops: ISD::FP_TO_BF16, VTs: {MVT::i16, MVT::f32, MVT::f64}, Action: Expand);
568
569 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i32,
570 Action: Custom);
571 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i16,
572 Action: Custom);
573 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i1,
574 Action: Custom);
575
576 // Custom lower these because we can't specify a rule based on an illegal
577 // source bf16.
578 setOperationAction(Ops: {ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, VT: MVT::f32, Action: Custom);
579 setOperationAction(Ops: {ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, VT: MVT::f64, Action: Custom);
580
581 if (Subtarget->has16BitInsts()) {
582 setOperationAction(Ops: {ISD::Constant, ISD::SMIN, ISD::SMAX, ISD::UMIN,
583 ISD::UMAX, ISD::UADDSAT, ISD::USUBSAT},
584 VT: MVT::i16, Action: Legal);
585
586 AddPromotedToType(Opc: ISD::SIGN_EXTEND, OrigVT: MVT::i16, DestVT: MVT::i32);
587
588 setOperationAction(Ops: {ISD::ROTR, ISD::ROTL, ISD::SELECT_CC, ISD::BR_CC},
589 VT: MVT::i16, Action: Expand);
590
591 setOperationAction(Ops: {ISD::SIGN_EXTEND, ISD::SDIV, ISD::UDIV, ISD::SREM,
592 ISD::UREM, ISD::BITREVERSE, ISD::CTTZ,
593 ISD::CTTZ_ZERO_UNDEF, ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF,
594 ISD::CTPOP},
595 VT: MVT::i16, Action: Promote);
596
597 setOperationAction(Op: ISD::LOAD, VT: MVT::i16, Action: Custom);
598
599 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i16, Action: Expand);
600
601 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::i16, Action: Promote);
602 AddPromotedToType(Opc: ISD::FP16_TO_FP, OrigVT: MVT::i16, DestVT: MVT::i32);
603 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::i16, Action: Promote);
604 AddPromotedToType(Opc: ISD::FP_TO_FP16, OrigVT: MVT::i16, DestVT: MVT::i32);
605
606 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: MVT::i16, Action: Custom);
607 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i16, Action: Custom);
608 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i1, Action: Custom);
609
610 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i32, Action: Custom);
611
612 // F16 - Constant Actions.
613 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Legal);
614 setOperationAction(Op: ISD::ConstantFP, VT: MVT::bf16, Action: Legal);
615
616 // F16 - Load/Store Actions.
617 setOperationAction(Op: ISD::LOAD, VT: MVT::f16, Action: Promote);
618 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f16, DestVT: MVT::i16);
619 setOperationAction(Op: ISD::STORE, VT: MVT::f16, Action: Promote);
620 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f16, DestVT: MVT::i16);
621
622 // BF16 - Load/Store Actions.
623 setOperationAction(Op: ISD::LOAD, VT: MVT::bf16, Action: Promote);
624 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::bf16, DestVT: MVT::i16);
625 setOperationAction(Op: ISD::STORE, VT: MVT::bf16, Action: Promote);
626 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::bf16, DestVT: MVT::i16);
627
628 // F16 - VOP1 Actions.
629 setOperationAction(Ops: {ISD::FP_ROUND, ISD::STRICT_FP_ROUND, ISD::FCOS,
630 ISD::FSIN, ISD::FROUND},
631 VT: MVT::f16, Action: Custom);
632
633 // BF16 - VOP1 Actions.
634 if (Subtarget->hasBF16TransInsts())
635 setOperationAction(Ops: {ISD::FCOS, ISD::FSIN, ISD::FDIV}, VT: MVT::bf16, Action: Custom);
636
637 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT,
638 ISD::FP_TO_UINT_SAT},
639 VT: MVT::f16, Action: Promote);
640 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT,
641 ISD::FP_TO_UINT_SAT},
642 VT: MVT::bf16, Action: Promote);
643
644 // F16 - VOP2 Actions.
645 setOperationAction(Ops: {ISD::BR_CC, ISD::SELECT_CC}, VTs: {MVT::f16, MVT::bf16},
646 Action: Expand);
647 setOperationAction(Ops: {ISD::FLDEXP, ISD::STRICT_FLDEXP}, VT: MVT::f16, Action: Custom);
648 setOperationAction(Op: ISD::FFREXP, VT: MVT::f16, Action: Custom);
649 setOperationAction(Op: ISD::FDIV, VT: MVT::f16, Action: Custom);
650
651 // F16 - VOP3 Actions.
652 setOperationAction(Op: ISD::FMA, VT: MVT::f16, Action: Legal);
653 if (STI.hasMadF16())
654 setOperationAction(Op: ISD::FMAD, VT: MVT::f16, Action: Legal);
655
656 for (MVT VT :
657 {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v4i16, MVT::v4f16,
658 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16,
659 MVT::v16f16, MVT::v16bf16, MVT::v32i16, MVT::v32f16}) {
660 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
661 switch (Op) {
662 case ISD::LOAD:
663 case ISD::STORE:
664 case ISD::BUILD_VECTOR:
665 case ISD::BITCAST:
666 case ISD::UNDEF:
667 case ISD::EXTRACT_VECTOR_ELT:
668 case ISD::INSERT_VECTOR_ELT:
669 case ISD::INSERT_SUBVECTOR:
670 case ISD::SCALAR_TO_VECTOR:
671 case ISD::IS_FPCLASS:
672 break;
673 case ISD::EXTRACT_SUBVECTOR:
674 case ISD::CONCAT_VECTORS:
675 case ISD::FSIN:
676 case ISD::FCOS:
677 setOperationAction(Op, VT, Action: Custom);
678 break;
679 default:
680 setOperationAction(Op, VT, Action: Expand);
681 break;
682 }
683 }
684 }
685
686 // v_perm_b32 can handle either of these.
687 setOperationAction(Ops: ISD::BSWAP, VTs: {MVT::i16, MVT::v2i16}, Action: Legal);
688 setOperationAction(Op: ISD::BSWAP, VT: MVT::v4i16, Action: Custom);
689
690 // XXX - Do these do anything? Vector constants turn into build_vector.
691 setOperationAction(Ops: ISD::Constant, VTs: {MVT::v2i16, MVT::v2f16}, Action: Legal);
692
693 setOperationAction(Ops: ISD::UNDEF, VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
694 Action: Legal);
695
696 setOperationAction(Op: ISD::STORE, VT: MVT::v2i16, Action: Promote);
697 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2i16, DestVT: MVT::i32);
698 setOperationAction(Op: ISD::STORE, VT: MVT::v2f16, Action: Promote);
699 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f16, DestVT: MVT::i32);
700
701 setOperationAction(Op: ISD::LOAD, VT: MVT::v2i16, Action: Promote);
702 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2i16, DestVT: MVT::i32);
703 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f16, Action: Promote);
704 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f16, DestVT: MVT::i32);
705
706 setOperationAction(Op: ISD::AND, VT: MVT::v2i16, Action: Promote);
707 AddPromotedToType(Opc: ISD::AND, OrigVT: MVT::v2i16, DestVT: MVT::i32);
708 setOperationAction(Op: ISD::OR, VT: MVT::v2i16, Action: Promote);
709 AddPromotedToType(Opc: ISD::OR, OrigVT: MVT::v2i16, DestVT: MVT::i32);
710 setOperationAction(Op: ISD::XOR, VT: MVT::v2i16, Action: Promote);
711 AddPromotedToType(Opc: ISD::XOR, OrigVT: MVT::v2i16, DestVT: MVT::i32);
712
713 setOperationAction(Op: ISD::LOAD, VT: MVT::v4i16, Action: Promote);
714 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
715 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f16, Action: Promote);
716 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
717 setOperationAction(Op: ISD::LOAD, VT: MVT::v4bf16, Action: Promote);
718 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4bf16, DestVT: MVT::v2i32);
719
720 setOperationAction(Op: ISD::STORE, VT: MVT::v4i16, Action: Promote);
721 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
722 setOperationAction(Op: ISD::STORE, VT: MVT::v4f16, Action: Promote);
723 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
724 setOperationAction(Op: ISD::STORE, VT: MVT::v4bf16, Action: Promote);
725 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4bf16, DestVT: MVT::v2i32);
726
727 setOperationAction(Op: ISD::LOAD, VT: MVT::v8i16, Action: Promote);
728 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8i16, DestVT: MVT::v4i32);
729 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f16, Action: Promote);
730 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f16, DestVT: MVT::v4i32);
731 setOperationAction(Op: ISD::LOAD, VT: MVT::v8bf16, Action: Promote);
732 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8bf16, DestVT: MVT::v4i32);
733
734 setOperationAction(Op: ISD::STORE, VT: MVT::v4i16, Action: Promote);
735 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
736 setOperationAction(Op: ISD::STORE, VT: MVT::v4f16, Action: Promote);
737 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
738
739 setOperationAction(Op: ISD::STORE, VT: MVT::v8i16, Action: Promote);
740 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8i16, DestVT: MVT::v4i32);
741 setOperationAction(Op: ISD::STORE, VT: MVT::v8f16, Action: Promote);
742 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f16, DestVT: MVT::v4i32);
743 setOperationAction(Op: ISD::STORE, VT: MVT::v8bf16, Action: Promote);
744 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8bf16, DestVT: MVT::v4i32);
745
746 setOperationAction(Op: ISD::LOAD, VT: MVT::v16i16, Action: Promote);
747 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16i16, DestVT: MVT::v8i32);
748 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f16, Action: Promote);
749 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f16, DestVT: MVT::v8i32);
750 setOperationAction(Op: ISD::LOAD, VT: MVT::v16bf16, Action: Promote);
751 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16bf16, DestVT: MVT::v8i32);
752
753 setOperationAction(Op: ISD::STORE, VT: MVT::v16i16, Action: Promote);
754 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16i16, DestVT: MVT::v8i32);
755 setOperationAction(Op: ISD::STORE, VT: MVT::v16f16, Action: Promote);
756 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f16, DestVT: MVT::v8i32);
757 setOperationAction(Op: ISD::STORE, VT: MVT::v16bf16, Action: Promote);
758 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16bf16, DestVT: MVT::v8i32);
759
760 setOperationAction(Op: ISD::LOAD, VT: MVT::v32i16, Action: Promote);
761 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32i16, DestVT: MVT::v16i32);
762 setOperationAction(Op: ISD::LOAD, VT: MVT::v32f16, Action: Promote);
763 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32f16, DestVT: MVT::v16i32);
764 setOperationAction(Op: ISD::LOAD, VT: MVT::v32bf16, Action: Promote);
765 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32bf16, DestVT: MVT::v16i32);
766
767 setOperationAction(Op: ISD::STORE, VT: MVT::v32i16, Action: Promote);
768 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32i16, DestVT: MVT::v16i32);
769 setOperationAction(Op: ISD::STORE, VT: MVT::v32f16, Action: Promote);
770 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32f16, DestVT: MVT::v16i32);
771 setOperationAction(Op: ISD::STORE, VT: MVT::v32bf16, Action: Promote);
772 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32bf16, DestVT: MVT::v16i32);
773
774 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
775 VT: MVT::v2i32, Action: Expand);
776 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Expand);
777
778 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
779 VT: MVT::v4i32, Action: Expand);
780
781 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
782 VT: MVT::v8i32, Action: Expand);
783
784 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
785 Action: Subtarget->hasVOP3PInsts() ? Legal : Custom);
786
787 setOperationAction(Ops: ISD::FNEG, VTs: {MVT::v2f16, MVT::v2bf16}, Action: Legal);
788 // This isn't really legal, but this avoids the legalizer unrolling it (and
789 // allows matching fneg (fabs x) patterns)
790 setOperationAction(Ops: ISD::FABS, VTs: {MVT::v2f16, MVT::v2bf16}, Action: Legal);
791
792 // Can do this in one BFI plus a constant materialize.
793 setOperationAction(Ops: ISD::FCOPYSIGN,
794 VTs: {MVT::v2f16, MVT::v2bf16, MVT::v4f16, MVT::v4bf16,
795 MVT::v8f16, MVT::v8bf16, MVT::v16f16, MVT::v16bf16,
796 MVT::v32f16, MVT::v32bf16},
797 Action: Custom);
798
799 setOperationAction(
800 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
801 VT: MVT::f16, Action: Custom);
802 setOperationAction(Ops: {ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE}, VT: MVT::f16, Action: Legal);
803
804 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE, ISD::FMINIMUMNUM,
805 ISD::FMAXIMUMNUM},
806 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
807 Action: Custom);
808
809 setOperationAction(Ops: {ISD::FMINNUM, ISD::FMAXNUM},
810 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
811 Action: Expand);
812
813 for (MVT Vec16 :
814 {MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
815 MVT::v16bf16, MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
816 setOperationAction(
817 Ops: {ISD::BUILD_VECTOR, ISD::EXTRACT_VECTOR_ELT, ISD::SCALAR_TO_VECTOR},
818 VT: Vec16, Action: Custom);
819 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec16, Action: Expand);
820 }
821 }
822
823 if (Subtarget->hasVOP3PInsts()) {
824 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::MUL, ISD::SHL, ISD::SRL,
825 ISD::SRA, ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX,
826 ISD::UADDSAT, ISD::USUBSAT, ISD::SADDSAT, ISD::SSUBSAT},
827 VT: MVT::v2i16, Action: Legal);
828
829 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FMINNUM_IEEE,
830 ISD::FMAXNUM_IEEE, ISD::FCANONICALIZE},
831 VT: MVT::v2f16, Action: Legal);
832
833 setOperationAction(Ops: ISD::EXTRACT_VECTOR_ELT,
834 VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Action: Custom);
835
836 setOperationAction(Ops: ISD::VECTOR_SHUFFLE,
837 VTs: {MVT::v4f16, MVT::v4i16, MVT::v4bf16, MVT::v8f16,
838 MVT::v8i16, MVT::v8bf16, MVT::v16f16, MVT::v16i16,
839 MVT::v16bf16, MVT::v32f16, MVT::v32i16, MVT::v32bf16},
840 Action: Custom);
841
842 for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16})
843 // Split vector operations.
844 setOperationAction(Ops: {ISD::SHL, ISD::SRA, ISD::SRL, ISD::ADD, ISD::SUB,
845 ISD::MUL, ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN,
846 ISD::UMAX, ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT,
847 ISD::SSUBSAT},
848 VT, Action: Custom);
849
850 for (MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16})
851 // Split vector operations.
852 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FCANONICALIZE},
853 VT, Action: Custom);
854
855 setOperationAction(
856 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
857 VTs: {MVT::v2f16, MVT::v4f16}, Action: Custom);
858
859 setOperationAction(Op: ISD::FEXP, VT: MVT::v2f16, Action: Custom);
860 setOperationAction(Ops: ISD::SELECT, VTs: {MVT::v4i16, MVT::v4f16, MVT::v4bf16},
861 Action: Custom);
862
863 if (Subtarget->hasBF16PackedInsts()) {
864 for (MVT VT : {MVT::v4bf16, MVT::v8bf16, MVT::v16bf16, MVT::v32bf16})
865 // Split vector operations.
866 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FCANONICALIZE},
867 VT, Action: Custom);
868 }
869
870 if (Subtarget->hasPackedFP32Ops()) {
871 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
872 VT: MVT::v2f32, Action: Legal);
873 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA},
874 VTs: {MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32},
875 Action: Custom);
876 }
877 }
878
879 setOperationAction(Ops: {ISD::FNEG, ISD::FABS}, VT: MVT::v4f16, Action: Custom);
880
881 if (Subtarget->has16BitInsts()) {
882 setOperationAction(Op: ISD::SELECT, VT: MVT::v2i16, Action: Promote);
883 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2i16, DestVT: MVT::i32);
884 setOperationAction(Op: ISD::SELECT, VT: MVT::v2f16, Action: Promote);
885 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2f16, DestVT: MVT::i32);
886 } else {
887 // Legalization hack.
888 setOperationAction(Ops: ISD::SELECT, VTs: {MVT::v2i16, MVT::v2f16}, Action: Custom);
889
890 setOperationAction(Ops: {ISD::FNEG, ISD::FABS}, VT: MVT::v2f16, Action: Custom);
891 }
892
893 setOperationAction(Ops: ISD::SELECT,
894 VTs: {MVT::v4i16, MVT::v4f16, MVT::v4bf16, MVT::v2i8, MVT::v4i8,
895 MVT::v8i8, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
896 MVT::v16i16, MVT::v16f16, MVT::v16bf16, MVT::v32i16,
897 MVT::v32f16, MVT::v32bf16},
898 Action: Custom);
899
900 setOperationAction(Ops: {ISD::SMULO, ISD::UMULO}, VT: MVT::i64, Action: Custom);
901
902 if (Subtarget->hasVectorMulU64())
903 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Legal);
904 else if (Subtarget->hasScalarSMulU64())
905 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Custom);
906
907 if (Subtarget->hasMad64_32())
908 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT: MVT::i32, Action: Custom);
909
910 if (Subtarget->hasSafeSmemPrefetch() || Subtarget->hasVmemPrefInsts())
911 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
912
913 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
914 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM},
915 VTs: {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16}, Action: Legal);
916 } else {
917 // FIXME: For nnan fmaximum, emit the fmaximum3 instead of fmaxnum
918 if (Subtarget->hasMinimum3Maximum3F32())
919 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f32, Action: Legal);
920
921 if (Subtarget->hasMinimum3Maximum3PKF16()) {
922 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::v2f16, Action: Legal);
923
924 // If only the vector form is available, we need to widen to a vector.
925 if (!Subtarget->hasMinimum3Maximum3F16())
926 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f16, Action: Custom);
927 }
928 }
929
930 if (Subtarget->hasVOP3PInsts()) {
931 // We want to break these into v2f16 pieces, not scalarize.
932 setOperationAction(Ops: {ISD::FMINIMUM, ISD::FMAXIMUM},
933 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
934 Action: Custom);
935 }
936
937 if (Subtarget->hasIntMinMax64())
938 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT: MVT::i64,
939 Action: Legal);
940
941 setOperationAction(Ops: ISD::INTRINSIC_WO_CHAIN,
942 VTs: {MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16,
943 MVT::bf16, MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::i128,
944 MVT::i8},
945 Action: Custom);
946
947 setOperationAction(Ops: ISD::INTRINSIC_W_CHAIN,
948 VTs: {MVT::v2f16, MVT::v2i16, MVT::v2bf16, MVT::v3f16,
949 MVT::v3i16, MVT::v4f16, MVT::v4i16, MVT::v4bf16,
950 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::Other, MVT::f16,
951 MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
952 Action: Custom);
953
954 setOperationAction(Ops: ISD::INTRINSIC_VOID,
955 VTs: {MVT::Other, MVT::v2i16, MVT::v2f16, MVT::v2bf16,
956 MVT::v3i16, MVT::v3f16, MVT::v4f16, MVT::v4i16,
957 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
958 MVT::f16, MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
959 Action: Custom);
960
961 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Custom);
962 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
963 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
964 setOperationAction(Op: ISD::GET_FPENV, VT: MVT::i64, Action: Custom);
965 setOperationAction(Op: ISD::SET_FPENV, VT: MVT::i64, Action: Custom);
966
967 // TODO: Could move this to custom lowering, could benefit from combines on
968 // extract of relevant bits.
969 setOperationAction(Op: ISD::GET_FPMODE, VT: MVT::i32, Action: Legal);
970
971 setOperationAction(Op: ISD::MUL, VT: MVT::i1, Action: Promote);
972
973 if (Subtarget->hasBF16ConversionInsts()) {
974 setOperationAction(Ops: ISD::FP_ROUND, VTs: {MVT::bf16, MVT::v2bf16}, Action: Custom);
975 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2bf16, Action: Legal);
976 }
977
978 if (Subtarget->hasBF16PackedInsts()) {
979 setOperationAction(
980 Ops: {ISD::FADD, ISD::FMUL, ISD::FMINNUM, ISD::FMAXNUM, ISD::FMA},
981 VT: MVT::v2bf16, Action: Legal);
982 }
983
984 if (Subtarget->hasBF16TransInsts()) {
985 setOperationAction(Ops: {ISD::FEXP2, ISD::FLOG2, ISD::FSQRT}, VT: MVT::bf16, Action: Legal);
986 }
987
988 if (Subtarget->hasCvtPkF16F32Inst()) {
989 setOperationAction(Ops: ISD::FP_ROUND,
990 VTs: {MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16},
991 Action: Custom);
992 }
993
994 setTargetDAGCombine({ISD::ADD,
995 ISD::PTRADD,
996 ISD::UADDO_CARRY,
997 ISD::SUB,
998 ISD::USUBO_CARRY,
999 ISD::MUL,
1000 ISD::FADD,
1001 ISD::FSUB,
1002 ISD::FDIV,
1003 ISD::FMUL,
1004 ISD::FMINNUM,
1005 ISD::FMAXNUM,
1006 ISD::FMINNUM_IEEE,
1007 ISD::FMAXNUM_IEEE,
1008 ISD::FMINIMUM,
1009 ISD::FMAXIMUM,
1010 ISD::FMINIMUMNUM,
1011 ISD::FMAXIMUMNUM,
1012 ISD::FMA,
1013 ISD::SMIN,
1014 ISD::SMAX,
1015 ISD::UMIN,
1016 ISD::UMAX,
1017 ISD::SETCC,
1018 ISD::SELECT,
1019 ISD::SMIN,
1020 ISD::SMAX,
1021 ISD::UMIN,
1022 ISD::UMAX,
1023 ISD::AND,
1024 ISD::OR,
1025 ISD::XOR,
1026 ISD::SHL,
1027 ISD::SRL,
1028 ISD::SRA,
1029 ISD::FSHR,
1030 ISD::SINT_TO_FP,
1031 ISD::UINT_TO_FP,
1032 ISD::FCANONICALIZE,
1033 ISD::SCALAR_TO_VECTOR,
1034 ISD::ZERO_EXTEND,
1035 ISD::SIGN_EXTEND_INREG,
1036 ISD::ANY_EXTEND,
1037 ISD::EXTRACT_VECTOR_ELT,
1038 ISD::INSERT_VECTOR_ELT,
1039 ISD::FCOPYSIGN});
1040
1041 if (Subtarget->has16BitInsts() && !Subtarget->hasMed3_16())
1042 setTargetDAGCombine(ISD::FP_ROUND);
1043
1044 // All memory operations. Some folding on the pointer operand is done to help
1045 // matching the constant offsets in the addressing modes.
1046 setTargetDAGCombine({ISD::LOAD,
1047 ISD::STORE,
1048 ISD::ATOMIC_LOAD,
1049 ISD::ATOMIC_STORE,
1050 ISD::ATOMIC_CMP_SWAP,
1051 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
1052 ISD::ATOMIC_SWAP,
1053 ISD::ATOMIC_LOAD_ADD,
1054 ISD::ATOMIC_LOAD_SUB,
1055 ISD::ATOMIC_LOAD_AND,
1056 ISD::ATOMIC_LOAD_OR,
1057 ISD::ATOMIC_LOAD_XOR,
1058 ISD::ATOMIC_LOAD_NAND,
1059 ISD::ATOMIC_LOAD_MIN,
1060 ISD::ATOMIC_LOAD_MAX,
1061 ISD::ATOMIC_LOAD_UMIN,
1062 ISD::ATOMIC_LOAD_UMAX,
1063 ISD::ATOMIC_LOAD_FADD,
1064 ISD::ATOMIC_LOAD_FMIN,
1065 ISD::ATOMIC_LOAD_FMAX,
1066 ISD::ATOMIC_LOAD_UINC_WRAP,
1067 ISD::ATOMIC_LOAD_UDEC_WRAP,
1068 ISD::ATOMIC_LOAD_USUB_COND,
1069 ISD::ATOMIC_LOAD_USUB_SAT,
1070 ISD::INTRINSIC_VOID,
1071 ISD::INTRINSIC_W_CHAIN});
1072
1073 // FIXME: In other contexts we pretend this is a per-function property.
1074 setStackPointerRegisterToSaveRestore(AMDGPU::SGPR32);
1075
1076 setSchedulingPreference(Sched::RegPressure);
1077}
1078
1079const GCNSubtarget *SITargetLowering::getSubtarget() const { return Subtarget; }
1080
1081ArrayRef<MCPhysReg> SITargetLowering::getRoundingControlRegisters() const {
1082 static const MCPhysReg RCRegs[] = {AMDGPU::MODE};
1083 return RCRegs;
1084}
1085
1086//===----------------------------------------------------------------------===//
1087// TargetLowering queries
1088//===----------------------------------------------------------------------===//
1089
1090// v_mad_mix* support a conversion from f16 to f32.
1091//
1092// There is only one special case when denormals are enabled we don't currently,
1093// where this is OK to use.
1094bool SITargetLowering::isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
1095 EVT DestVT, EVT SrcVT) const {
1096 return DestVT.getScalarType() == MVT::f32 &&
1097 ((((Opcode == ISD::FMAD && Subtarget->hasMadMixInsts()) ||
1098 (Opcode == ISD::FMA && Subtarget->hasFmaMixInsts())) &&
1099 SrcVT.getScalarType() == MVT::f16) ||
1100 (Opcode == ISD::FMA && Subtarget->hasFmaMixBF16Insts() &&
1101 SrcVT.getScalarType() == MVT::bf16)) &&
1102 // TODO: This probably only requires no input flushing?
1103 denormalModeIsFlushAllF32(MF: DAG.getMachineFunction());
1104}
1105
1106bool SITargetLowering::isFPExtFoldable(const MachineInstr &MI, unsigned Opcode,
1107 LLT DestTy, LLT SrcTy) const {
1108 return ((Opcode == TargetOpcode::G_FMAD && Subtarget->hasMadMixInsts()) ||
1109 (Opcode == TargetOpcode::G_FMA && Subtarget->hasFmaMixInsts())) &&
1110 DestTy.getScalarSizeInBits() == 32 &&
1111 SrcTy.getScalarSizeInBits() == 16 &&
1112 // TODO: This probably only requires no input flushing?
1113 denormalModeIsFlushAllF32(MF: *MI.getMF());
1114}
1115
1116bool SITargetLowering::isShuffleMaskLegal(ArrayRef<int>, EVT) const {
1117 // SI has some legal vector types, but no legal vector operations. Say no
1118 // shuffles are legal in order to prefer scalarizing some vector operations.
1119 return false;
1120}
1121
1122MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
1123 CallingConv::ID CC,
1124 EVT VT) const {
1125 if (CC == CallingConv::AMDGPU_KERNEL)
1126 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1127
1128 if (VT.isVector()) {
1129 EVT ScalarVT = VT.getScalarType();
1130 unsigned Size = ScalarVT.getSizeInBits();
1131 if (Size == 16) {
1132 return Subtarget->has16BitInsts()
1133 ? MVT::getVectorVT(VT: ScalarVT.getSimpleVT(), NumElements: 2)
1134 : MVT::i32;
1135 }
1136
1137 if (Size < 16)
1138 return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32;
1139 return Size == 32 ? ScalarVT.getSimpleVT() : MVT::i32;
1140 }
1141
1142 if (!Subtarget->has16BitInsts() && VT.getSizeInBits() == 16)
1143 return MVT::i32;
1144
1145 if (VT.getSizeInBits() > 32)
1146 return MVT::i32;
1147
1148 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1149}
1150
1151unsigned SITargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
1152 CallingConv::ID CC,
1153 EVT VT) const {
1154 if (CC == CallingConv::AMDGPU_KERNEL)
1155 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1156
1157 if (VT.isVector()) {
1158 unsigned NumElts = VT.getVectorNumElements();
1159 EVT ScalarVT = VT.getScalarType();
1160 unsigned Size = ScalarVT.getSizeInBits();
1161
1162 // FIXME: Should probably promote 8-bit vectors to i16.
1163 if (Size == 16)
1164 return (NumElts + 1) / 2;
1165
1166 if (Size <= 32)
1167 return NumElts;
1168
1169 if (Size > 32)
1170 return NumElts * ((Size + 31) / 32);
1171 } else if (VT.getSizeInBits() > 32)
1172 return (VT.getSizeInBits() + 31) / 32;
1173
1174 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1175}
1176
1177unsigned SITargetLowering::getVectorTypeBreakdownForCallingConv(
1178 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
1179 unsigned &NumIntermediates, MVT &RegisterVT) const {
1180 if (CC != CallingConv::AMDGPU_KERNEL && VT.isVector()) {
1181 unsigned NumElts = VT.getVectorNumElements();
1182 EVT ScalarVT = VT.getScalarType();
1183 unsigned Size = ScalarVT.getSizeInBits();
1184 // FIXME: We should fix the ABI to be the same on targets without 16-bit
1185 // support, but unless we can properly handle 3-vectors, it will be still be
1186 // inconsistent.
1187 if (Size == 16) {
1188 MVT SimpleIntermediateVT =
1189 MVT::getVectorVT(VT: ScalarVT.getSimpleVT(), EC: ElementCount::getFixed(MinVal: 2));
1190 IntermediateVT = SimpleIntermediateVT;
1191 RegisterVT = Subtarget->has16BitInsts() ? SimpleIntermediateVT : MVT::i32;
1192 NumIntermediates = (NumElts + 1) / 2;
1193 return (NumElts + 1) / 2;
1194 }
1195
1196 if (Size == 32) {
1197 RegisterVT = ScalarVT.getSimpleVT();
1198 IntermediateVT = RegisterVT;
1199 NumIntermediates = NumElts;
1200 return NumIntermediates;
1201 }
1202
1203 if (Size < 16 && Subtarget->has16BitInsts()) {
1204 // FIXME: Should probably form v2i16 pieces
1205 RegisterVT = MVT::i16;
1206 IntermediateVT = ScalarVT;
1207 NumIntermediates = NumElts;
1208 return NumIntermediates;
1209 }
1210
1211 if (Size != 16 && Size <= 32) {
1212 RegisterVT = MVT::i32;
1213 IntermediateVT = ScalarVT;
1214 NumIntermediates = NumElts;
1215 return NumIntermediates;
1216 }
1217
1218 if (Size > 32) {
1219 RegisterVT = MVT::i32;
1220 IntermediateVT = RegisterVT;
1221 NumIntermediates = NumElts * ((Size + 31) / 32);
1222 return NumIntermediates;
1223 }
1224 }
1225
1226 return TargetLowering::getVectorTypeBreakdownForCallingConv(
1227 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
1228}
1229
1230static EVT memVTFromLoadIntrData(const SITargetLowering &TLI,
1231 const DataLayout &DL, Type *Ty,
1232 unsigned MaxNumLanes) {
1233 assert(MaxNumLanes != 0);
1234
1235 LLVMContext &Ctx = Ty->getContext();
1236 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
1237 unsigned NumElts = std::min(a: MaxNumLanes, b: VT->getNumElements());
1238 return EVT::getVectorVT(Context&: Ctx, VT: TLI.getValueType(DL, Ty: VT->getElementType()),
1239 NumElements: NumElts);
1240 }
1241
1242 return TLI.getValueType(DL, Ty);
1243}
1244
1245// Peek through TFE struct returns to only use the data size.
1246static EVT memVTFromLoadIntrReturn(const SITargetLowering &TLI,
1247 const DataLayout &DL, Type *Ty,
1248 unsigned MaxNumLanes) {
1249 auto *ST = dyn_cast<StructType>(Val: Ty);
1250 if (!ST)
1251 return memVTFromLoadIntrData(TLI, DL, Ty, MaxNumLanes);
1252
1253 // TFE intrinsics return an aggregate type.
1254 assert(ST->getNumContainedTypes() == 2 &&
1255 ST->getContainedType(1)->isIntegerTy(32));
1256 return memVTFromLoadIntrData(TLI, DL, Ty: ST->getContainedType(i: 0), MaxNumLanes);
1257}
1258
1259/// Map address space 7 to MVT::amdgpuBufferFatPointer because that's its
1260/// in-memory representation. This return value is a custom type because there
1261/// is no MVT::i160 and adding one breaks integer promotion logic. While this
1262/// could cause issues during codegen, these address space 7 pointers will be
1263/// rewritten away by then. Therefore, we can return MVT::amdgpuBufferFatPointer
1264/// in order to allow pre-codegen passes that query TargetTransformInfo, often
1265/// for cost modeling, to work. (This also sets us up decently for doing the
1266/// buffer lowering in GlobalISel if SelectionDAG ever goes away.)
1267MVT SITargetLowering::getPointerTy(const DataLayout &DL, unsigned AS) const {
1268 if (AMDGPUAS::BUFFER_FAT_POINTER == AS && DL.getPointerSizeInBits(AS) == 160)
1269 return MVT::amdgpuBufferFatPointer;
1270 if (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
1271 DL.getPointerSizeInBits(AS) == 192)
1272 return MVT::amdgpuBufferStridedPointer;
1273 return AMDGPUTargetLowering::getPointerTy(DL, AS);
1274}
1275/// Similarly, the in-memory representation of a p7 is {p8, i32}, aka
1276/// v8i32 when padding is added.
1277/// The in-memory representation of a p9 is {p8, i32, i32}, which is
1278/// also v8i32 with padding.
1279MVT SITargetLowering::getPointerMemTy(const DataLayout &DL, unsigned AS) const {
1280 if ((AMDGPUAS::BUFFER_FAT_POINTER == AS &&
1281 DL.getPointerSizeInBits(AS) == 160) ||
1282 (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
1283 DL.getPointerSizeInBits(AS) == 192))
1284 return MVT::v8i32;
1285 return AMDGPUTargetLowering::getPointerMemTy(DL, AS);
1286}
1287
1288static unsigned getIntrMemWidth(unsigned IntrID) {
1289 switch (IntrID) {
1290 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1291 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1292 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1293 return 8;
1294 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1295 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1296 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1297 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
1298 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
1299 case Intrinsic::amdgcn_flat_load_monitor_b32:
1300 case Intrinsic::amdgcn_global_load_monitor_b32:
1301 return 32;
1302 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1303 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1304 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1305 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
1306 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
1307 case Intrinsic::amdgcn_flat_load_monitor_b64:
1308 case Intrinsic::amdgcn_global_load_monitor_b64:
1309 return 64;
1310 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1311 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
1312 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
1313 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
1314 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
1315 case Intrinsic::amdgcn_flat_load_monitor_b128:
1316 case Intrinsic::amdgcn_global_load_monitor_b128:
1317 return 128;
1318 default:
1319 llvm_unreachable("Unknown width");
1320 }
1321}
1322
1323static AtomicOrdering parseAtomicOrderingCABIArg(const CallBase &CI,
1324 unsigned ArgIdx) {
1325 Value *OrderingArg = CI.getArgOperand(i: ArgIdx);
1326 unsigned Ord = cast<ConstantInt>(Val: OrderingArg)->getZExtValue();
1327 switch (AtomicOrderingCABI(Ord)) {
1328 case AtomicOrderingCABI::acquire:
1329 return AtomicOrdering::Acquire;
1330 break;
1331 case AtomicOrderingCABI::release:
1332 return AtomicOrdering::Release;
1333 break;
1334 case AtomicOrderingCABI::seq_cst:
1335 return AtomicOrdering::SequentiallyConsistent;
1336 break;
1337 default:
1338 return AtomicOrdering::Monotonic;
1339 }
1340}
1341
1342static unsigned parseSyncscopeMDArg(const CallBase &CI, unsigned ArgIdx) {
1343 MDNode *ScopeMD = cast<MDNode>(
1344 Val: cast<MetadataAsValue>(Val: CI.getArgOperand(i: ArgIdx))->getMetadata());
1345 StringRef Scope = cast<MDString>(Val: ScopeMD->getOperand(I: 0))->getString();
1346 return CI.getContext().getOrInsertSyncScopeID(SSN: Scope);
1347}
1348
1349void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
1350 const CallBase &CI,
1351 MachineFunction &MF,
1352 unsigned IntrID) const {
1353 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
1354 if (CI.hasMetadata(KindID: LLVMContext::MD_invariant_load))
1355 Flags |= MachineMemOperand::MOInvariant;
1356 if (CI.hasMetadata(KindID: LLVMContext::MD_nontemporal))
1357 Flags |= MachineMemOperand::MONonTemporal;
1358 Flags |= getTargetMMOFlags(I: CI);
1359
1360 if (const AMDGPU::RsrcIntrinsic *RsrcIntr =
1361 AMDGPU::lookupRsrcIntrinsic(Intr: IntrID)) {
1362 AttributeSet Attr =
1363 Intrinsic::getFnAttributes(C&: CI.getContext(), id: (Intrinsic::ID)IntrID);
1364 MemoryEffects ME = Attr.getMemoryEffects();
1365 if (ME.doesNotAccessMemory())
1366 return;
1367
1368 bool IsSPrefetch = IntrID == Intrinsic::amdgcn_s_buffer_prefetch_data;
1369 if (!IsSPrefetch) {
1370 auto *Aux = cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1));
1371 if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE)
1372 Flags |= MachineMemOperand::MOVolatile;
1373 }
1374 Flags |= MachineMemOperand::MODereferenceable;
1375
1376 IntrinsicInfo Info;
1377 // TODO: Should images get their own address space?
1378 Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
1379
1380 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = nullptr;
1381 if (RsrcIntr->IsImage) {
1382 const AMDGPU::ImageDimIntrinsicInfo *Intr =
1383 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrID);
1384 BaseOpcode = AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Intr->BaseOpcode);
1385 Info.align.reset();
1386 }
1387
1388 Value *RsrcArg = CI.getArgOperand(i: RsrcIntr->RsrcArg);
1389 if (auto *RsrcPtrTy = dyn_cast<PointerType>(Val: RsrcArg->getType())) {
1390 if (RsrcPtrTy->getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE)
1391 // We conservatively set the memory operand of a buffer intrinsic to the
1392 // base resource pointer, so that we can access alias information about
1393 // those pointers. Cases like "this points at the same value
1394 // but with a different offset" are handled in
1395 // areMemAccessesTriviallyDisjoint.
1396 Info.ptrVal = RsrcArg;
1397 }
1398
1399 if (ME.onlyReadsMemory()) {
1400 if (RsrcIntr->IsImage) {
1401 unsigned MaxNumLanes = 4;
1402
1403 if (!BaseOpcode->Gather4) {
1404 // If this isn't a gather, we may have excess loaded elements in the
1405 // IR type. Check the dmask for the real number of elements loaded.
1406 unsigned DMask =
1407 cast<ConstantInt>(Val: CI.getArgOperand(i: 0))->getZExtValue();
1408 MaxNumLanes = DMask == 0 ? 1 : llvm::popcount(Value: DMask);
1409 }
1410
1411 Info.memVT = memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(),
1412 Ty: CI.getType(), MaxNumLanes);
1413 } else {
1414 Info.memVT =
1415 memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(), Ty: CI.getType(),
1416 MaxNumLanes: std::numeric_limits<unsigned>::max());
1417 }
1418
1419 // FIXME: What does alignment mean for an image?
1420 Info.opc = ISD::INTRINSIC_W_CHAIN;
1421 Info.flags = Flags | MachineMemOperand::MOLoad;
1422 } else if (ME.onlyWritesMemory()) {
1423 Info.opc = ISD::INTRINSIC_VOID;
1424
1425 Type *DataTy = CI.getArgOperand(i: 0)->getType();
1426 if (RsrcIntr->IsImage) {
1427 unsigned DMask = cast<ConstantInt>(Val: CI.getArgOperand(i: 1))->getZExtValue();
1428 unsigned DMaskLanes = DMask == 0 ? 1 : llvm::popcount(Value: DMask);
1429 Info.memVT = memVTFromLoadIntrData(TLI: *this, DL: MF.getDataLayout(), Ty: DataTy,
1430 MaxNumLanes: DMaskLanes);
1431 } else
1432 Info.memVT = getValueType(DL: MF.getDataLayout(), Ty: DataTy);
1433
1434 Info.flags = Flags | MachineMemOperand::MOStore;
1435 } else {
1436 // Atomic, NoReturn Sampler or prefetch
1437 Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID
1438 : ISD::INTRINSIC_W_CHAIN;
1439
1440 switch (IntrID) {
1441 default:
1442 Info.flags = Flags | MachineMemOperand::MOLoad;
1443 if (!IsSPrefetch)
1444 Info.flags |= MachineMemOperand::MOStore;
1445
1446 if ((RsrcIntr->IsImage && BaseOpcode->NoReturn) || IsSPrefetch) {
1447 // Fake memory access type for no return sampler intrinsics
1448 Info.memVT = MVT::i32;
1449 } else {
1450 // XXX - Should this be volatile without known ordering?
1451 Info.flags |= MachineMemOperand::MOVolatile;
1452 Info.memVT = MVT::getVT(Ty: CI.getArgOperand(i: 0)->getType());
1453 }
1454 break;
1455 case Intrinsic::amdgcn_raw_buffer_load_lds:
1456 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
1457 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
1458 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
1459 case Intrinsic::amdgcn_struct_buffer_load_lds:
1460 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
1461 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
1462 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
1463 unsigned Width = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getZExtValue();
1464
1465 // Entry 0: Load from buffer.
1466 // Don't set an offset, since the pointer value always represents the
1467 // base of the buffer.
1468 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: Width * 8);
1469 Info.flags = Flags | MachineMemOperand::MOLoad;
1470 Infos.push_back(Elt: Info);
1471
1472 // Entry 1: Store to LDS.
1473 // Instruction offset is applied, and an additional per-lane offset
1474 // which we simulate using a larger memory type.
1475 Info.memVT = EVT::getIntegerVT(
1476 Context&: CI.getContext(), BitWidth: Width * 8 * Subtarget->getWavefrontSize());
1477 Info.ptrVal = CI.getArgOperand(i: 1); // LDS destination pointer
1478 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 2))
1479 ->getZExtValue();
1480 Info.fallbackAddressSpace = AMDGPUAS::LOCAL_ADDRESS;
1481 Info.flags = Flags | MachineMemOperand::MOStore;
1482 Infos.push_back(Elt: Info);
1483 return;
1484 }
1485 case Intrinsic::amdgcn_raw_atomic_buffer_load:
1486 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
1487 case Intrinsic::amdgcn_struct_atomic_buffer_load:
1488 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
1489 Info.memVT =
1490 memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(), Ty: CI.getType(),
1491 MaxNumLanes: std::numeric_limits<unsigned>::max());
1492 Info.flags = Flags | MachineMemOperand::MOLoad;
1493 Infos.push_back(Elt: Info);
1494 return;
1495 }
1496 }
1497 }
1498 Infos.push_back(Elt: Info);
1499 return;
1500 }
1501
1502 IntrinsicInfo Info;
1503 switch (IntrID) {
1504 case Intrinsic::amdgcn_ds_ordered_add:
1505 case Intrinsic::amdgcn_ds_ordered_swap: {
1506 Info.opc = ISD::INTRINSIC_W_CHAIN;
1507 Info.memVT = MVT::getVT(Ty: CI.getType());
1508 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1509 Info.align.reset();
1510 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1511
1512 const ConstantInt *Vol = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 4));
1513 if (!Vol->isZero())
1514 Info.flags |= MachineMemOperand::MOVolatile;
1515
1516 Infos.push_back(Elt: Info);
1517 return;
1518 }
1519 case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
1520 case Intrinsic::amdgcn_ds_sub_gs_reg_rtn: {
1521 Info.opc = ISD::INTRINSIC_W_CHAIN;
1522 Info.memVT = MVT::getVT(Ty: CI.getOperand(i_nocapture: 0)->getType());
1523 Info.ptrVal = nullptr;
1524 Info.fallbackAddressSpace = AMDGPUAS::STREAMOUT_REGISTER;
1525 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1526 Infos.push_back(Elt: Info);
1527 return;
1528 }
1529 case Intrinsic::amdgcn_ds_append:
1530 case Intrinsic::amdgcn_ds_consume: {
1531 Info.opc = ISD::INTRINSIC_W_CHAIN;
1532 Info.memVT = MVT::getVT(Ty: CI.getType());
1533 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1534 Info.align.reset();
1535 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1536
1537 const ConstantInt *Vol = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1));
1538 if (!Vol->isZero())
1539 Info.flags |= MachineMemOperand::MOVolatile;
1540
1541 Infos.push_back(Elt: Info);
1542 return;
1543 }
1544 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
1545 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64: {
1546 Info.opc = (IntrID == Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64)
1547 ? ISD::INTRINSIC_W_CHAIN
1548 : ISD::INTRINSIC_VOID;
1549 Info.memVT = MVT::getVT(Ty: CI.getType());
1550 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1551 Info.memVT = MVT::i64;
1552 Info.size = 8;
1553 Info.align.reset();
1554 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1555 Infos.push_back(Elt: Info);
1556 return;
1557 }
1558 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
1559 case Intrinsic::amdgcn_image_bvh_intersect_ray:
1560 case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
1561 Info.opc = ISD::INTRINSIC_W_CHAIN;
1562 Info.memVT =
1563 MVT::getVT(Ty: IntrID == Intrinsic::amdgcn_image_bvh_intersect_ray
1564 ? CI.getType()
1565 : cast<StructType>(Val: CI.getType())
1566 ->getElementType(N: 0)); // XXX: what is correct VT?
1567
1568 Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
1569 Info.align.reset();
1570 Info.flags = Flags | MachineMemOperand::MOLoad |
1571 MachineMemOperand::MODereferenceable;
1572 Infos.push_back(Elt: Info);
1573 return;
1574 }
1575 case Intrinsic::amdgcn_global_atomic_fmin_num:
1576 case Intrinsic::amdgcn_global_atomic_fmax_num:
1577 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1578 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1579 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
1580 Info.opc = ISD::INTRINSIC_W_CHAIN;
1581 Info.memVT = MVT::getVT(Ty: CI.getType());
1582 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1583 Info.align.reset();
1584 Info.flags =
1585 Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
1586 MachineMemOperand::MODereferenceable | MachineMemOperand::MOVolatile;
1587 Infos.push_back(Elt: Info);
1588 return;
1589 }
1590 case Intrinsic::amdgcn_cluster_load_b32:
1591 case Intrinsic::amdgcn_cluster_load_b64:
1592 case Intrinsic::amdgcn_cluster_load_b128:
1593 case Intrinsic::amdgcn_ds_load_tr6_b96:
1594 case Intrinsic::amdgcn_ds_load_tr4_b64:
1595 case Intrinsic::amdgcn_ds_load_tr8_b64:
1596 case Intrinsic::amdgcn_ds_load_tr16_b128:
1597 case Intrinsic::amdgcn_global_load_tr6_b96:
1598 case Intrinsic::amdgcn_global_load_tr4_b64:
1599 case Intrinsic::amdgcn_global_load_tr_b64:
1600 case Intrinsic::amdgcn_global_load_tr_b128:
1601 case Intrinsic::amdgcn_ds_read_tr4_b64:
1602 case Intrinsic::amdgcn_ds_read_tr6_b96:
1603 case Intrinsic::amdgcn_ds_read_tr8_b64:
1604 case Intrinsic::amdgcn_ds_read_tr16_b64: {
1605 Info.opc = ISD::INTRINSIC_W_CHAIN;
1606 Info.memVT = MVT::getVT(Ty: CI.getType());
1607 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1608 Info.align.reset();
1609 Info.flags = Flags | MachineMemOperand::MOLoad;
1610 Infos.push_back(Elt: Info);
1611 return;
1612 }
1613 case Intrinsic::amdgcn_flat_load_monitor_b32:
1614 case Intrinsic::amdgcn_flat_load_monitor_b64:
1615 case Intrinsic::amdgcn_flat_load_monitor_b128:
1616 case Intrinsic::amdgcn_global_load_monitor_b32:
1617 case Intrinsic::amdgcn_global_load_monitor_b64:
1618 case Intrinsic::amdgcn_global_load_monitor_b128: {
1619 Info.opc = ISD::INTRINSIC_W_CHAIN;
1620 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1621 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1622 Info.align.reset();
1623 Info.flags = MachineMemOperand::MOLoad;
1624 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 1);
1625 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 2);
1626 Infos.push_back(Elt: Info);
1627 return;
1628 }
1629 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
1630 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
1631 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
1632 Info.opc = ISD::INTRINSIC_W_CHAIN;
1633 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1634 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1635 Info.align.reset();
1636 Info.flags = (MachineMemOperand::MOLoad | MOCooperative);
1637 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 1);
1638 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 2);
1639 Infos.push_back(Elt: Info);
1640 return;
1641 }
1642 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
1643 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
1644 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
1645 Info.opc = ISD::INTRINSIC_VOID;
1646 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1647 Info.ptrVal = CI.getArgOperand(i: 0);
1648 Info.align.reset();
1649 Info.flags = (MachineMemOperand::MOStore | MOCooperative);
1650 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 2);
1651 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 3);
1652 Infos.push_back(Elt: Info);
1653 return;
1654 }
1655 case Intrinsic::amdgcn_ds_gws_init:
1656 case Intrinsic::amdgcn_ds_gws_barrier:
1657 case Intrinsic::amdgcn_ds_gws_sema_v:
1658 case Intrinsic::amdgcn_ds_gws_sema_br:
1659 case Intrinsic::amdgcn_ds_gws_sema_p:
1660 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
1661 Info.opc = ISD::INTRINSIC_VOID;
1662
1663 const GCNTargetMachine &TM =
1664 static_cast<const GCNTargetMachine &>(getTargetMachine());
1665
1666 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1667 Info.ptrVal = MFI->getGWSPSV(TM);
1668
1669 // This is an abstract access, but we need to specify a type and size.
1670 Info.memVT = MVT::i32;
1671 Info.size = 4;
1672 Info.align = Align(4);
1673
1674 if (IntrID == Intrinsic::amdgcn_ds_gws_barrier)
1675 Info.flags = Flags | MachineMemOperand::MOLoad;
1676 else
1677 Info.flags = Flags | MachineMemOperand::MOStore;
1678 Infos.push_back(Elt: Info);
1679 return;
1680 }
1681 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1682 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1683 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1684 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1685 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1686 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1687 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1688 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
1689 // Entry 0: Load from source (global/flat).
1690 Info.opc = ISD::INTRINSIC_VOID;
1691 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1692 Info.ptrVal = CI.getArgOperand(i: 0); // Global pointer
1693 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getSExtValue();
1694 Info.flags = Flags | MachineMemOperand::MOLoad;
1695 Infos.push_back(Elt: Info);
1696
1697 // Entry 1: Store to LDS (same offset).
1698 Info.flags = Flags | MachineMemOperand::MOStore;
1699 Info.ptrVal = CI.getArgOperand(i: 1); // LDS pointer
1700 Infos.push_back(Elt: Info);
1701 return;
1702 }
1703 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1704 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1705 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1706 case Intrinsic::amdgcn_global_store_async_from_lds_b128: {
1707 // Entry 0: Load from LDS.
1708 Info.opc = ISD::INTRINSIC_VOID;
1709 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1710 Info.ptrVal = CI.getArgOperand(i: 1); // LDS pointer
1711 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getSExtValue();
1712 Info.flags = Flags | MachineMemOperand::MOLoad;
1713 Infos.push_back(Elt: Info);
1714
1715 // Entry 1: Store to global (same offset).
1716 Info.flags = Flags | MachineMemOperand::MOStore;
1717 Info.ptrVal = CI.getArgOperand(i: 0); // Global pointer
1718 Infos.push_back(Elt: Info);
1719 return;
1720 }
1721 case Intrinsic::amdgcn_load_to_lds:
1722 case Intrinsic::amdgcn_load_async_to_lds:
1723 case Intrinsic::amdgcn_global_load_lds:
1724 case Intrinsic::amdgcn_global_load_async_lds: {
1725 unsigned Width = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getZExtValue();
1726 auto *Aux = cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1));
1727 bool IsVolatile = Aux->getZExtValue() & AMDGPU::CPol::VOLATILE;
1728 if (IsVolatile)
1729 Flags |= MachineMemOperand::MOVolatile;
1730
1731 // Entry 0: Load from source (global/flat).
1732 Info.opc = ISD::INTRINSIC_VOID;
1733 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: Width * 8);
1734 Info.ptrVal = CI.getArgOperand(i: 0); // Source pointer
1735 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 3))->getSExtValue();
1736 Info.flags = Flags | MachineMemOperand::MOLoad;
1737 Infos.push_back(Elt: Info);
1738
1739 // Entry 1: Store to LDS.
1740 // Same offset from the instruction, but an additional per-lane offset is
1741 // added. Represent that using a wider memory type.
1742 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(),
1743 BitWidth: Width * 8 * Subtarget->getWavefrontSize());
1744 Info.ptrVal = CI.getArgOperand(i: 1); // LDS destination pointer
1745 Info.flags = Flags | MachineMemOperand::MOStore;
1746 Infos.push_back(Elt: Info);
1747 return;
1748 }
1749 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
1750 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
1751 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
1752 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn: {
1753 Info.opc = ISD::INTRINSIC_W_CHAIN;
1754
1755 const GCNTargetMachine &TM =
1756 static_cast<const GCNTargetMachine &>(getTargetMachine());
1757
1758 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1759 Info.ptrVal = MFI->getGWSPSV(TM);
1760
1761 // This is an abstract access, but we need to specify a type and size.
1762 Info.memVT = MVT::i32;
1763 Info.size = 4;
1764 Info.align = Align(4);
1765
1766 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1767 Infos.push_back(Elt: Info);
1768 return;
1769 }
1770 case Intrinsic::amdgcn_s_prefetch_data:
1771 case Intrinsic::amdgcn_flat_prefetch:
1772 case Intrinsic::amdgcn_global_prefetch: {
1773 Info.opc = ISD::INTRINSIC_VOID;
1774 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: 8);
1775 Info.ptrVal = CI.getArgOperand(i: 0);
1776 Info.flags = Flags | MachineMemOperand::MOLoad;
1777 Infos.push_back(Elt: Info);
1778 return;
1779 }
1780 default:
1781 return;
1782 }
1783}
1784
1785void SITargetLowering::CollectTargetIntrinsicOperands(
1786 const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
1787 switch (cast<IntrinsicInst>(Val: I).getIntrinsicID()) {
1788 case Intrinsic::amdgcn_addrspacecast_nonnull: {
1789 // The DAG's ValueType loses the addrspaces.
1790 // Add them as 2 extra Constant operands "from" and "to".
1791 unsigned SrcAS = I.getOperand(i_nocapture: 0)->getType()->getPointerAddressSpace();
1792 unsigned DstAS = I.getType()->getPointerAddressSpace();
1793 Ops.push_back(Elt: DAG.getTargetConstant(Val: SrcAS, DL: SDLoc(), VT: MVT::i32));
1794 Ops.push_back(Elt: DAG.getTargetConstant(Val: DstAS, DL: SDLoc(), VT: MVT::i32));
1795 break;
1796 }
1797 default:
1798 break;
1799 }
1800}
1801
1802bool SITargetLowering::getAddrModeArguments(const IntrinsicInst *II,
1803 SmallVectorImpl<Value *> &Ops,
1804 Type *&AccessTy) const {
1805 Value *Ptr = nullptr;
1806 switch (II->getIntrinsicID()) {
1807 case Intrinsic::amdgcn_cluster_load_b128:
1808 case Intrinsic::amdgcn_cluster_load_b64:
1809 case Intrinsic::amdgcn_cluster_load_b32:
1810 case Intrinsic::amdgcn_ds_append:
1811 case Intrinsic::amdgcn_ds_consume:
1812 case Intrinsic::amdgcn_ds_load_tr8_b64:
1813 case Intrinsic::amdgcn_ds_load_tr16_b128:
1814 case Intrinsic::amdgcn_ds_load_tr4_b64:
1815 case Intrinsic::amdgcn_ds_load_tr6_b96:
1816 case Intrinsic::amdgcn_ds_read_tr4_b64:
1817 case Intrinsic::amdgcn_ds_read_tr6_b96:
1818 case Intrinsic::amdgcn_ds_read_tr8_b64:
1819 case Intrinsic::amdgcn_ds_read_tr16_b64:
1820 case Intrinsic::amdgcn_ds_ordered_add:
1821 case Intrinsic::amdgcn_ds_ordered_swap:
1822 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
1823 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
1824 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1825 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1826 case Intrinsic::amdgcn_global_atomic_fmax_num:
1827 case Intrinsic::amdgcn_global_atomic_fmin_num:
1828 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1829 case Intrinsic::amdgcn_global_load_tr_b64:
1830 case Intrinsic::amdgcn_global_load_tr_b128:
1831 case Intrinsic::amdgcn_global_load_tr4_b64:
1832 case Intrinsic::amdgcn_global_load_tr6_b96:
1833 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1834 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1835 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1836 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
1837 Ptr = II->getArgOperand(i: 0);
1838 break;
1839 case Intrinsic::amdgcn_load_to_lds:
1840 case Intrinsic::amdgcn_load_async_to_lds:
1841 case Intrinsic::amdgcn_global_load_lds:
1842 case Intrinsic::amdgcn_global_load_async_lds:
1843 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1844 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1845 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1846 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1847 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1848 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1849 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1850 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
1851 Ptr = II->getArgOperand(i: 1);
1852 break;
1853 default:
1854 return false;
1855 }
1856 AccessTy = II->getType();
1857 Ops.push_back(Elt: Ptr);
1858 return true;
1859}
1860
1861bool SITargetLowering::isLegalFlatAddressingMode(const AddrMode &AM,
1862 unsigned AddrSpace) const {
1863 if (!Subtarget->hasFlatInstOffsets()) {
1864 // Flat instructions do not have offsets, and only have the register
1865 // address.
1866 return AM.BaseOffs == 0 && AM.Scale == 0;
1867 }
1868
1869 decltype(SIInstrFlags::FLAT) FlatVariant =
1870 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS ? SIInstrFlags::FlatGlobal
1871 : AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ? SIInstrFlags::FlatScratch
1872 : SIInstrFlags::FLAT;
1873
1874 return AM.Scale == 0 &&
1875 (AM.BaseOffs == 0 || Subtarget->getInstrInfo()->isLegalFLATOffset(
1876 Offset: AM.BaseOffs, AddrSpace, FlatVariant));
1877}
1878
1879bool SITargetLowering::isLegalGlobalAddressingMode(const AddrMode &AM) const {
1880 if (Subtarget->hasFlatGlobalInsts())
1881 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS);
1882
1883 if (!Subtarget->hasAddr64() || Subtarget->useFlatForGlobal()) {
1884 // Assume the we will use FLAT for all global memory accesses
1885 // on VI.
1886 // FIXME: This assumption is currently wrong. On VI we still use
1887 // MUBUF instructions for the r + i addressing mode. As currently
1888 // implemented, the MUBUF instructions only work on buffer < 4GB.
1889 // It may be possible to support > 4GB buffers with MUBUF instructions,
1890 // by setting the stride value in the resource descriptor which would
1891 // increase the size limit to (stride * 4GB). However, this is risky,
1892 // because it has never been validated.
1893 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::FLAT_ADDRESS);
1894 }
1895
1896 return isLegalMUBUFAddressingMode(AM);
1897}
1898
1899bool SITargetLowering::isLegalMUBUFAddressingMode(const AddrMode &AM) const {
1900 // MUBUF / MTBUF instructions have a 12-bit unsigned byte offset, and
1901 // additionally can do r + r + i with addr64. 32-bit has more addressing
1902 // mode options. Depending on the resource constant, it can also do
1903 // (i64 r0) + (i32 r1) * (i14 i).
1904 //
1905 // Private arrays end up using a scratch buffer most of the time, so also
1906 // assume those use MUBUF instructions. Scratch loads / stores are currently
1907 // implemented as mubuf instructions with offen bit set, so slightly
1908 // different than the normal addr64.
1909 const SIInstrInfo *TII = Subtarget->getInstrInfo();
1910 if (!TII->isLegalMUBUFImmOffset(Imm: AM.BaseOffs))
1911 return false;
1912
1913 // FIXME: Since we can split immediate into soffset and immediate offset,
1914 // would it make sense to allow any immediate?
1915
1916 switch (AM.Scale) {
1917 case 0: // r + i or just i, depending on HasBaseReg.
1918 return true;
1919 case 1:
1920 return true; // We have r + r or r + i.
1921 case 2:
1922 if (AM.HasBaseReg) {
1923 // Reject 2 * r + r.
1924 return false;
1925 }
1926
1927 // Allow 2 * r as r + r
1928 // Or 2 * r + i is allowed as r + r + i.
1929 return true;
1930 default: // Don't allow n * r
1931 return false;
1932 }
1933}
1934
1935bool SITargetLowering::isLegalAddressingMode(const DataLayout &DL,
1936 const AddrMode &AM, Type *Ty,
1937 unsigned AS,
1938 Instruction *I) const {
1939 // No global is ever allowed as a base.
1940 if (AM.BaseGV)
1941 return false;
1942
1943 if (AS == AMDGPUAS::GLOBAL_ADDRESS)
1944 return isLegalGlobalAddressingMode(AM);
1945
1946 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
1947 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
1948 AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE ||
1949 AS == AMDGPUAS::BUFFER_STRIDED_POINTER) {
1950 // If the offset isn't a multiple of 4, it probably isn't going to be
1951 // correctly aligned.
1952 // FIXME: Can we get the real alignment here?
1953 if (AM.BaseOffs % 4 != 0)
1954 return isLegalMUBUFAddressingMode(AM);
1955
1956 if (!Subtarget->hasScalarSubwordLoads()) {
1957 // There are no SMRD extloads, so if we have to do a small type access we
1958 // will use a MUBUF load.
1959 // FIXME?: We also need to do this if unaligned, but we don't know the
1960 // alignment here.
1961 if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4)
1962 return isLegalGlobalAddressingMode(AM);
1963 }
1964
1965 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) {
1966 // SMRD instructions have an 8-bit, dword offset on SI.
1967 if (!isUInt<8>(x: AM.BaseOffs / 4))
1968 return false;
1969 } else if (Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS) {
1970 // On CI+, this can also be a 32-bit literal constant offset. If it fits
1971 // in 8-bits, it can use a smaller encoding.
1972 if (!isUInt<32>(x: AM.BaseOffs / 4))
1973 return false;
1974 } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX9) {
1975 // On VI, these use the SMEM format and the offset is 20-bit in bytes.
1976 if (!isUInt<20>(x: AM.BaseOffs))
1977 return false;
1978 } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX12) {
1979 // On GFX9 the offset is signed 21-bit in bytes (but must not be negative
1980 // for S_BUFFER_* instructions).
1981 if (!isInt<21>(x: AM.BaseOffs))
1982 return false;
1983 } else {
1984 // On GFX12, all offsets are signed 24-bit in bytes.
1985 if (!isInt<24>(x: AM.BaseOffs))
1986 return false;
1987 }
1988
1989 if ((AS == AMDGPUAS::CONSTANT_ADDRESS ||
1990 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
1991 AM.BaseOffs < 0) {
1992 // Scalar (non-buffer) loads can only use a negative offset if
1993 // soffset+offset is non-negative. Since the compiler can only prove that
1994 // in a few special cases, it is safer to claim that negative offsets are
1995 // not supported.
1996 return false;
1997 }
1998
1999 if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
2000 return true;
2001
2002 if (AM.Scale == 1 && AM.HasBaseReg)
2003 return true;
2004
2005 return false;
2006 }
2007
2008 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
2009 return Subtarget->hasFlatScratchEnabled()
2010 ? isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS)
2011 : isLegalMUBUFAddressingMode(AM);
2012
2013 if (AS == AMDGPUAS::LOCAL_ADDRESS ||
2014 (AS == AMDGPUAS::REGION_ADDRESS && Subtarget->hasGDS())) {
2015 // Basic, single offset DS instructions allow a 16-bit unsigned immediate
2016 // field.
2017 // XXX - If doing a 4-byte aligned 8-byte type access, we effectively have
2018 // an 8-bit dword offset but we don't know the alignment here.
2019 if (!isUInt<16>(x: AM.BaseOffs))
2020 return false;
2021
2022 if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
2023 return true;
2024
2025 if (AM.Scale == 1 && AM.HasBaseReg)
2026 return true;
2027
2028 return false;
2029 }
2030
2031 if (AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::UNKNOWN_ADDRESS_SPACE) {
2032 // For an unknown address space, this usually means that this is for some
2033 // reason being used for pure arithmetic, and not based on some addressing
2034 // computation. We don't have instructions that compute pointers with any
2035 // addressing modes, so treat them as having no offset like flat
2036 // instructions.
2037 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::FLAT_ADDRESS);
2038 }
2039
2040 // Assume a user alias of global for unknown address spaces.
2041 return isLegalGlobalAddressingMode(AM);
2042}
2043
2044bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
2045 const MachineFunction &MF) const {
2046 if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS)
2047 return (MemVT.getSizeInBits() <= 4 * 32);
2048 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
2049 unsigned MaxPrivateBits = 8 * getSubtarget()->getMaxPrivateElementSize();
2050 return (MemVT.getSizeInBits() <= MaxPrivateBits);
2051 }
2052 if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS)
2053 return (MemVT.getSizeInBits() <= 2 * 32);
2054 return true;
2055}
2056
2057bool SITargetLowering::allowsMisalignedMemoryAccessesImpl(
2058 unsigned Size, unsigned AddrSpace, Align Alignment,
2059 MachineMemOperand::Flags Flags, unsigned *IsFast) const {
2060 if (IsFast)
2061 *IsFast = 0;
2062
2063 if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
2064 AddrSpace == AMDGPUAS::REGION_ADDRESS) {
2065 // Check if alignment requirements for ds_read/write instructions are
2066 // disabled.
2067 if (!Subtarget->hasUnalignedDSAccessEnabled() && Alignment < Align(4))
2068 return false;
2069
2070 Align RequiredAlignment(
2071 PowerOf2Ceil(A: divideCeil(Numerator: Size, Denominator: 8))); // Natural alignment.
2072 if (Subtarget->hasLDSMisalignedBugInWGPMode() && Size > 32 &&
2073 Alignment < RequiredAlignment)
2074 return false;
2075
2076 // Either, the alignment requirements are "enabled", or there is an
2077 // unaligned LDS access related hardware bug though alignment requirements
2078 // are "disabled". In either case, we need to check for proper alignment
2079 // requirements.
2080 //
2081 switch (Size) {
2082 case 64:
2083 // SI has a hardware bug in the LDS / GDS bounds checking: if the base
2084 // address is negative, then the instruction is incorrectly treated as
2085 // out-of-bounds even if base + offsets is in bounds. Split vectorized
2086 // loads here to avoid emitting ds_read2_b32. We may re-combine the
2087 // load later in the SILoadStoreOptimizer.
2088 if (!Subtarget->hasUsableDSOffset() && Alignment < Align(8))
2089 return false;
2090
2091 // 8 byte accessing via ds_read/write_b64 require 8-byte alignment, but we
2092 // can do a 4 byte aligned, 8 byte access in a single operation using
2093 // ds_read2/write2_b32 with adjacent offsets.
2094 RequiredAlignment = Align(4);
2095
2096 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2097 // We will either select ds_read_b64/ds_write_b64 or ds_read2_b32/
2098 // ds_write2_b32 depending on the alignment. In either case with either
2099 // alignment there is no faster way of doing this.
2100
2101 // The numbers returned here and below are not additive, it is a 'speed
2102 // rank'. They are just meant to be compared to decide if a certain way
2103 // of lowering an operation is faster than another. For that purpose
2104 // naturally aligned operation gets it bitsize to indicate that "it
2105 // operates with a speed comparable to N-bit wide load". With the full
2106 // alignment ds128 is slower than ds96 for example. If underaligned it
2107 // is comparable to a speed of a single dword access, which would then
2108 // mean 32 < 128 and it is faster to issue a wide load regardless.
2109 // 1 is simply "slow, don't do it". I.e. comparing an aligned load to a
2110 // wider load which will not be aligned anymore the latter is slower.
2111 if (IsFast)
2112 *IsFast = (Alignment >= RequiredAlignment) ? 64
2113 : (Alignment < Align(4)) ? 32
2114 : 1;
2115 return true;
2116 }
2117
2118 break;
2119 case 96:
2120 if (!Subtarget->hasDS96AndDS128())
2121 return false;
2122
2123 // 12 byte accessing via ds_read/write_b96 require 16-byte alignment on
2124 // gfx8 and older.
2125
2126 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2127 // Naturally aligned access is fastest. However, also report it is Fast
2128 // if memory is aligned less than DWORD. A narrow load or store will be
2129 // be equally slow as a single ds_read_b96/ds_write_b96, but there will
2130 // be more of them, so overall we will pay less penalty issuing a single
2131 // instruction.
2132
2133 // See comment on the values above.
2134 if (IsFast)
2135 *IsFast = (Alignment >= RequiredAlignment) ? 96
2136 : (Alignment < Align(4)) ? 32
2137 : 1;
2138 return true;
2139 }
2140
2141 break;
2142 case 128:
2143 if (!Subtarget->hasDS96AndDS128() || !Subtarget->useDS128())
2144 return false;
2145
2146 // 16 byte accessing via ds_read/write_b128 require 16-byte alignment on
2147 // gfx8 and older, but we can do a 8 byte aligned, 16 byte access in a
2148 // single operation using ds_read2/write2_b64.
2149 RequiredAlignment = Align(8);
2150
2151 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2152 // Naturally aligned access is fastest. However, also report it is Fast
2153 // if memory is aligned less than DWORD. A narrow load or store will be
2154 // be equally slow as a single ds_read_b128/ds_write_b128, but there
2155 // will be more of them, so overall we will pay less penalty issuing a
2156 // single instruction.
2157
2158 // See comment on the values above.
2159 if (IsFast)
2160 *IsFast = (Alignment >= RequiredAlignment) ? 128
2161 : (Alignment < Align(4)) ? 32
2162 : 1;
2163 return true;
2164 }
2165
2166 break;
2167 default:
2168 if (Size > 32)
2169 return false;
2170
2171 break;
2172 }
2173
2174 // See comment on the values above.
2175 // Note that we have a single-dword or sub-dword here, so if underaligned
2176 // it is a slowest possible access, hence returned value is 0.
2177 if (IsFast)
2178 *IsFast = (Alignment >= RequiredAlignment) ? Size : 0;
2179
2180 return Alignment >= RequiredAlignment ||
2181 Subtarget->hasUnalignedDSAccessEnabled();
2182 }
2183
2184 // FIXME: We have to be conservative here and assume that flat operations
2185 // will access scratch. If we had access to the IR function, then we
2186 // could determine if any private memory was used in the function.
2187 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ||
2188 AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
2189 bool AlignedBy4 = Alignment >= Align(4);
2190 if (Subtarget->hasUnalignedScratchAccessEnabled()) {
2191 if (IsFast)
2192 *IsFast = AlignedBy4 ? Size : 1;
2193 return true;
2194 }
2195
2196 if (IsFast)
2197 *IsFast = AlignedBy4;
2198
2199 return AlignedBy4;
2200 }
2201
2202 // So long as they are correct, wide global memory operations perform better
2203 // than multiple smaller memory ops -- even when misaligned
2204 if (AMDGPU::isExtendedGlobalAddrSpace(AS: AddrSpace)) {
2205 if (IsFast)
2206 *IsFast = Size;
2207
2208 return Alignment >= Align(4) ||
2209 Subtarget->hasUnalignedBufferAccessEnabled();
2210 }
2211
2212 // Ensure robust out-of-bounds guarantees for buffer accesses are met if
2213 // RelaxedBufferOOBMode is disabled. Normally hardware will ensure proper
2214 // out-of-bounds behavior, but in the edge case where an access starts
2215 // out-of-bounds and then enter in-bounds, the entire access would be treated
2216 // as out-of-bounds. Prevent misaligned memory accesses by requiring the
2217 // natural alignment of buffer accesses.
2218 if (AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
2219 AddrSpace == AMDGPUAS::BUFFER_RESOURCE ||
2220 AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) {
2221 if (!Subtarget->hasRelaxedBufferOOBMode() &&
2222 Alignment < Align(PowerOf2Ceil(A: divideCeil(Numerator: Size, Denominator: 8))))
2223 return false;
2224 }
2225
2226 // Smaller than dword value must be aligned.
2227 if (Size < 32)
2228 return false;
2229
2230 // 8.1.6 - For Dword or larger reads or writes, the two LSBs of the
2231 // byte-address are ignored, thus forcing Dword alignment.
2232 // This applies to private, global, and constant memory.
2233 if (IsFast)
2234 *IsFast = 1;
2235
2236 return Size >= 32 && Alignment >= Align(4);
2237}
2238
2239bool SITargetLowering::allowsMisalignedMemoryAccesses(
2240 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
2241 unsigned *IsFast) const {
2242 return allowsMisalignedMemoryAccessesImpl(Size: VT.getSizeInBits(), AddrSpace,
2243 Alignment, Flags, IsFast);
2244}
2245
2246EVT SITargetLowering::getOptimalMemOpType(
2247 LLVMContext &Context, const MemOp &Op,
2248 const AttributeList &FuncAttributes) const {
2249 // FIXME: Should account for address space here.
2250
2251 // The default fallback uses the private pointer size as a guess for a type to
2252 // use. Make sure we switch these to 64-bit accesses.
2253
2254 if (Op.size() >= 16 &&
2255 Op.isDstAligned(AlignCheck: Align(4))) // XXX: Should only do for global
2256 return MVT::v4i32;
2257
2258 if (Op.size() >= 8 && Op.isDstAligned(AlignCheck: Align(4)))
2259 return MVT::v2i32;
2260
2261 // Use the default.
2262 return MVT::Other;
2263}
2264
2265bool SITargetLowering::isMemOpHasNoClobberedMemOperand(const SDNode *N) const {
2266 const MemSDNode *MemNode = cast<MemSDNode>(Val: N);
2267 return MemNode->getMemOperand()->getFlags() & MONoClobber;
2268}
2269
2270bool SITargetLowering::isNonGlobalAddrSpace(unsigned AS) {
2271 return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS ||
2272 AS == AMDGPUAS::PRIVATE_ADDRESS;
2273}
2274
2275bool SITargetLowering::isFreeAddrSpaceCast(unsigned SrcAS,
2276 unsigned DestAS) const {
2277 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
2278 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
2279 Subtarget->hasGloballyAddressableScratch()) {
2280 // Flat -> private requires subtracting src_flat_scratch_base_lo.
2281 return false;
2282 }
2283
2284 // Flat -> private/local is a simple truncate.
2285 // Flat -> global is no-op
2286 return true;
2287 }
2288
2289 const GCNTargetMachine &TM =
2290 static_cast<const GCNTargetMachine &>(getTargetMachine());
2291 return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
2292}
2293
2294TargetLoweringBase::LegalizeTypeAction
2295SITargetLowering::getPreferredVectorAction(MVT VT) const {
2296 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
2297 VT.getScalarType().bitsLE(VT: MVT::i16))
2298 return VT.isPow2VectorType() ? TypeSplitVector : TypeWidenVector;
2299 return TargetLoweringBase::getPreferredVectorAction(VT);
2300}
2301
2302bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
2303 Type *Ty) const {
2304 // FIXME: Could be smarter if called for vector constants.
2305 return true;
2306}
2307
2308bool SITargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
2309 unsigned Index) const {
2310 if (!isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: ResVT))
2311 return false;
2312
2313 // TODO: Add more cases that are cheap.
2314 return Index == 0;
2315}
2316
2317bool SITargetLowering::isExtractVecEltCheap(EVT VT, unsigned Index) const {
2318 // TODO: This should be more aggressive, particular for 16-bit element
2319 // vectors. However there are some mixed improvements and regressions.
2320 EVT EltTy = VT.getVectorElementType();
2321 unsigned MinAlign = Subtarget->useRealTrue16Insts() ? 16 : 32;
2322 return EltTy.getSizeInBits() % MinAlign == 0;
2323}
2324
2325bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT) const {
2326 if (Subtarget->has16BitInsts() && VT == MVT::i16) {
2327 switch (Op) {
2328 case ISD::LOAD:
2329 case ISD::STORE:
2330 return true;
2331 default:
2332 return false;
2333 }
2334 }
2335
2336 // SimplifySetCC uses this function to determine whether or not it should
2337 // create setcc with i1 operands. We don't have instructions for i1 setcc.
2338 if (VT == MVT::i1 && Op == ISD::SETCC)
2339 return false;
2340
2341 return TargetLowering::isTypeDesirableForOp(Op, VT);
2342}
2343
2344MachinePointerInfo
2345SITargetLowering::getKernargSegmentPtrInfo(MachineFunction &MF) const {
2346 // This isn't really a constant pool but close enough.
2347 MachinePointerInfo PtrInfo(MF.getPSVManager().getConstantPool());
2348 PtrInfo.AddrSpace = AMDGPUAS::CONSTANT_ADDRESS;
2349 return PtrInfo;
2350}
2351
2352SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
2353 const SDLoc &SL,
2354 SDValue Chain,
2355 uint64_t Offset) const {
2356 const DataLayout &DL = DAG.getDataLayout();
2357 MachineFunction &MF = DAG.getMachineFunction();
2358 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
2359 MVT PtrVT = getPointerTy(DL, AS: AMDGPUAS::CONSTANT_ADDRESS);
2360
2361 auto [InputPtrReg, RC, ArgTy] =
2362 Info->getPreloadedValue(Value: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
2363
2364 // We may not have the kernarg segment argument if we have no kernel
2365 // arguments.
2366 if (!InputPtrReg)
2367 return DAG.getConstant(Val: Offset, DL: SL, VT: PtrVT);
2368
2369 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
2370 SDValue BasePtr = DAG.getCopyFromReg(
2371 Chain, dl: SL, Reg: MRI.getLiveInVirtReg(PReg: InputPtrReg->getRegister()), VT: PtrVT);
2372
2373 return DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
2374}
2375
2376SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG,
2377 const SDLoc &SL) const {
2378 uint64_t Offset =
2379 getImplicitParameterOffset(MF: DAG.getMachineFunction(), Param: FIRST_IMPLICIT);
2380 return lowerKernArgParameterPtr(DAG, SL, Chain: DAG.getEntryNode(), Offset);
2381}
2382
2383SDValue SITargetLowering::getLDSKernelId(SelectionDAG &DAG,
2384 const SDLoc &SL) const {
2385
2386 Function &F = DAG.getMachineFunction().getFunction();
2387 std::optional<uint32_t> KnownSize =
2388 AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F);
2389 if (KnownSize.has_value())
2390 return DAG.getConstant(Val: *KnownSize, DL: SL, VT: MVT::i32);
2391 return SDValue();
2392}
2393
2394SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
2395 const SDLoc &SL, SDValue Val,
2396 bool Signed,
2397 const ISD::InputArg *Arg) const {
2398 // First, if it is a widened vector, narrow it.
2399 if (VT.isVector() &&
2400 VT.getVectorNumElements() != MemVT.getVectorNumElements()) {
2401 EVT NarrowedVT =
2402 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MemVT.getVectorElementType(),
2403 NumElements: VT.getVectorNumElements());
2404 Val = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: NarrowedVT, N1: Val,
2405 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
2406 }
2407
2408 // Then convert the vector elements or scalar value.
2409 if (Arg && (Arg->Flags.isSExt() || Arg->Flags.isZExt()) && VT.bitsLT(VT: MemVT)) {
2410 unsigned Opc = Arg->Flags.isZExt() ? ISD::AssertZext : ISD::AssertSext;
2411 Val = DAG.getNode(Opcode: Opc, DL: SL, VT: MemVT, N1: Val, N2: DAG.getValueType(VT));
2412 }
2413
2414 if (MemVT.isFloatingPoint()) {
2415 if (VT.isFloatingPoint()) {
2416 Val = getFPExtOrFPRound(DAG, Op: Val, DL: SL, VT);
2417 } else {
2418 assert(!MemVT.isVector());
2419 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
2420 SDValue Cast = DAG.getBitcast(VT: IntVT, V: Val);
2421 Val = DAG.getAnyExtOrTrunc(Op: Cast, DL: SL, VT);
2422 }
2423 } else if (Signed)
2424 Val = DAG.getSExtOrTrunc(Op: Val, DL: SL, VT);
2425 else
2426 Val = DAG.getZExtOrTrunc(Op: Val, DL: SL, VT);
2427
2428 return Val;
2429}
2430
2431SDValue SITargetLowering::lowerKernargMemParameter(
2432 SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain,
2433 uint64_t Offset, Align Alignment, bool Signed,
2434 const ISD::InputArg *Arg) const {
2435
2436 MachinePointerInfo PtrInfo =
2437 getKernargSegmentPtrInfo(MF&: DAG.getMachineFunction());
2438
2439 // Try to avoid using an extload by loading earlier than the argument address,
2440 // and extracting the relevant bits. The load should hopefully be merged with
2441 // the previous argument.
2442 if (MemVT.getStoreSize() < 4 && Alignment < 4) {
2443 // TODO: Handle align < 4 and size >= 4 (can happen with packed structs).
2444 int64_t AlignDownOffset = alignDown(Value: Offset, Align: 4);
2445 int64_t OffsetDiff = Offset - AlignDownOffset;
2446
2447 EVT IntVT = MemVT.changeTypeToInteger();
2448
2449 // TODO: If we passed in the base kernel offset we could have a better
2450 // alignment than 4, but we don't really need it.
2451 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset: AlignDownOffset);
2452 SDValue Load = DAG.getLoad(VT: MVT::i32, dl: SL, Chain, Ptr,
2453 PtrInfo: PtrInfo.getWithOffset(O: AlignDownOffset), Alignment: Align(4),
2454 MMOFlags: MachineMemOperand::MODereferenceable |
2455 MachineMemOperand::MOInvariant);
2456
2457 SDValue ShiftAmt = DAG.getConstant(Val: OffsetDiff * 8, DL: SL, VT: MVT::i32);
2458 SDValue Extract = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: Load, N2: ShiftAmt);
2459
2460 SDValue ArgVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: IntVT, Operand: Extract);
2461 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MemVT, Operand: ArgVal);
2462 ArgVal = convertArgType(DAG, VT, MemVT, SL, Val: ArgVal, Signed, Arg);
2463
2464 return DAG.getMergeValues(Ops: {ArgVal, Load.getValue(R: 1)}, dl: SL);
2465 }
2466
2467 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
2468 SDValue Load = DAG.getLoad(
2469 VT: MemVT, dl: SL, Chain, Ptr, PtrInfo: PtrInfo.getWithOffset(O: Offset), Alignment,
2470 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
2471
2472 SDValue Val = convertArgType(DAG, VT, MemVT, SL, Val: Load, Signed, Arg);
2473 return DAG.getMergeValues(Ops: {Val, Load.getValue(R: 1)}, dl: SL);
2474}
2475
2476/// Coerce an argument which was passed in a different ABI type to the original
2477/// expected value type.
2478SDValue SITargetLowering::convertABITypeToValueType(SelectionDAG &DAG,
2479 SDValue Val,
2480 CCValAssign &VA,
2481 const SDLoc &SL) const {
2482 EVT ValVT = VA.getValVT();
2483
2484 // If this is an 8 or 16-bit value, it is really passed promoted
2485 // to 32 bits. Insert an assert[sz]ext to capture this, then
2486 // truncate to the right size.
2487 switch (VA.getLocInfo()) {
2488 case CCValAssign::Full:
2489 return Val;
2490 case CCValAssign::BCvt:
2491 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ValVT, Operand: Val);
2492 case CCValAssign::SExt:
2493 Val = DAG.getNode(Opcode: ISD::AssertSext, DL: SL, VT: VA.getLocVT(), N1: Val,
2494 N2: DAG.getValueType(ValVT));
2495 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2496 case CCValAssign::ZExt:
2497 Val = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: VA.getLocVT(), N1: Val,
2498 N2: DAG.getValueType(ValVT));
2499 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2500 case CCValAssign::AExt:
2501 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2502 default:
2503 llvm_unreachable("Unknown loc info!");
2504 }
2505}
2506
2507SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG,
2508 CCValAssign &VA, const SDLoc &SL,
2509 SDValue Chain,
2510 const ISD::InputArg &Arg) const {
2511 MachineFunction &MF = DAG.getMachineFunction();
2512 MachineFrameInfo &MFI = MF.getFrameInfo();
2513
2514 if (Arg.Flags.isByVal()) {
2515 unsigned Size = Arg.Flags.getByValSize();
2516 int FrameIdx = MFI.CreateFixedObject(Size, SPOffset: VA.getLocMemOffset(), IsImmutable: false);
2517 return DAG.getFrameIndex(FI: FrameIdx, VT: MVT::i32);
2518 }
2519
2520 unsigned ArgOffset = VA.getLocMemOffset();
2521 unsigned ArgSize = VA.getValVT().getStoreSize();
2522
2523 int FI = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: true);
2524
2525 // Create load nodes to retrieve arguments from the stack.
2526 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
2527
2528 // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
2529 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
2530 MVT MemVT = VA.getValVT();
2531
2532 switch (VA.getLocInfo()) {
2533 default:
2534 break;
2535 case CCValAssign::BCvt:
2536 MemVT = VA.getLocVT();
2537 break;
2538 case CCValAssign::SExt:
2539 ExtType = ISD::SEXTLOAD;
2540 break;
2541 case CCValAssign::ZExt:
2542 ExtType = ISD::ZEXTLOAD;
2543 break;
2544 case CCValAssign::AExt:
2545 ExtType = ISD::EXTLOAD;
2546 break;
2547 }
2548
2549 SDValue ArgValue = DAG.getExtLoad(
2550 ExtType, dl: SL, VT: VA.getLocVT(), Chain, Ptr: FIN,
2551 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), MemVT);
2552
2553 SDValue ConvertedVal = convertABITypeToValueType(DAG, Val: ArgValue, VA, SL);
2554 if (ConvertedVal == ArgValue)
2555 return ConvertedVal;
2556
2557 return DAG.getMergeValues(Ops: {ConvertedVal, ArgValue.getValue(R: 1)}, dl: SL);
2558}
2559
2560SDValue SITargetLowering::lowerWorkGroupId(
2561 SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
2562 AMDGPUFunctionArgInfo::PreloadedValue WorkGroupIdPV,
2563 AMDGPUFunctionArgInfo::PreloadedValue ClusterMaxIdPV,
2564 AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const {
2565 if (!Subtarget->hasClusters())
2566 return getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
2567
2568 // Clusters are supported. Return the global position in the grid. If clusters
2569 // are enabled, WorkGroupIdPV returns the cluster ID not the workgroup ID.
2570
2571 // WorkGroupIdXYZ = ClusterId == 0 ?
2572 // ClusterIdXYZ :
2573 // ClusterIdXYZ * (ClusterMaxIdXYZ + 1) + ClusterWorkGroupIdXYZ
2574 SDValue ClusterIdXYZ = getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
2575 SDLoc SL(ClusterIdXYZ);
2576 SDValue ClusterMaxIdXYZ = getPreloadedValue(DAG, MFI, VT, ClusterMaxIdPV);
2577 SDValue One = DAG.getConstant(Val: 1, DL: SL, VT);
2578 SDValue ClusterSizeXYZ = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ClusterMaxIdXYZ, N2: One);
2579 SDValue ClusterWorkGroupIdXYZ =
2580 getPreloadedValue(DAG, MFI, VT, ClusterWorkGroupIdPV);
2581 SDValue GlobalIdXYZ =
2582 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ClusterWorkGroupIdXYZ,
2583 N2: DAG.getNode(Opcode: ISD::MUL, DL: SL, VT, N1: ClusterIdXYZ, N2: ClusterSizeXYZ));
2584
2585 switch (MFI.getClusterDims().getKind()) {
2586 case AMDGPU::ClusterDimsAttr::Kind::FixedDims:
2587 case AMDGPU::ClusterDimsAttr::Kind::VariableDims:
2588 return GlobalIdXYZ;
2589 case AMDGPU::ClusterDimsAttr::Kind::NoCluster:
2590 return ClusterIdXYZ;
2591 case AMDGPU::ClusterDimsAttr::Kind::Unknown: {
2592 using namespace AMDGPU::Hwreg;
2593 SDValue ClusterIdField =
2594 DAG.getTargetConstant(Val: HwregEncoding::encode(Values: ID_IB_STS2, Values: 6, Values: 4), DL: SL, VT);
2595 SDNode *GetReg =
2596 DAG.getMachineNode(Opcode: AMDGPU::S_GETREG_B32_const, dl: SL, VT, Op1: ClusterIdField);
2597 SDValue ClusterId(GetReg, 0);
2598 SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT);
2599 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SL, VT, N1: ClusterId, N2: Zero, N3: ClusterIdXYZ,
2600 N4: GlobalIdXYZ, N5: DAG.getCondCode(Cond: ISD::SETEQ));
2601 }
2602 }
2603
2604 llvm_unreachable("nothing should reach here");
2605}
2606
2607SDValue SITargetLowering::getPreloadedValue(
2608 SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
2609 AMDGPUFunctionArgInfo::PreloadedValue PVID) const {
2610 const ArgDescriptor *Reg = nullptr;
2611 const TargetRegisterClass *RC;
2612 LLT Ty;
2613
2614 CallingConv::ID CC = DAG.getMachineFunction().getFunction().getCallingConv();
2615 const ArgDescriptor WorkGroupIDX =
2616 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP9);
2617 // If GridZ is not programmed in an entry function then the hardware will set
2618 // it to all zeros, so there is no need to mask the GridY value in the low
2619 // order bits.
2620 const ArgDescriptor WorkGroupIDY = ArgDescriptor::createRegister(
2621 Reg: AMDGPU::TTMP7,
2622 Mask: AMDGPU::isEntryFunctionCC(CC) && !MFI.hasWorkGroupIDZ() ? ~0u : 0xFFFFu);
2623 const ArgDescriptor WorkGroupIDZ =
2624 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP7, Mask: 0xFFFF0000u);
2625 const ArgDescriptor ClusterWorkGroupIDX =
2626 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0000000Fu);
2627 const ArgDescriptor ClusterWorkGroupIDY =
2628 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x000000F0u);
2629 const ArgDescriptor ClusterWorkGroupIDZ =
2630 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x00000F00u);
2631 const ArgDescriptor ClusterWorkGroupMaxIDX =
2632 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0000F000u);
2633 const ArgDescriptor ClusterWorkGroupMaxIDY =
2634 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x000F0000u);
2635 const ArgDescriptor ClusterWorkGroupMaxIDZ =
2636 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x00F00000u);
2637 const ArgDescriptor ClusterWorkGroupMaxFlatID =
2638 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0F000000u);
2639
2640 auto LoadConstant = [&](unsigned N) {
2641 return DAG.getConstant(Val: N, DL: SDLoc(), VT);
2642 };
2643
2644 if (Subtarget->hasArchitectedSGPRs() &&
2645 (AMDGPU::isCompute(CC) || CC == CallingConv::AMDGPU_Gfx)) {
2646 AMDGPU::ClusterDimsAttr ClusterDims = MFI.getClusterDims();
2647 bool HasFixedDims = ClusterDims.isFixedDims();
2648
2649 switch (PVID) {
2650 case AMDGPUFunctionArgInfo::WORKGROUP_ID_X:
2651 Reg = &WorkGroupIDX;
2652 RC = &AMDGPU::SReg_32RegClass;
2653 Ty = LLT::scalar(SizeInBits: 32);
2654 break;
2655 case AMDGPUFunctionArgInfo::WORKGROUP_ID_Y:
2656 Reg = &WorkGroupIDY;
2657 RC = &AMDGPU::SReg_32RegClass;
2658 Ty = LLT::scalar(SizeInBits: 32);
2659 break;
2660 case AMDGPUFunctionArgInfo::WORKGROUP_ID_Z:
2661 Reg = &WorkGroupIDZ;
2662 RC = &AMDGPU::SReg_32RegClass;
2663 Ty = LLT::scalar(SizeInBits: 32);
2664 break;
2665 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X:
2666 if (HasFixedDims && ClusterDims.getDims()[0] == 1)
2667 return LoadConstant(0);
2668 Reg = &ClusterWorkGroupIDX;
2669 RC = &AMDGPU::SReg_32RegClass;
2670 Ty = LLT::scalar(SizeInBits: 32);
2671 break;
2672 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y:
2673 if (HasFixedDims && ClusterDims.getDims()[1] == 1)
2674 return LoadConstant(0);
2675 Reg = &ClusterWorkGroupIDY;
2676 RC = &AMDGPU::SReg_32RegClass;
2677 Ty = LLT::scalar(SizeInBits: 32);
2678 break;
2679 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z:
2680 if (HasFixedDims && ClusterDims.getDims()[2] == 1)
2681 return LoadConstant(0);
2682 Reg = &ClusterWorkGroupIDZ;
2683 RC = &AMDGPU::SReg_32RegClass;
2684 Ty = LLT::scalar(SizeInBits: 32);
2685 break;
2686 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X:
2687 if (HasFixedDims)
2688 return LoadConstant(ClusterDims.getDims()[0] - 1);
2689 Reg = &ClusterWorkGroupMaxIDX;
2690 RC = &AMDGPU::SReg_32RegClass;
2691 Ty = LLT::scalar(SizeInBits: 32);
2692 break;
2693 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y:
2694 if (HasFixedDims)
2695 return LoadConstant(ClusterDims.getDims()[1] - 1);
2696 Reg = &ClusterWorkGroupMaxIDY;
2697 RC = &AMDGPU::SReg_32RegClass;
2698 Ty = LLT::scalar(SizeInBits: 32);
2699 break;
2700 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z:
2701 if (HasFixedDims)
2702 return LoadConstant(ClusterDims.getDims()[2] - 1);
2703 Reg = &ClusterWorkGroupMaxIDZ;
2704 RC = &AMDGPU::SReg_32RegClass;
2705 Ty = LLT::scalar(SizeInBits: 32);
2706 break;
2707 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID:
2708 Reg = &ClusterWorkGroupMaxFlatID;
2709 RC = &AMDGPU::SReg_32RegClass;
2710 Ty = LLT::scalar(SizeInBits: 32);
2711 break;
2712 default:
2713 break;
2714 }
2715 }
2716
2717 if (!Reg)
2718 std::tie(args&: Reg, args&: RC, args&: Ty) = MFI.getPreloadedValue(Value: PVID);
2719 if (!Reg) {
2720 if (PVID == AMDGPUFunctionArgInfo::PreloadedValue::KERNARG_SEGMENT_PTR) {
2721 // It's possible for a kernarg intrinsic call to appear in a kernel with
2722 // no allocated segment, in which case we do not add the user sgpr
2723 // argument, so just return null.
2724 return DAG.getConstant(Val: 0, DL: SDLoc(), VT);
2725 }
2726
2727 // It's undefined behavior if a function marked with the amdgpu-no-*
2728 // attributes uses the corresponding intrinsic.
2729 return DAG.getPOISON(VT);
2730 }
2731
2732 return loadInputValue(DAG, RC, VT, SL: SDLoc(DAG.getEntryNode()), Arg: *Reg);
2733}
2734
2735static void processPSInputArgs(SmallVectorImpl<ISD::InputArg> &Splits,
2736 CallingConv::ID CallConv,
2737 ArrayRef<ISD::InputArg> Ins, BitVector &Skipped,
2738 FunctionType *FType,
2739 SIMachineFunctionInfo *Info) {
2740 for (unsigned I = 0, E = Ins.size(), PSInputNum = 0; I != E; ++I) {
2741 const ISD::InputArg *Arg = &Ins[I];
2742
2743 assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
2744 "vector type argument should have been split");
2745
2746 // First check if it's a PS input addr.
2747 if (CallConv == CallingConv::AMDGPU_PS && !Arg->Flags.isInReg() &&
2748 PSInputNum <= 15) {
2749 bool SkipArg = !Arg->Used && !Info->isPSInputAllocated(Index: PSInputNum);
2750
2751 // Inconveniently only the first part of the split is marked as isSplit,
2752 // so skip to the end. We only want to increment PSInputNum once for the
2753 // entire split argument.
2754 if (Arg->Flags.isSplit()) {
2755 while (!Arg->Flags.isSplitEnd()) {
2756 assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
2757 "unexpected vector split in ps argument type");
2758 if (!SkipArg)
2759 Splits.push_back(Elt: *Arg);
2760 Arg = &Ins[++I];
2761 }
2762 }
2763
2764 if (SkipArg) {
2765 // We can safely skip PS inputs.
2766 Skipped.set(Arg->getOrigArgIndex());
2767 ++PSInputNum;
2768 continue;
2769 }
2770
2771 Info->markPSInputAllocated(Index: PSInputNum);
2772 if (Arg->Used)
2773 Info->markPSInputEnabled(Index: PSInputNum);
2774
2775 ++PSInputNum;
2776 }
2777
2778 Splits.push_back(Elt: *Arg);
2779 }
2780}
2781
2782// Allocate special inputs passed in VGPRs.
2783void SITargetLowering::allocateSpecialEntryInputVGPRs(
2784 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
2785 SIMachineFunctionInfo &Info) const {
2786 const LLT S32 = LLT::scalar(SizeInBits: 32);
2787 MachineRegisterInfo &MRI = MF.getRegInfo();
2788
2789 if (Info.hasWorkItemIDX()) {
2790 Register Reg = AMDGPU::VGPR0;
2791 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: S32);
2792
2793 CCInfo.AllocateReg(Reg);
2794 unsigned Mask =
2795 (Subtarget->hasPackedTID() && Info.hasWorkItemIDY()) ? 0x3ff : ~0u;
2796 Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
2797 }
2798
2799 if (Info.hasWorkItemIDY()) {
2800 assert(Info.hasWorkItemIDX());
2801 if (Subtarget->hasPackedTID()) {
2802 Info.setWorkItemIDY(
2803 ArgDescriptor::createRegister(Reg: AMDGPU::VGPR0, Mask: 0x3ff << 10));
2804 } else {
2805 unsigned Reg = AMDGPU::VGPR1;
2806 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: S32);
2807
2808 CCInfo.AllocateReg(Reg);
2809 Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg));
2810 }
2811 }
2812
2813 if (Info.hasWorkItemIDZ()) {
2814 assert(Info.hasWorkItemIDX() && Info.hasWorkItemIDY());
2815 if (Subtarget->hasPackedTID()) {
2816 Info.setWorkItemIDZ(
2817 ArgDescriptor::createRegister(Reg: AMDGPU::VGPR0, Mask: 0x3ff << 20));
2818 } else {
2819 unsigned Reg = AMDGPU::VGPR2;
2820 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: S32);
2821
2822 CCInfo.AllocateReg(Reg);
2823 Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg));
2824 }
2825 }
2826}
2827
2828// Try to allocate a VGPR at the end of the argument list, or if no argument
2829// VGPRs are left allocating a stack slot.
2830// If \p Mask is is given it indicates bitfield position in the register.
2831// If \p Arg is given use it with new ]p Mask instead of allocating new.
2832static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u,
2833 ArgDescriptor Arg = ArgDescriptor()) {
2834 if (Arg.isSet())
2835 return ArgDescriptor::createArg(Arg, Mask);
2836
2837 ArrayRef<MCPhysReg> ArgVGPRs = ArrayRef(AMDGPU::VGPR_32RegClass.begin(), 32);
2838 unsigned RegIdx = CCInfo.getFirstUnallocated(Regs: ArgVGPRs);
2839 if (RegIdx == ArgVGPRs.size()) {
2840 // Spill to stack required.
2841 int64_t Offset = CCInfo.AllocateStack(Size: 4, Alignment: Align(4));
2842
2843 return ArgDescriptor::createStack(Offset, Mask);
2844 }
2845
2846 unsigned Reg = ArgVGPRs[RegIdx];
2847 Reg = CCInfo.AllocateReg(Reg);
2848 assert(Reg != AMDGPU::NoRegister);
2849
2850 MachineFunction &MF = CCInfo.getMachineFunction();
2851 Register LiveInVReg = MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass);
2852 MF.getRegInfo().setType(VReg: LiveInVReg, Ty: LLT::scalar(SizeInBits: 32));
2853 return ArgDescriptor::createRegister(Reg, Mask);
2854}
2855
2856static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo,
2857 const TargetRegisterClass *RC,
2858 unsigned NumArgRegs) {
2859 ArrayRef<MCPhysReg> ArgSGPRs = ArrayRef(RC->begin(), 32);
2860 unsigned RegIdx = CCInfo.getFirstUnallocated(Regs: ArgSGPRs);
2861 if (RegIdx == ArgSGPRs.size())
2862 report_fatal_error(reason: "ran out of SGPRs for arguments");
2863
2864 unsigned Reg = ArgSGPRs[RegIdx];
2865 Reg = CCInfo.AllocateReg(Reg);
2866 assert(Reg != AMDGPU::NoRegister);
2867
2868 MachineFunction &MF = CCInfo.getMachineFunction();
2869 MF.addLiveIn(PReg: Reg, RC);
2870 return ArgDescriptor::createRegister(Reg);
2871}
2872
2873// If this has a fixed position, we still should allocate the register in the
2874// CCInfo state. Technically we could get away with this for values passed
2875// outside of the normal argument range.
2876static void allocateFixedSGPRInputImpl(CCState &CCInfo,
2877 const TargetRegisterClass *RC,
2878 MCRegister Reg) {
2879 Reg = CCInfo.AllocateReg(Reg);
2880 assert(Reg != AMDGPU::NoRegister);
2881 MachineFunction &MF = CCInfo.getMachineFunction();
2882 MF.addLiveIn(PReg: Reg, RC);
2883}
2884
2885static void allocateSGPR32Input(CCState &CCInfo, ArgDescriptor &Arg) {
2886 if (Arg) {
2887 allocateFixedSGPRInputImpl(CCInfo, RC: &AMDGPU::SGPR_32RegClass,
2888 Reg: Arg.getRegister());
2889 } else
2890 Arg = allocateSGPR32InputImpl(CCInfo, RC: &AMDGPU::SGPR_32RegClass, NumArgRegs: 32);
2891}
2892
2893static void allocateSGPR64Input(CCState &CCInfo, ArgDescriptor &Arg) {
2894 if (Arg) {
2895 allocateFixedSGPRInputImpl(CCInfo, RC: &AMDGPU::SGPR_64RegClass,
2896 Reg: Arg.getRegister());
2897 } else
2898 Arg = allocateSGPR32InputImpl(CCInfo, RC: &AMDGPU::SGPR_64RegClass, NumArgRegs: 16);
2899}
2900
2901/// Allocate implicit function VGPR arguments at the end of allocated user
2902/// arguments.
2903void SITargetLowering::allocateSpecialInputVGPRs(
2904 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
2905 SIMachineFunctionInfo &Info) const {
2906 const unsigned Mask = 0x3ff;
2907 ArgDescriptor Arg;
2908
2909 if (Info.hasWorkItemIDX()) {
2910 Arg = allocateVGPR32Input(CCInfo, Mask);
2911 Info.setWorkItemIDX(Arg);
2912 }
2913
2914 if (Info.hasWorkItemIDY()) {
2915 Arg = allocateVGPR32Input(CCInfo, Mask: Mask << 10, Arg);
2916 Info.setWorkItemIDY(Arg);
2917 }
2918
2919 if (Info.hasWorkItemIDZ())
2920 Info.setWorkItemIDZ(allocateVGPR32Input(CCInfo, Mask: Mask << 20, Arg));
2921}
2922
2923/// Allocate implicit function VGPR arguments in fixed registers.
2924void SITargetLowering::allocateSpecialInputVGPRsFixed(
2925 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
2926 SIMachineFunctionInfo &Info) const {
2927 Register Reg = CCInfo.AllocateReg(Reg: AMDGPU::VGPR31);
2928 if (!Reg)
2929 report_fatal_error(reason: "failed to allocate VGPR for implicit arguments");
2930
2931 const unsigned Mask = 0x3ff;
2932 Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
2933 Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg, Mask: Mask << 10));
2934 Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg, Mask: Mask << 20));
2935}
2936
2937void SITargetLowering::allocateSpecialInputSGPRs(
2938 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
2939 SIMachineFunctionInfo &Info) const {
2940 auto &ArgInfo = Info.getArgInfo();
2941 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
2942
2943 // TODO: Unify handling with private memory pointers.
2944 if (UserSGPRInfo.hasDispatchPtr())
2945 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.DispatchPtr);
2946
2947 if (UserSGPRInfo.hasQueuePtr())
2948 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.QueuePtr);
2949
2950 // Implicit arg ptr takes the place of the kernarg segment pointer. This is a
2951 // constant offset from the kernarg segment.
2952 if (Info.hasImplicitArgPtr())
2953 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.ImplicitArgPtr);
2954
2955 if (UserSGPRInfo.hasDispatchID())
2956 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.DispatchID);
2957
2958 // flat_scratch_init is not applicable for non-kernel functions.
2959
2960 if (Info.hasWorkGroupIDX())
2961 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDX);
2962
2963 if (Info.hasWorkGroupIDY())
2964 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDY);
2965
2966 if (Info.hasWorkGroupIDZ())
2967 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDZ);
2968
2969 if (Info.hasLDSKernelId())
2970 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.LDSKernelId);
2971}
2972
2973// Allocate special inputs passed in user SGPRs.
2974void SITargetLowering::allocateHSAUserSGPRs(CCState &CCInfo,
2975 MachineFunction &MF,
2976 const SIRegisterInfo &TRI,
2977 SIMachineFunctionInfo &Info) const {
2978 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
2979 if (UserSGPRInfo.hasImplicitBufferPtr()) {
2980 Register ImplicitBufferPtrReg = Info.addImplicitBufferPtr(TRI);
2981 MF.addLiveIn(PReg: ImplicitBufferPtrReg, RC: &AMDGPU::SGPR_64RegClass);
2982 CCInfo.AllocateReg(Reg: ImplicitBufferPtrReg);
2983 }
2984
2985 // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
2986 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
2987 Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
2988 MF.addLiveIn(PReg: PrivateSegmentBufferReg, RC: &AMDGPU::SGPR_128RegClass);
2989 CCInfo.AllocateReg(Reg: PrivateSegmentBufferReg);
2990 }
2991
2992 if (UserSGPRInfo.hasDispatchPtr()) {
2993 Register DispatchPtrReg = Info.addDispatchPtr(TRI);
2994 MF.addLiveIn(PReg: DispatchPtrReg, RC: &AMDGPU::SGPR_64RegClass);
2995 CCInfo.AllocateReg(Reg: DispatchPtrReg);
2996 }
2997
2998 if (UserSGPRInfo.hasQueuePtr()) {
2999 Register QueuePtrReg = Info.addQueuePtr(TRI);
3000 MF.addLiveIn(PReg: QueuePtrReg, RC: &AMDGPU::SGPR_64RegClass);
3001 CCInfo.AllocateReg(Reg: QueuePtrReg);
3002 }
3003
3004 if (UserSGPRInfo.hasKernargSegmentPtr()) {
3005 MachineRegisterInfo &MRI = MF.getRegInfo();
3006 Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
3007 CCInfo.AllocateReg(Reg: InputPtrReg);
3008
3009 Register VReg = MF.addLiveIn(PReg: InputPtrReg, RC: &AMDGPU::SGPR_64RegClass);
3010 MRI.setType(VReg, Ty: LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64));
3011 }
3012
3013 if (UserSGPRInfo.hasDispatchID()) {
3014 Register DispatchIDReg = Info.addDispatchID(TRI);
3015 MF.addLiveIn(PReg: DispatchIDReg, RC: &AMDGPU::SGPR_64RegClass);
3016 CCInfo.AllocateReg(Reg: DispatchIDReg);
3017 }
3018
3019 if (UserSGPRInfo.hasFlatScratchInit() && !getSubtarget()->isAmdPalOS()) {
3020 Register FlatScratchInitReg = Info.addFlatScratchInit(TRI);
3021 MF.addLiveIn(PReg: FlatScratchInitReg, RC: &AMDGPU::SGPR_64RegClass);
3022 CCInfo.AllocateReg(Reg: FlatScratchInitReg);
3023 }
3024
3025 if (UserSGPRInfo.hasPrivateSegmentSize()) {
3026 Register PrivateSegmentSizeReg = Info.addPrivateSegmentSize(TRI);
3027 MF.addLiveIn(PReg: PrivateSegmentSizeReg, RC: &AMDGPU::SGPR_32RegClass);
3028 CCInfo.AllocateReg(Reg: PrivateSegmentSizeReg);
3029 }
3030
3031 // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
3032 // these from the dispatch pointer.
3033}
3034
3035// Allocate pre-loaded kernel arguemtns. Arguments to be preloading must be
3036// sequential starting from the first argument.
3037void SITargetLowering::allocatePreloadKernArgSGPRs(
3038 CCState &CCInfo, SmallVectorImpl<CCValAssign> &ArgLocs,
3039 const SmallVectorImpl<ISD::InputArg> &Ins, MachineFunction &MF,
3040 const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const {
3041 Function &F = MF.getFunction();
3042 unsigned LastExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
3043 GCNUserSGPRUsageInfo &SGPRInfo = Info.getUserSGPRInfo();
3044 bool InPreloadSequence = true;
3045 unsigned InIdx = 0;
3046 bool AlignedForImplictArgs = false;
3047 unsigned ImplicitArgOffset = 0;
3048 for (auto &Arg : F.args()) {
3049 if (!InPreloadSequence || !Arg.hasInRegAttr())
3050 break;
3051
3052 unsigned ArgIdx = Arg.getArgNo();
3053 // Don't preload non-original args or parts not in the current preload
3054 // sequence.
3055 if (InIdx < Ins.size() &&
3056 (!Ins[InIdx].isOrigArg() || Ins[InIdx].getOrigArgIndex() != ArgIdx))
3057 break;
3058
3059 for (; InIdx < Ins.size() && Ins[InIdx].isOrigArg() &&
3060 Ins[InIdx].getOrigArgIndex() == ArgIdx;
3061 InIdx++) {
3062 assert(ArgLocs[ArgIdx].isMemLoc());
3063 auto &ArgLoc = ArgLocs[InIdx];
3064 const Align KernelArgBaseAlign = Align(16);
3065 unsigned ArgOffset = ArgLoc.getLocMemOffset();
3066 Align Alignment = commonAlignment(A: KernelArgBaseAlign, Offset: ArgOffset);
3067 unsigned NumAllocSGPRs =
3068 alignTo(Value: ArgLoc.getLocVT().getFixedSizeInBits(), Align: 32) / 32;
3069
3070 // Fix alignment for hidden arguments.
3071 if (Arg.hasAttribute(Kind: "amdgpu-hidden-argument")) {
3072 if (!AlignedForImplictArgs) {
3073 ImplicitArgOffset =
3074 alignTo(Size: LastExplicitArgOffset,
3075 A: Subtarget->getAlignmentForImplicitArgPtr()) -
3076 LastExplicitArgOffset;
3077 AlignedForImplictArgs = true;
3078 }
3079 ArgOffset += ImplicitArgOffset;
3080 }
3081
3082 // Arg is preloaded into the previous SGPR.
3083 if (ArgLoc.getLocVT().getStoreSize() < 4 && Alignment < 4) {
3084 assert(InIdx >= 1 && "No previous SGPR");
3085 Info.getArgInfo().PreloadKernArgs[InIdx].Regs.push_back(
3086 Elt: Info.getArgInfo().PreloadKernArgs[InIdx - 1].Regs[0]);
3087 continue;
3088 }
3089
3090 unsigned Padding = ArgOffset - LastExplicitArgOffset;
3091 unsigned PaddingSGPRs = alignTo(Value: Padding, Align: 4) / 4;
3092 // Check for free user SGPRs for preloading.
3093 if (PaddingSGPRs + NumAllocSGPRs > SGPRInfo.getNumFreeUserSGPRs()) {
3094 InPreloadSequence = false;
3095 break;
3096 }
3097
3098 // Preload this argument.
3099 const TargetRegisterClass *RC =
3100 TRI.getSGPRClassForBitWidth(BitWidth: NumAllocSGPRs * 32);
3101 SmallVectorImpl<MCRegister> *PreloadRegs =
3102 Info.addPreloadedKernArg(TRI, RC, AllocSizeDWord: NumAllocSGPRs, KernArgIdx: InIdx, PaddingSGPRs);
3103
3104 if (PreloadRegs->size() > 1)
3105 RC = &AMDGPU::SGPR_32RegClass;
3106 for (auto &Reg : *PreloadRegs) {
3107 assert(Reg);
3108 MF.addLiveIn(PReg: Reg, RC);
3109 CCInfo.AllocateReg(Reg);
3110 }
3111
3112 LastExplicitArgOffset = NumAllocSGPRs * 4 + ArgOffset;
3113 }
3114 }
3115}
3116
3117void SITargetLowering::allocateLDSKernelId(CCState &CCInfo, MachineFunction &MF,
3118 const SIRegisterInfo &TRI,
3119 SIMachineFunctionInfo &Info) const {
3120 // Always allocate this last since it is a synthetic preload.
3121 if (Info.hasLDSKernelId()) {
3122 Register Reg = Info.addLDSKernelId();
3123 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3124 CCInfo.AllocateReg(Reg);
3125 }
3126}
3127
3128// Allocate special input registers that are initialized per-wave.
3129void SITargetLowering::allocateSystemSGPRs(CCState &CCInfo, MachineFunction &MF,
3130 SIMachineFunctionInfo &Info,
3131 CallingConv::ID CallConv,
3132 bool IsShader) const {
3133 bool HasArchitectedSGPRs = Subtarget->hasArchitectedSGPRs();
3134 if (Subtarget->hasUserSGPRInit16BugInWave32() && !IsShader) {
3135 // Note: user SGPRs are handled by the front-end for graphics shaders
3136 // Pad up the used user SGPRs with dead inputs.
3137
3138 // TODO: NumRequiredSystemSGPRs computation should be adjusted appropriately
3139 // before enabling architected SGPRs for workgroup IDs.
3140 assert(!HasArchitectedSGPRs && "Unhandled feature for the subtarget");
3141
3142 unsigned CurrentUserSGPRs = Info.getNumUserSGPRs();
3143 // Note we do not count the PrivateSegmentWaveByteOffset. We do not want to
3144 // rely on it to reach 16 since if we end up having no stack usage, it will
3145 // not really be added.
3146 unsigned NumRequiredSystemSGPRs =
3147 Info.hasWorkGroupIDX() + Info.hasWorkGroupIDY() +
3148 Info.hasWorkGroupIDZ() + Info.hasWorkGroupInfo();
3149 for (unsigned i = NumRequiredSystemSGPRs + CurrentUserSGPRs; i < 16; ++i) {
3150 Register Reg = Info.addReservedUserSGPR();
3151 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3152 CCInfo.AllocateReg(Reg);
3153 }
3154 }
3155
3156 if (!HasArchitectedSGPRs) {
3157 if (Info.hasWorkGroupIDX()) {
3158 Register Reg = Info.addWorkGroupIDX();
3159 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3160 CCInfo.AllocateReg(Reg);
3161 }
3162
3163 if (Info.hasWorkGroupIDY()) {
3164 Register Reg = Info.addWorkGroupIDY();
3165 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3166 CCInfo.AllocateReg(Reg);
3167 }
3168
3169 if (Info.hasWorkGroupIDZ()) {
3170 Register Reg = Info.addWorkGroupIDZ();
3171 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3172 CCInfo.AllocateReg(Reg);
3173 }
3174 }
3175
3176 if (Info.hasWorkGroupInfo()) {
3177 Register Reg = Info.addWorkGroupInfo();
3178 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3179 CCInfo.AllocateReg(Reg);
3180 }
3181
3182 if (Info.hasPrivateSegmentWaveByteOffset()) {
3183 // Scratch wave offset passed in system SGPR.
3184 unsigned PrivateSegmentWaveByteOffsetReg;
3185
3186 if (IsShader) {
3187 PrivateSegmentWaveByteOffsetReg =
3188 Info.getPrivateSegmentWaveByteOffsetSystemSGPR();
3189
3190 // This is true if the scratch wave byte offset doesn't have a fixed
3191 // location.
3192 if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) {
3193 PrivateSegmentWaveByteOffsetReg = findFirstFreeSGPR(CCInfo);
3194 Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
3195 }
3196 } else
3197 PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset();
3198
3199 MF.addLiveIn(PReg: PrivateSegmentWaveByteOffsetReg, RC: &AMDGPU::SGPR_32RegClass);
3200 CCInfo.AllocateReg(Reg: PrivateSegmentWaveByteOffsetReg);
3201 }
3202
3203 assert(!Subtarget->hasUserSGPRInit16BugInWave32() || IsShader ||
3204 Info.getNumPreloadedSGPRs() >= 16);
3205}
3206
3207static void reservePrivateMemoryRegs(const TargetMachine &TM,
3208 MachineFunction &MF,
3209 const SIRegisterInfo &TRI,
3210 SIMachineFunctionInfo &Info) {
3211 // Now that we've figured out where the scratch register inputs are, see if
3212 // should reserve the arguments and use them directly.
3213 MachineFrameInfo &MFI = MF.getFrameInfo();
3214 bool HasStackObjects = MFI.hasStackObjects();
3215 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
3216
3217 // Record that we know we have non-spill stack objects so we don't need to
3218 // check all stack objects later.
3219 if (HasStackObjects)
3220 Info.setHasNonSpillStackObjects(true);
3221
3222 // Everything live out of a block is spilled with fast regalloc, so it's
3223 // almost certain that spilling will be required.
3224 if (TM.getOptLevel() == CodeGenOptLevel::None)
3225 HasStackObjects = true;
3226
3227 // For now assume stack access is needed in any callee functions, so we need
3228 // the scratch registers to pass in.
3229 bool RequiresStackAccess = HasStackObjects || MFI.hasCalls();
3230
3231 if (!ST.hasFlatScratchEnabled()) {
3232 if (RequiresStackAccess && ST.isAmdHsaOrMesa(F: MF.getFunction())) {
3233 // If we have stack objects, we unquestionably need the private buffer
3234 // resource. For the Code Object V2 ABI, this will be the first 4 user
3235 // SGPR inputs. We can reserve those and use them directly.
3236
3237 Register PrivateSegmentBufferReg =
3238 Info.getPreloadedReg(Value: AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
3239 Info.setScratchRSrcReg(PrivateSegmentBufferReg);
3240 } else {
3241 unsigned ReservedBufferReg = TRI.reservedPrivateSegmentBufferReg(MF);
3242 // We tentatively reserve the last registers (skipping the last registers
3243 // which may contain VCC, FLAT_SCR, and XNACK). After register allocation,
3244 // we'll replace these with the ones immediately after those which were
3245 // really allocated. In the prologue copies will be inserted from the
3246 // argument to these reserved registers.
3247
3248 // Without HSA, relocations are used for the scratch pointer and the
3249 // buffer resource setup is always inserted in the prologue. Scratch wave
3250 // offset is still in an input SGPR.
3251 Info.setScratchRSrcReg(ReservedBufferReg);
3252 }
3253 }
3254
3255 MachineRegisterInfo &MRI = MF.getRegInfo();
3256
3257 // For entry functions we have to set up the stack pointer if we use it,
3258 // whereas non-entry functions get this "for free". This means there is no
3259 // intrinsic advantage to using S32 over S34 in cases where we do not have
3260 // calls but do need a frame pointer (i.e. if we are requested to have one
3261 // because frame pointer elimination is disabled). To keep things simple we
3262 // only ever use S32 as the call ABI stack pointer, and so using it does not
3263 // imply we need a separate frame pointer.
3264 //
3265 // Try to use s32 as the SP, but move it if it would interfere with input
3266 // arguments. This won't work with calls though.
3267 //
3268 // FIXME: Move SP to avoid any possible inputs, or find a way to spill input
3269 // registers.
3270 if (!MRI.isLiveIn(Reg: AMDGPU::SGPR32)) {
3271 Info.setStackPtrOffsetReg(AMDGPU::SGPR32);
3272 } else {
3273 assert(AMDGPU::isShader(MF.getFunction().getCallingConv()));
3274
3275 if (MFI.hasCalls())
3276 report_fatal_error(reason: "call in graphics shader with too many input SGPRs");
3277
3278 for (unsigned Reg : AMDGPU::SGPR_32RegClass) {
3279 if (!MRI.isLiveIn(Reg)) {
3280 Info.setStackPtrOffsetReg(Reg);
3281 break;
3282 }
3283 }
3284
3285 if (Info.getStackPtrOffsetReg() == AMDGPU::SP_REG)
3286 report_fatal_error(reason: "failed to find register for SP");
3287 }
3288
3289 // hasFP should be accurate for entry functions even before the frame is
3290 // finalized, because it does not rely on the known stack size, only
3291 // properties like whether variable sized objects are present.
3292 if (ST.getFrameLowering()->hasFP(MF)) {
3293 Info.setFrameOffsetReg(AMDGPU::SGPR33);
3294 }
3295}
3296
3297bool SITargetLowering::supportSplitCSR(MachineFunction *MF) const {
3298 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
3299 return !Info->isEntryFunction();
3300}
3301
3302void SITargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {}
3303
3304void SITargetLowering::insertCopiesSplitCSR(
3305 MachineBasicBlock *Entry,
3306 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
3307 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3308
3309 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(MF: Entry->getParent());
3310 if (!IStart)
3311 return;
3312
3313 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
3314 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
3315 MachineBasicBlock::iterator MBBI = Entry->begin();
3316 for (const MCPhysReg *I = IStart; *I; ++I) {
3317 const TargetRegisterClass *RC = nullptr;
3318 if (AMDGPU::SReg_64RegClass.contains(Reg: *I))
3319 RC = &AMDGPU::SGPR_64RegClass;
3320 else if (AMDGPU::SReg_32RegClass.contains(Reg: *I))
3321 RC = &AMDGPU::SGPR_32RegClass;
3322 else
3323 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3324
3325 Register NewVR = MRI->createVirtualRegister(RegClass: RC);
3326 // Create copy from CSR to a virtual register.
3327 Entry->addLiveIn(PhysReg: *I);
3328 BuildMI(BB&: *Entry, I: MBBI, MIMD: DebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewVR)
3329 .addReg(RegNo: *I);
3330
3331 // Insert the copy-back instructions right before the terminator.
3332 for (auto *Exit : Exits)
3333 BuildMI(BB&: *Exit, I: Exit->getFirstTerminator(), MIMD: DebugLoc(),
3334 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: *I)
3335 .addReg(RegNo: NewVR);
3336 }
3337}
3338
3339SDValue SITargetLowering::LowerFormalArguments(
3340 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3341 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3342 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3343 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3344
3345 MachineFunction &MF = DAG.getMachineFunction();
3346 const Function &Fn = MF.getFunction();
3347 FunctionType *FType = MF.getFunction().getFunctionType();
3348 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
3349 bool IsError = false;
3350
3351 if (Subtarget->isAmdHsaOS() && AMDGPU::isGraphics(CC: CallConv)) {
3352 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
3353 Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc()));
3354 IsError = true;
3355 }
3356
3357 SmallVector<ISD::InputArg, 16> Splits;
3358 SmallVector<CCValAssign, 16> ArgLocs;
3359 BitVector Skipped(Ins.size());
3360 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
3361 *DAG.getContext());
3362
3363 bool IsGraphics = AMDGPU::isGraphics(CC: CallConv);
3364 bool IsKernel = AMDGPU::isKernel(CC: CallConv);
3365 bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CC: CallConv);
3366
3367 if (IsGraphics) {
3368 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info->getUserSGPRInfo();
3369 assert(!UserSGPRInfo.hasDispatchPtr() &&
3370 !UserSGPRInfo.hasKernargSegmentPtr() && !Info->hasWorkGroupInfo() &&
3371 !Info->hasLDSKernelId() && !Info->hasWorkItemIDX() &&
3372 !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ());
3373 (void)UserSGPRInfo;
3374 if (!Subtarget->hasFlatScratchEnabled())
3375 assert(!UserSGPRInfo.hasFlatScratchInit());
3376 if ((CallConv != CallingConv::AMDGPU_CS &&
3377 CallConv != CallingConv::AMDGPU_Gfx &&
3378 CallConv != CallingConv::AMDGPU_Gfx_WholeWave) ||
3379 !Subtarget->hasArchitectedSGPRs())
3380 assert(!Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() &&
3381 !Info->hasWorkGroupIDZ());
3382 }
3383
3384 bool IsWholeWaveFunc = Info->isWholeWaveFunction();
3385
3386 if (CallConv == CallingConv::AMDGPU_PS) {
3387 processPSInputArgs(Splits, CallConv, Ins, Skipped, FType, Info);
3388
3389 // At least one interpolation mode must be enabled or else the GPU will
3390 // hang.
3391 //
3392 // Check PSInputAddr instead of PSInputEnable. The idea is that if the user
3393 // set PSInputAddr, the user wants to enable some bits after the compilation
3394 // based on run-time states. Since we can't know what the final PSInputEna
3395 // will look like, so we shouldn't do anything here and the user should take
3396 // responsibility for the correct programming.
3397 //
3398 // Otherwise, the following restrictions apply:
3399 // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
3400 // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
3401 // enabled too.
3402 if ((Info->getPSInputAddr() & 0x7F) == 0 ||
3403 ((Info->getPSInputAddr() & 0xF) == 0 && Info->isPSInputAllocated(Index: 11))) {
3404 CCInfo.AllocateReg(Reg: AMDGPU::VGPR0);
3405 CCInfo.AllocateReg(Reg: AMDGPU::VGPR1);
3406 Info->markPSInputAllocated(Index: 0);
3407 Info->markPSInputEnabled(Index: 0);
3408 }
3409 if (Subtarget->isAmdPalOS()) {
3410 // For isAmdPalOS, the user does not enable some bits after compilation
3411 // based on run-time states; the register values being generated here are
3412 // the final ones set in hardware. Therefore we need to apply the
3413 // workaround to PSInputAddr and PSInputEnable together. (The case where
3414 // a bit is set in PSInputAddr but not PSInputEnable is where the
3415 // frontend set up an input arg for a particular interpolation mode, but
3416 // nothing uses that input arg. Really we should have an earlier pass
3417 // that removes such an arg.)
3418 unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
3419 if ((PsInputBits & 0x7F) == 0 ||
3420 ((PsInputBits & 0xF) == 0 && (PsInputBits >> 11 & 1)))
3421 Info->markPSInputEnabled(Index: llvm::countr_zero(Val: Info->getPSInputAddr()));
3422 }
3423 } else if (IsKernel) {
3424 assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX());
3425 } else {
3426 Splits.append(in_start: IsWholeWaveFunc ? std::next(x: Ins.begin()) : Ins.begin(),
3427 in_end: Ins.end());
3428 }
3429
3430 if (IsKernel)
3431 analyzeFormalArgumentsCompute(State&: CCInfo, Ins);
3432
3433 if (IsEntryFunc) {
3434 allocateSpecialEntryInputVGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3435 allocateHSAUserSGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3436 if (IsKernel && Subtarget->hasKernargPreload())
3437 allocatePreloadKernArgSGPRs(CCInfo, ArgLocs, Ins, MF, TRI: *TRI, Info&: *Info);
3438
3439 allocateLDSKernelId(CCInfo, MF, TRI: *TRI, Info&: *Info);
3440 } else if (!IsGraphics) {
3441 // For the fixed ABI, pass workitem IDs in the last argument register.
3442 allocateSpecialInputVGPRsFixed(CCInfo, MF, TRI: *TRI, Info&: *Info);
3443
3444 // FIXME: Sink this into allocateSpecialInputSGPRs
3445 if (!Subtarget->hasFlatScratchEnabled())
3446 CCInfo.AllocateReg(Reg: Info->getScratchRSrcReg());
3447
3448 allocateSpecialInputSGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3449 }
3450
3451 if (!IsKernel) {
3452 CCAssignFn *AssignFn = CCAssignFnForCall(CC: CallConv, IsVarArg: isVarArg);
3453 CCInfo.AnalyzeFormalArguments(Ins: Splits, Fn: AssignFn);
3454
3455 // This assumes the registers are allocated by CCInfo in ascending order
3456 // with no gaps.
3457 Info->setNumWaveDispatchSGPRs(
3458 CCInfo.getFirstUnallocated(Regs: AMDGPU::SGPR_32RegClass.getRegisters()));
3459 Info->setNumWaveDispatchVGPRs(
3460 CCInfo.getFirstUnallocated(Regs: AMDGPU::VGPR_32RegClass.getRegisters()));
3461 } else if (Info->getNumKernargPreloadedSGPRs()) {
3462 Info->setNumWaveDispatchSGPRs(Info->getNumUserSGPRs());
3463 }
3464
3465 SmallVector<SDValue, 16> Chains;
3466
3467 if (IsWholeWaveFunc) {
3468 SDValue Setup = DAG.getNode(Opcode: AMDGPUISD::WHOLE_WAVE_SETUP, DL,
3469 ResultTys: {MVT::i1, MVT::Other}, Ops: Chain);
3470 InVals.push_back(Elt: Setup.getValue(R: 0));
3471 Chains.push_back(Elt: Setup.getValue(R: 1));
3472 }
3473
3474 // FIXME: This is the minimum kernel argument alignment. We should improve
3475 // this to the maximum alignment of the arguments.
3476 //
3477 // FIXME: Alignment of explicit arguments totally broken with non-0 explicit
3478 // kern arg offset.
3479 const Align KernelArgBaseAlign = Align(16);
3480
3481 for (unsigned i = IsWholeWaveFunc ? 1 : 0, e = Ins.size(), ArgIdx = 0; i != e;
3482 ++i) {
3483 const ISD::InputArg &Arg = Ins[i];
3484 if ((Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) || IsError) {
3485 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
3486 continue;
3487 }
3488
3489 CCValAssign &VA = ArgLocs[ArgIdx++];
3490 MVT VT = VA.getLocVT();
3491
3492 if (IsEntryFunc && VA.isMemLoc()) {
3493 VT = Ins[i].VT;
3494 EVT MemVT = VA.getLocVT();
3495
3496 const uint64_t Offset = VA.getLocMemOffset();
3497 Align Alignment = commonAlignment(A: KernelArgBaseAlign, Offset);
3498
3499 if (Arg.Flags.isByRef()) {
3500 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL: DL, Chain, Offset);
3501
3502 const GCNTargetMachine &TM =
3503 static_cast<const GCNTargetMachine &>(getTargetMachine());
3504 if (!TM.isNoopAddrSpaceCast(SrcAS: AMDGPUAS::CONSTANT_ADDRESS,
3505 DestAS: Arg.Flags.getPointerAddrSpace())) {
3506 Ptr = DAG.getAddrSpaceCast(dl: DL, VT, Ptr, SrcAS: AMDGPUAS::CONSTANT_ADDRESS,
3507 DestAS: Arg.Flags.getPointerAddrSpace());
3508 }
3509
3510 InVals.push_back(Elt: Ptr);
3511 continue;
3512 }
3513
3514 SDValue NewArg;
3515 if (Arg.isOrigArg() && Info->getArgInfo().PreloadKernArgs.count(Val: i)) {
3516 if (MemVT.getStoreSize() < 4 && Alignment < 4) {
3517 // In this case the argument is packed into the previous preload SGPR.
3518 int64_t AlignDownOffset = alignDown(Value: Offset, Align: 4);
3519 int64_t OffsetDiff = Offset - AlignDownOffset;
3520 EVT IntVT = MemVT.changeTypeToInteger();
3521
3522 const SIMachineFunctionInfo *Info =
3523 MF.getInfo<SIMachineFunctionInfo>();
3524 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
3525 Register Reg =
3526 Info->getArgInfo().PreloadKernArgs.find(Val: i)->getSecond().Regs[0];
3527
3528 assert(Reg);
3529 Register VReg = MRI.getLiveInVirtReg(PReg: Reg);
3530 SDValue Copy = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::i32);
3531
3532 SDValue ShiftAmt = DAG.getConstant(Val: OffsetDiff * 8, DL, VT: MVT::i32);
3533 SDValue Extract = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Copy, N2: ShiftAmt);
3534
3535 SDValue ArgVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: Extract);
3536 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MemVT, Operand: ArgVal);
3537 NewArg = convertArgType(DAG, VT, MemVT, SL: DL, Val: ArgVal,
3538 Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3539
3540 NewArg = DAG.getMergeValues(Ops: {NewArg, Copy.getValue(R: 1)}, dl: DL);
3541 } else {
3542 const SIMachineFunctionInfo *Info =
3543 MF.getInfo<SIMachineFunctionInfo>();
3544 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
3545 const SmallVectorImpl<MCRegister> &PreloadRegs =
3546 Info->getArgInfo().PreloadKernArgs.find(Val: i)->getSecond().Regs;
3547
3548 SDValue Copy;
3549 if (PreloadRegs.size() == 1) {
3550 Register VReg = MRI.getLiveInVirtReg(PReg: PreloadRegs[0]);
3551 const TargetRegisterClass *RC = MRI.getRegClass(Reg: VReg);
3552 NewArg = DAG.getCopyFromReg(
3553 Chain, dl: DL, Reg: VReg,
3554 VT: EVT::getIntegerVT(Context&: *DAG.getContext(),
3555 BitWidth: TRI->getRegSizeInBits(RC: *RC)));
3556
3557 } else {
3558 // If the kernarg alignment does not match the alignment of the SGPR
3559 // tuple RC that can accommodate this argument, it will be built up
3560 // via copies from from the individual SGPRs that the argument was
3561 // preloaded to.
3562 SmallVector<SDValue, 4> Elts;
3563 for (auto Reg : PreloadRegs) {
3564 Register VReg = MRI.getLiveInVirtReg(PReg: Reg);
3565 Copy = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::i32);
3566 Elts.push_back(Elt: Copy);
3567 }
3568 NewArg =
3569 DAG.getBuildVector(VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
3570 NumElements: PreloadRegs.size()),
3571 DL, Ops: Elts);
3572 }
3573
3574 // If the argument was preloaded to multiple consecutive 32-bit
3575 // registers because of misalignment between addressable SGPR tuples
3576 // and the argument size, we can still assume that because of kernarg
3577 // segment alignment restrictions that NewArg's size is the same as
3578 // MemVT and just do a bitcast. If MemVT is less than 32-bits we add a
3579 // truncate since we cannot preload to less than a single SGPR and the
3580 // MemVT may be smaller.
3581 EVT MemVTInt =
3582 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
3583 if (MemVT.bitsLT(VT: NewArg.getSimpleValueType()))
3584 NewArg = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MemVTInt, Operand: NewArg);
3585
3586 NewArg = DAG.getBitcast(VT: MemVT, V: NewArg);
3587 NewArg = convertArgType(DAG, VT, MemVT, SL: DL, Val: NewArg,
3588 Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3589 NewArg = DAG.getMergeValues(Ops: {NewArg, Chain}, dl: DL);
3590 }
3591 } else {
3592 // Hidden arguments that are in the kernel signature must be preloaded
3593 // to user SGPRs. Print a diagnostic error if a hidden argument is in
3594 // the argument list and is not preloaded.
3595 if (Arg.isOrigArg()) {
3596 Argument *OrigArg = Fn.getArg(i: Arg.getOrigArgIndex());
3597 if (OrigArg->hasAttribute(Kind: "amdgpu-hidden-argument")) {
3598 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
3599 *OrigArg->getParent(),
3600 "hidden argument in kernel signature was not preloaded",
3601 DL.getDebugLoc()));
3602 }
3603 }
3604
3605 NewArg =
3606 lowerKernargMemParameter(DAG, VT, MemVT, SL: DL, Chain, Offset,
3607 Alignment, Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3608 }
3609 Chains.push_back(Elt: NewArg.getValue(R: 1));
3610
3611 auto *ParamTy =
3612 dyn_cast<PointerType>(Val: FType->getParamType(i: Ins[i].getOrigArgIndex()));
3613 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS &&
3614 ParamTy &&
3615 (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
3616 ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS)) {
3617 // On SI local pointers are just offsets into LDS, so they are always
3618 // less than 16-bits. On CI and newer they could potentially be
3619 // real pointers, so we can't guarantee their size.
3620 NewArg = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: NewArg.getValueType(), N1: NewArg,
3621 N2: DAG.getValueType(MVT::i16));
3622 }
3623
3624 InVals.push_back(Elt: NewArg);
3625 continue;
3626 }
3627 if (!IsEntryFunc && VA.isMemLoc()) {
3628 SDValue Val = lowerStackParameter(DAG, VA, SL: DL, Chain, Arg);
3629 InVals.push_back(Elt: Val);
3630 if (!Arg.Flags.isByVal())
3631 Chains.push_back(Elt: Val.getValue(R: 1));
3632 continue;
3633 }
3634
3635 assert(VA.isRegLoc() && "Parameter must be in a register!");
3636
3637 Register Reg = VA.getLocReg();
3638 const TargetRegisterClass *RC = nullptr;
3639 if (AMDGPU::VGPR_32RegClass.contains(Reg))
3640 RC = &AMDGPU::VGPR_32RegClass;
3641 else if (AMDGPU::SGPR_32RegClass.contains(Reg))
3642 RC = &AMDGPU::SGPR_32RegClass;
3643 else
3644 llvm_unreachable("Unexpected register class in LowerFormalArguments!");
3645
3646 Reg = MF.addLiveIn(PReg: Reg, RC);
3647 SDValue Val = DAG.getCopyFromReg(Chain, dl: DL, Reg, VT);
3648 if (Arg.Flags.isInReg() && RC == &AMDGPU::VGPR_32RegClass) {
3649 // FIXME: Need to forward the chains created by `CopyFromReg`s, make sure
3650 // they will read physical regs before any side effect instructions.
3651 SDValue ReadFirstLane =
3652 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
3653 Val = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Val.getValueType(),
3654 N1: ReadFirstLane, N2: Val);
3655 }
3656
3657 if (Arg.Flags.isSRet()) {
3658 // The return object should be reasonably addressable.
3659
3660 // FIXME: This helps when the return is a real sret. If it is a
3661 // automatically inserted sret (i.e. CanLowerReturn returns false), an
3662 // extra copy is inserted in SelectionDAGBuilder which obscures this.
3663 unsigned NumBits =
3664 32 - getSubtarget()->getKnownHighZeroBitsForFrameIndex();
3665 Val = DAG.getNode(
3666 Opcode: ISD::AssertZext, DL, VT, N1: Val,
3667 N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits)));
3668 }
3669
3670 Val = convertABITypeToValueType(DAG, Val, VA, SL: DL);
3671 InVals.push_back(Elt: Val);
3672 }
3673
3674 // Start adding system SGPRs.
3675 if (IsEntryFunc)
3676 allocateSystemSGPRs(CCInfo, MF, Info&: *Info, CallConv, IsShader: IsGraphics);
3677
3678 unsigned StackArgSize = CCInfo.getStackSize();
3679 Info->setBytesInStackArgArea(StackArgSize);
3680
3681 return Chains.empty() ? Chain
3682 : DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
3683}
3684
3685// TODO: If return values can't fit in registers, we should return as many as
3686// possible in registers before passing on stack.
3687bool SITargetLowering::CanLowerReturn(
3688 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
3689 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
3690 const Type *RetTy) const {
3691 // Replacing returns with sret/stack usage doesn't make sense for shaders.
3692 // FIXME: Also sort of a workaround for custom vector splitting in LowerReturn
3693 // for shaders. Vector types should be explicitly handled by CC.
3694 if (AMDGPU::isEntryFunctionCC(CC: CallConv))
3695 return true;
3696
3697 SmallVector<CCValAssign, 16> RVLocs;
3698 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
3699 if (!CCInfo.CheckReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, IsVarArg)))
3700 return false;
3701
3702 // We must use the stack if return would require unavailable registers.
3703 unsigned MaxNumVGPRs = Subtarget->getMaxNumVGPRs(MF);
3704 unsigned TotalNumVGPRs = Subtarget->getAddressableNumArchVGPRs();
3705 for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i)
3706 if (CCInfo.isAllocated(Reg: AMDGPU::VGPR_32RegClass.getRegister(i)))
3707 return false;
3708
3709 return true;
3710}
3711
3712SDValue
3713SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
3714 bool isVarArg,
3715 const SmallVectorImpl<ISD::OutputArg> &Outs,
3716 const SmallVectorImpl<SDValue> &OutVals,
3717 const SDLoc &DL, SelectionDAG &DAG) const {
3718 MachineFunction &MF = DAG.getMachineFunction();
3719 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
3720 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3721
3722 if (AMDGPU::isKernel(CC: CallConv)) {
3723 return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs,
3724 OutVals, DL, DAG);
3725 }
3726
3727 bool IsShader = AMDGPU::isShader(CC: CallConv);
3728
3729 Info->setIfReturnsVoid(Outs.empty());
3730 bool IsWaveEnd = Info->returnsVoid() && IsShader;
3731
3732 // CCValAssign - represent the assignment of the return value to a location.
3733 SmallVector<CCValAssign, 48> RVLocs;
3734
3735 // CCState - Info about the registers and stack slots.
3736 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
3737 *DAG.getContext());
3738
3739 // Analyze outgoing return values.
3740 CCInfo.AnalyzeReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, IsVarArg: isVarArg));
3741
3742 SDValue Glue;
3743 SmallVector<SDValue, 48> RetOps;
3744 RetOps.push_back(Elt: Chain); // Operand #0 = Chain (updated below)
3745
3746 SDValue ReadFirstLane =
3747 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
3748 // Copy the result values into the output registers.
3749 for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E;
3750 ++I, ++RealRVLocIdx) {
3751 CCValAssign &VA = RVLocs[I];
3752 assert(VA.isRegLoc() && "Can only return in registers!");
3753 // TODO: Partially return in registers if return values don't fit.
3754 SDValue Arg = OutVals[RealRVLocIdx];
3755
3756 // Copied from other backends.
3757 switch (VA.getLocInfo()) {
3758 case CCValAssign::Full:
3759 break;
3760 case CCValAssign::BCvt:
3761 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getLocVT(), Operand: Arg);
3762 break;
3763 case CCValAssign::SExt:
3764 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3765 break;
3766 case CCValAssign::ZExt:
3767 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3768 break;
3769 case CCValAssign::AExt:
3770 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3771 break;
3772 default:
3773 llvm_unreachable("Unknown loc info!");
3774 }
3775 if (TRI->isSGPRPhysReg(Reg: VA.getLocReg()))
3776 Arg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Arg.getValueType(),
3777 N1: ReadFirstLane, N2: Arg);
3778 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VA.getLocReg(), N: Arg, Glue);
3779 Glue = Chain.getValue(R: 1);
3780 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
3781 }
3782
3783 // FIXME: Does sret work properly?
3784 if (!Info->isEntryFunction()) {
3785 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
3786 const MCPhysReg *I =
3787 TRI->getCalleeSavedRegsViaCopy(MF: &DAG.getMachineFunction());
3788 if (I) {
3789 for (; *I; ++I) {
3790 if (AMDGPU::SReg_64RegClass.contains(Reg: *I))
3791 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::i64));
3792 else if (AMDGPU::SReg_32RegClass.contains(Reg: *I))
3793 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::i32));
3794 else
3795 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3796 }
3797 }
3798 }
3799
3800 // Update chain and glue.
3801 RetOps[0] = Chain;
3802 if (Glue.getNode())
3803 RetOps.push_back(Elt: Glue);
3804
3805 unsigned Opc = AMDGPUISD::ENDPGM;
3806 if (!IsWaveEnd)
3807 Opc = Info->isWholeWaveFunction() ? AMDGPUISD::WHOLE_WAVE_RETURN
3808 : IsShader ? AMDGPUISD::RETURN_TO_EPILOG
3809 : AMDGPUISD::RET_GLUE;
3810 return DAG.getNode(Opcode: Opc, DL, VT: MVT::Other, Ops: RetOps);
3811}
3812
3813SDValue SITargetLowering::LowerCallResult(
3814 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool IsVarArg,
3815 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3816 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn,
3817 SDValue ThisVal) const {
3818 CCAssignFn *RetCC = CCAssignFnForReturn(CC: CallConv, IsVarArg);
3819
3820 // Assign locations to each value returned by this call.
3821 SmallVector<CCValAssign, 16> RVLocs;
3822 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
3823 *DAG.getContext());
3824 CCInfo.AnalyzeCallResult(Ins, Fn: RetCC);
3825
3826 // Copy all of the result registers out of their specified physreg.
3827 for (CCValAssign VA : RVLocs) {
3828 SDValue Val;
3829
3830 if (VA.isRegLoc()) {
3831 Val =
3832 DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue: InGlue);
3833 Chain = Val.getValue(R: 1);
3834 InGlue = Val.getValue(R: 2);
3835 } else if (VA.isMemLoc()) {
3836 report_fatal_error(reason: "TODO: return values in memory");
3837 } else
3838 llvm_unreachable("unknown argument location type");
3839
3840 switch (VA.getLocInfo()) {
3841 case CCValAssign::Full:
3842 break;
3843 case CCValAssign::BCvt:
3844 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Val);
3845 break;
3846 case CCValAssign::ZExt:
3847 Val = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: VA.getLocVT(), N1: Val,
3848 N2: DAG.getValueType(VA.getValVT()));
3849 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
3850 break;
3851 case CCValAssign::SExt:
3852 Val = DAG.getNode(Opcode: ISD::AssertSext, DL, VT: VA.getLocVT(), N1: Val,
3853 N2: DAG.getValueType(VA.getValVT()));
3854 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
3855 break;
3856 case CCValAssign::AExt:
3857 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
3858 break;
3859 default:
3860 llvm_unreachable("Unknown loc info!");
3861 }
3862
3863 InVals.push_back(Elt: Val);
3864 }
3865
3866 return Chain;
3867}
3868
3869// Add code to pass special inputs required depending on used features separate
3870// from the explicit user arguments present in the IR.
3871void SITargetLowering::passSpecialInputs(
3872 CallLoweringInfo &CLI, CCState &CCInfo, const SIMachineFunctionInfo &Info,
3873 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
3874 SmallVectorImpl<SDValue> &MemOpChains, SDValue Chain) const {
3875 // If we don't have a call site, this was a call inserted by
3876 // legalization. These can never use special inputs.
3877 if (!CLI.CB)
3878 return;
3879
3880 SelectionDAG &DAG = CLI.DAG;
3881 const SDLoc &DL = CLI.DL;
3882 const Function &F = DAG.getMachineFunction().getFunction();
3883
3884 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
3885 const AMDGPUFunctionArgInfo &CallerArgInfo = Info.getArgInfo();
3886
3887 const AMDGPUFunctionArgInfo &CalleeArgInfo =
3888 AMDGPUFunctionArgInfo::FixedABIFunctionInfo;
3889
3890 // TODO: Unify with private memory register handling. This is complicated by
3891 // the fact that at least in kernels, the input argument is not necessarily
3892 // in the same location as the input.
3893 // clang-format off
3894 static constexpr std::pair<AMDGPUFunctionArgInfo::PreloadedValue,
3895 std::array<StringLiteral, 2>> ImplicitAttrs[] = {
3896 {AMDGPUFunctionArgInfo::DISPATCH_PTR, {"amdgpu-no-dispatch-ptr", ""}},
3897 {AMDGPUFunctionArgInfo::QUEUE_PTR, {"amdgpu-no-queue-ptr", ""}},
3898 {AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR, {"amdgpu-no-implicitarg-ptr", ""}},
3899 {AMDGPUFunctionArgInfo::DISPATCH_ID, {"amdgpu-no-dispatch-id", ""}},
3900 {AMDGPUFunctionArgInfo::WORKGROUP_ID_X, {"amdgpu-no-workgroup-id-x", "amdgpu-no-cluster-id-x"}},
3901 {AMDGPUFunctionArgInfo::WORKGROUP_ID_Y, {"amdgpu-no-workgroup-id-y", "amdgpu-no-cluster-id-y"}},
3902 {AMDGPUFunctionArgInfo::WORKGROUP_ID_Z, {"amdgpu-no-workgroup-id-z", "amdgpu-no-cluster-id-z"}},
3903 {AMDGPUFunctionArgInfo::LDS_KERNEL_ID, {"amdgpu-no-lds-kernel-id", ""}},
3904 };
3905 // clang-format on
3906
3907 for (auto [InputID, Attrs] : ImplicitAttrs) {
3908 // If the callee does not use the attribute value, skip copying the value.
3909 if (all_of(Range&: Attrs, P: [&](StringRef Attr) {
3910 return Attr.empty() || CLI.CB->hasFnAttr(Kind: Attr);
3911 }))
3912 continue;
3913
3914 const auto [OutgoingArg, ArgRC, ArgTy] =
3915 CalleeArgInfo.getPreloadedValue(Value: InputID);
3916 if (!OutgoingArg)
3917 continue;
3918
3919 const auto [IncomingArg, IncomingArgRC, Ty] =
3920 CallerArgInfo.getPreloadedValue(Value: InputID);
3921 assert(IncomingArgRC == ArgRC);
3922
3923 // All special arguments are ints for now.
3924 EVT ArgVT = TRI->getSpillSize(RC: *ArgRC) == 8 ? MVT::i64 : MVT::i32;
3925 SDValue InputReg;
3926
3927 if (IncomingArg) {
3928 InputReg = loadInputValue(DAG, RC: ArgRC, VT: ArgVT, SL: DL, Arg: *IncomingArg);
3929 } else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) {
3930 // The implicit arg ptr is special because it doesn't have a corresponding
3931 // input for kernels, and is computed from the kernarg segment pointer.
3932 InputReg = getImplicitArgPtr(DAG, SL: DL);
3933 } else if (InputID == AMDGPUFunctionArgInfo::LDS_KERNEL_ID) {
3934 std::optional<uint32_t> Id =
3935 AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F);
3936 if (Id.has_value()) {
3937 InputReg = DAG.getConstant(Val: *Id, DL, VT: ArgVT);
3938 } else {
3939 InputReg = DAG.getPOISON(VT: ArgVT);
3940 }
3941 } else {
3942 // We may have proven the input wasn't needed, although the ABI is
3943 // requiring it. We just need to allocate the register appropriately.
3944 InputReg = DAG.getPOISON(VT: ArgVT);
3945 }
3946
3947 if (OutgoingArg->isRegister()) {
3948 RegsToPass.emplace_back(Args: OutgoingArg->getRegister(), Args&: InputReg);
3949 if (!CCInfo.AllocateReg(Reg: OutgoingArg->getRegister()))
3950 report_fatal_error(reason: "failed to allocate implicit input argument");
3951 } else {
3952 unsigned SpecialArgOffset =
3953 CCInfo.AllocateStack(Size: ArgVT.getStoreSize(), Alignment: Align(4));
3954 SDValue ArgStore =
3955 storeStackInputValue(DAG, SL: DL, Chain, ArgVal: InputReg, Offset: SpecialArgOffset);
3956 MemOpChains.push_back(Elt: ArgStore);
3957 }
3958 }
3959
3960 // Pack workitem IDs into a single register or pass it as is if already
3961 // packed.
3962
3963 auto [OutgoingArg, ArgRC, Ty] =
3964 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_X);
3965 if (!OutgoingArg)
3966 std::tie(args&: OutgoingArg, args&: ArgRC, args&: Ty) =
3967 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
3968 if (!OutgoingArg)
3969 std::tie(args&: OutgoingArg, args&: ArgRC, args&: Ty) =
3970 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
3971 if (!OutgoingArg)
3972 return;
3973
3974 const ArgDescriptor *IncomingArgX = std::get<0>(
3975 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_X));
3976 const ArgDescriptor *IncomingArgY = std::get<0>(
3977 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Y));
3978 const ArgDescriptor *IncomingArgZ = std::get<0>(
3979 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Z));
3980
3981 SDValue InputReg;
3982 SDLoc SL;
3983
3984 const bool NeedWorkItemIDX = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-x");
3985 const bool NeedWorkItemIDY = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-y");
3986 const bool NeedWorkItemIDZ = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-z");
3987
3988 // If incoming ids are not packed we need to pack them.
3989 if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo.WorkItemIDX &&
3990 NeedWorkItemIDX) {
3991 if (Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 0) != 0) {
3992 InputReg = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgX);
3993 } else {
3994 InputReg = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
3995 }
3996 }
3997
3998 if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo.WorkItemIDY &&
3999 NeedWorkItemIDY && Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 1) != 0) {
4000 SDValue Y = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgY);
4001 Y = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Y,
4002 N2: DAG.getShiftAmountConstant(Val: 10, VT: MVT::i32, DL: SL));
4003 InputReg = InputReg.getNode()
4004 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: InputReg, N2: Y)
4005 : Y;
4006 }
4007
4008 if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo.WorkItemIDZ &&
4009 NeedWorkItemIDZ && Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 2) != 0) {
4010 SDValue Z = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgZ);
4011 Z = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Z,
4012 N2: DAG.getShiftAmountConstant(Val: 20, VT: MVT::i32, DL: SL));
4013 InputReg = InputReg.getNode()
4014 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: InputReg, N2: Z)
4015 : Z;
4016 }
4017
4018 if (!InputReg && (NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ)) {
4019 if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {
4020 // We're in a situation where the outgoing function requires the workitem
4021 // ID, but the calling function does not have it (e.g a graphics function
4022 // calling a C calling convention function). This is illegal, but we need
4023 // to produce something.
4024 InputReg = DAG.getPOISON(VT: MVT::i32);
4025 } else {
4026 // Workitem ids are already packed, any of present incoming arguments
4027 // will carry all required fields.
4028 ArgDescriptor IncomingArg =
4029 ArgDescriptor::createArg(Arg: IncomingArgX ? *IncomingArgX
4030 : IncomingArgY ? *IncomingArgY
4031 : *IncomingArgZ,
4032 Mask: ~0u);
4033 InputReg = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: IncomingArg);
4034 }
4035 }
4036
4037 if (OutgoingArg->isRegister()) {
4038 if (InputReg)
4039 RegsToPass.emplace_back(Args: OutgoingArg->getRegister(), Args&: InputReg);
4040
4041 CCInfo.AllocateReg(Reg: OutgoingArg->getRegister());
4042 } else {
4043 unsigned SpecialArgOffset = CCInfo.AllocateStack(Size: 4, Alignment: Align(4));
4044 if (InputReg) {
4045 SDValue ArgStore =
4046 storeStackInputValue(DAG, SL: DL, Chain, ArgVal: InputReg, Offset: SpecialArgOffset);
4047 MemOpChains.push_back(Elt: ArgStore);
4048 }
4049 }
4050}
4051
4052bool SITargetLowering::isEligibleForTailCallOptimization(
4053 SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
4054 const SmallVectorImpl<ISD::OutputArg> &Outs,
4055 const SmallVectorImpl<SDValue> &OutVals,
4056 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
4057 if (AMDGPU::isChainCC(CC: CalleeCC))
4058 return true;
4059
4060 if (!AMDGPU::mayTailCallThisCC(CC: CalleeCC))
4061 return false;
4062
4063 // For a divergent call target, we need to do a waterfall loop over the
4064 // possible callees which precludes us from using a simple jump.
4065 if (Callee->isDivergent())
4066 return false;
4067
4068 MachineFunction &MF = DAG.getMachineFunction();
4069 const Function &CallerF = MF.getFunction();
4070 CallingConv::ID CallerCC = CallerF.getCallingConv();
4071 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
4072 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
4073
4074 // Kernels aren't callable, and don't have a live in return address so it
4075 // doesn't make sense to do a tail call with entry functions.
4076 if (!CallerPreserved)
4077 return false;
4078
4079 bool CCMatch = CallerCC == CalleeCC;
4080
4081 if (DAG.getTarget().Options.GuaranteedTailCallOpt) {
4082 if (AMDGPU::canGuaranteeTCO(CC: CalleeCC) && CCMatch)
4083 return true;
4084 return false;
4085 }
4086
4087 // TODO: Can we handle var args?
4088 if (IsVarArg)
4089 return false;
4090
4091 for (const Argument &Arg : CallerF.args()) {
4092 if (Arg.hasByValAttr())
4093 return false;
4094 }
4095
4096 LLVMContext &Ctx = *DAG.getContext();
4097
4098 // Check that the call results are passed in the same way.
4099 if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C&: Ctx, Ins,
4100 CalleeFn: CCAssignFnForCall(CC: CalleeCC, IsVarArg),
4101 CallerFn: CCAssignFnForCall(CC: CallerCC, IsVarArg)))
4102 return false;
4103
4104 // The callee has to preserve all registers the caller needs to preserve.
4105 if (!CCMatch) {
4106 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
4107 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved))
4108 return false;
4109 }
4110
4111 // Nothing more to check if the callee is taking no arguments.
4112 if (Outs.empty())
4113 return true;
4114
4115 SmallVector<CCValAssign, 16> ArgLocs;
4116 CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx);
4117
4118 // FIXME: We are not allocating special input registers, so we will be
4119 // deciding based on incorrect register assignments.
4120 CCInfo.AnalyzeCallOperands(Outs, Fn: CCAssignFnForCall(CC: CalleeCC, IsVarArg));
4121
4122 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
4123 // If the stack arguments for this call do not fit into our own save area then
4124 // the call cannot be made tail.
4125 // TODO: Is this really necessary?
4126 if (CCInfo.getStackSize() > FuncInfo->getBytesInStackArgArea())
4127 return false;
4128
4129 for (const auto &[CCVA, ArgVal] : zip_equal(t&: ArgLocs, u: OutVals)) {
4130 // FIXME: What about inreg arguments that end up passed in memory?
4131 if (!CCVA.isRegLoc())
4132 continue;
4133
4134 // If we are passing an argument in an SGPR, and the value is divergent,
4135 // this call requires a waterfall loop.
4136 if (ArgVal->isDivergent() && TRI->isSGPRPhysReg(Reg: CCVA.getLocReg())) {
4137 LLVM_DEBUG(
4138 dbgs() << "Cannot tail call due to divergent outgoing argument in "
4139 << printReg(CCVA.getLocReg(), TRI) << '\n');
4140 return false;
4141 }
4142 }
4143
4144 const MachineRegisterInfo &MRI = MF.getRegInfo();
4145 return parametersInCSRMatch(MRI, CallerPreservedMask: CallerPreserved, ArgLocs, OutVals);
4146}
4147
4148bool SITargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
4149 if (!CI->isTailCall())
4150 return false;
4151
4152 const Function *ParentFn = CI->getFunction();
4153 if (AMDGPU::isEntryFunctionCC(CC: ParentFn->getCallingConv()))
4154 return false;
4155 return true;
4156}
4157
4158namespace {
4159// Chain calls have special arguments that we need to handle. These are
4160// tagging along at the end of the arguments list(s), after the SGPR and VGPR
4161// arguments (index 0 and 1 respectively).
4162enum ChainCallArgIdx {
4163 Exec = 2,
4164 Flags,
4165 NumVGPRs,
4166 FallbackExec,
4167 FallbackCallee
4168};
4169} // anonymous namespace
4170
4171// The wave scratch offset register is used as the global base pointer.
4172SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
4173 SmallVectorImpl<SDValue> &InVals) const {
4174 CallingConv::ID CallConv = CLI.CallConv;
4175 bool IsChainCallConv = AMDGPU::isChainCC(CC: CallConv);
4176
4177 SelectionDAG &DAG = CLI.DAG;
4178
4179 const SDLoc &DL = CLI.DL;
4180 SDValue Chain = CLI.Chain;
4181 SDValue Callee = CLI.Callee;
4182
4183 llvm::SmallVector<SDValue, 6> ChainCallSpecialArgs;
4184 bool UsesDynamicVGPRs = false;
4185 if (IsChainCallConv) {
4186 // The last arguments should be the value that we need to put in EXEC,
4187 // followed by the flags and any other arguments with special meanings.
4188 // Pop them out of CLI.Outs and CLI.OutVals before we do any processing so
4189 // we don't treat them like the "real" arguments.
4190 auto RequestedExecIt =
4191 llvm::find_if(Range&: CLI.Outs, P: [](const ISD::OutputArg &Arg) {
4192 return Arg.OrigArgIndex == 2;
4193 });
4194 assert(RequestedExecIt != CLI.Outs.end() && "No node for EXEC");
4195
4196 size_t SpecialArgsBeginIdx = RequestedExecIt - CLI.Outs.begin();
4197 CLI.OutVals.erase(CS: CLI.OutVals.begin() + SpecialArgsBeginIdx,
4198 CE: CLI.OutVals.end());
4199 CLI.Outs.erase(CS: RequestedExecIt, CE: CLI.Outs.end());
4200
4201 assert(CLI.Outs.back().OrigArgIndex < 2 &&
4202 "Haven't popped all the special args");
4203
4204 TargetLowering::ArgListEntry RequestedExecArg =
4205 CLI.Args[ChainCallArgIdx::Exec];
4206 if (!RequestedExecArg.Ty->isIntegerTy(Bitwidth: Subtarget->getWavefrontSize()))
4207 return lowerUnhandledCall(CLI, InVals, Reason: "Invalid value for EXEC");
4208
4209 // Convert constants into TargetConstants, so they become immediate operands
4210 // instead of being selected into S_MOV.
4211 auto PushNodeOrTargetConstant = [&](TargetLowering::ArgListEntry Arg) {
4212 if (const auto *ArgNode = dyn_cast<ConstantSDNode>(Val&: Arg.Node)) {
4213 ChainCallSpecialArgs.push_back(Elt: DAG.getTargetConstant(
4214 Val: ArgNode->getAPIntValue(), DL, VT: ArgNode->getValueType(ResNo: 0)));
4215 } else
4216 ChainCallSpecialArgs.push_back(Elt: Arg.Node);
4217 };
4218
4219 PushNodeOrTargetConstant(RequestedExecArg);
4220
4221 // Process any other special arguments depending on the value of the flags.
4222 TargetLowering::ArgListEntry Flags = CLI.Args[ChainCallArgIdx::Flags];
4223
4224 const APInt &FlagsValue = cast<ConstantSDNode>(Val&: Flags.Node)->getAPIntValue();
4225 if (FlagsValue.isZero()) {
4226 if (CLI.Args.size() > ChainCallArgIdx::Flags + 1)
4227 return lowerUnhandledCall(CLI, InVals,
4228 Reason: "no additional args allowed if flags == 0");
4229 } else if (FlagsValue.isOneBitSet(BitNo: 0)) {
4230 if (CLI.Args.size() != ChainCallArgIdx::FallbackCallee + 1) {
4231 return lowerUnhandledCall(CLI, InVals, Reason: "expected 3 additional args");
4232 }
4233
4234 if (!Subtarget->isWave32()) {
4235 return lowerUnhandledCall(
4236 CLI, InVals, Reason: "dynamic VGPR mode is only supported for wave32");
4237 }
4238
4239 UsesDynamicVGPRs = true;
4240 std::for_each(first: CLI.Args.begin() + ChainCallArgIdx::NumVGPRs,
4241 last: CLI.Args.end(), f: PushNodeOrTargetConstant);
4242 }
4243 }
4244
4245 SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
4246 SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
4247 SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
4248 bool &IsTailCall = CLI.IsTailCall;
4249 bool IsVarArg = CLI.IsVarArg;
4250 bool IsSibCall = false;
4251 MachineFunction &MF = DAG.getMachineFunction();
4252
4253 if (Callee.isUndef() || isNullConstant(V: Callee)) {
4254 if (!CLI.IsTailCall) {
4255 for (ISD::InputArg &Arg : CLI.Ins)
4256 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
4257 }
4258
4259 return Chain;
4260 }
4261
4262 if (IsVarArg) {
4263 return lowerUnhandledCall(CLI, InVals,
4264 Reason: "unsupported call to variadic function ");
4265 }
4266
4267 if (!CLI.CB)
4268 return lowerUnhandledCall(CLI, InVals, Reason: "unsupported libcall legalization");
4269
4270 if (IsTailCall && MF.getTarget().Options.GuaranteedTailCallOpt) {
4271 return lowerUnhandledCall(CLI, InVals,
4272 Reason: "unsupported required tail call to function ");
4273 }
4274
4275 if (IsTailCall) {
4276 IsTailCall = isEligibleForTailCallOptimization(Callee, CalleeCC: CallConv, IsVarArg,
4277 Outs, OutVals, Ins, DAG);
4278 if (!IsTailCall &&
4279 ((CLI.CB && CLI.CB->isMustTailCall()) || IsChainCallConv)) {
4280 report_fatal_error(reason: "failed to perform tail call elimination on a call "
4281 "site marked musttail or on llvm.amdgcn.cs.chain");
4282 }
4283
4284 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
4285
4286 // A sibling call is one where we're under the usual C ABI and not planning
4287 // to change that but can still do a tail call:
4288 if (!TailCallOpt && IsTailCall)
4289 IsSibCall = true;
4290
4291 if (IsTailCall)
4292 ++NumTailCalls;
4293 }
4294
4295 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
4296 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
4297 SmallVector<SDValue, 8> MemOpChains;
4298
4299 // Analyze operands of the call, assigning locations to each operand.
4300 SmallVector<CCValAssign, 16> ArgLocs;
4301 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
4302 CCAssignFn *AssignFn = CCAssignFnForCall(CC: CallConv, IsVarArg);
4303
4304 if (CallConv != CallingConv::AMDGPU_Gfx && !AMDGPU::isChainCC(CC: CallConv) &&
4305 CallConv != CallingConv::AMDGPU_Gfx_WholeWave) {
4306 // With a fixed ABI, allocate fixed registers before user arguments.
4307 passSpecialInputs(CLI, CCInfo, Info: *Info, RegsToPass, MemOpChains, Chain);
4308 }
4309
4310 // Mark the scratch resource descriptor as allocated so the CC analysis
4311 // does not assign user arguments to these registers, matching the callee.
4312 if (!Subtarget->hasFlatScratchEnabled())
4313 CCInfo.AllocateReg(Reg: Info->getScratchRSrcReg());
4314
4315 CCInfo.AnalyzeCallOperands(Outs, Fn: AssignFn);
4316
4317 // Get a count of how many bytes are to be pushed on the stack.
4318 unsigned NumBytes = CCInfo.getStackSize();
4319
4320 if (IsSibCall) {
4321 // Since we're not changing the ABI to make this a tail call, the memory
4322 // operands are already available in the caller's incoming argument space.
4323 NumBytes = 0;
4324 }
4325
4326 // FPDiff is the byte offset of the call's argument area from the callee's.
4327 // Stores to callee stack arguments will be placed in FixedStackSlots offset
4328 // by this amount for a tail call. In a sibling call it must be 0 because the
4329 // caller will deallocate the entire stack and the callee still expects its
4330 // arguments to begin at SP+0. Completely unused for non-tail calls.
4331 int32_t FPDiff = 0;
4332 MachineFrameInfo &MFI = MF.getFrameInfo();
4333 auto *TRI = Subtarget->getRegisterInfo();
4334
4335 // Adjust the stack pointer for the new arguments...
4336 // These operations are automatically eliminated by the prolog/epilog pass
4337 if (!IsSibCall)
4338 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL);
4339
4340 if (!IsSibCall || IsChainCallConv) {
4341 if (!Subtarget->hasFlatScratchEnabled()) {
4342 SmallVector<SDValue, 4> CopyFromChains;
4343
4344 // In the HSA case, this should be an identity copy.
4345 SDValue ScratchRSrcReg =
4346 DAG.getCopyFromReg(Chain, dl: DL, Reg: Info->getScratchRSrcReg(), VT: MVT::v4i32);
4347 RegsToPass.emplace_back(Args: IsChainCallConv
4348 ? AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51
4349 : AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3,
4350 Args&: ScratchRSrcReg);
4351 CopyFromChains.push_back(Elt: ScratchRSrcReg.getValue(R: 1));
4352 Chain = DAG.getTokenFactor(DL, Vals&: CopyFromChains);
4353 }
4354 }
4355
4356 const unsigned NumSpecialInputs = RegsToPass.size();
4357
4358 MVT PtrVT = MVT::i32;
4359
4360 // Walk the register/memloc assignments, inserting copies/loads.
4361 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4362 CCValAssign &VA = ArgLocs[i];
4363 SDValue Arg = OutVals[i];
4364
4365 // Promote the value if needed.
4366 switch (VA.getLocInfo()) {
4367 case CCValAssign::Full:
4368 break;
4369 case CCValAssign::BCvt:
4370 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getLocVT(), Operand: Arg);
4371 break;
4372 case CCValAssign::ZExt:
4373 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4374 break;
4375 case CCValAssign::SExt:
4376 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4377 break;
4378 case CCValAssign::AExt:
4379 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4380 break;
4381 case CCValAssign::FPExt:
4382 Arg = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4383 break;
4384 default:
4385 llvm_unreachable("Unknown loc info!");
4386 }
4387
4388 if (VA.isRegLoc()) {
4389 RegsToPass.push_back(Elt: std::pair(VA.getLocReg(), Arg));
4390 } else {
4391 assert(VA.isMemLoc());
4392
4393 SDValue DstAddr;
4394 MachinePointerInfo DstInfo;
4395
4396 unsigned LocMemOffset = VA.getLocMemOffset();
4397 int32_t Offset = LocMemOffset;
4398
4399 SDValue PtrOff = DAG.getConstant(Val: Offset, DL, VT: PtrVT);
4400 MaybeAlign Alignment;
4401
4402 if (IsTailCall) {
4403 ISD::ArgFlagsTy Flags = Outs[i].Flags;
4404 unsigned OpSize = Flags.isByVal() ? Flags.getByValSize()
4405 : VA.getValVT().getStoreSize();
4406
4407 // FIXME: We can have better than the minimum byval required alignment.
4408 Alignment =
4409 Flags.isByVal()
4410 ? Flags.getNonZeroByValAlign()
4411 : commonAlignment(A: Subtarget->getStackAlignment(), Offset);
4412
4413 Offset = Offset + FPDiff;
4414 int FI = MFI.CreateFixedObject(Size: OpSize, SPOffset: Offset, IsImmutable: true);
4415
4416 DstAddr = DAG.getFrameIndex(FI, VT: PtrVT);
4417 DstInfo = MachinePointerInfo::getFixedStack(MF, FI);
4418
4419 // Make sure any stack arguments overlapping with where we're storing
4420 // are loaded before this eventual operation. Otherwise they'll be
4421 // clobbered.
4422
4423 // FIXME: Why is this really necessary? This seems to just result in a
4424 // lot of code to copy the stack and write them back to the same
4425 // locations, which are supposed to be immutable?
4426 Chain = addTokenForArgument(Chain, DAG, MFI, ClobberedFI: FI);
4427 } else {
4428 // Stores to the argument stack area are relative to the stack pointer.
4429 SDValue SP = DAG.getCopyFromReg(Chain, dl: DL, Reg: Info->getStackPtrOffsetReg(),
4430 VT: MVT::i32);
4431 DstAddr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SP, N2: PtrOff);
4432 DstInfo = MachinePointerInfo::getStack(MF, Offset: LocMemOffset);
4433 Alignment =
4434 commonAlignment(A: Subtarget->getStackAlignment(), Offset: LocMemOffset);
4435 }
4436
4437 if (Outs[i].Flags.isByVal()) {
4438 SDValue SizeNode =
4439 DAG.getConstant(Val: Outs[i].Flags.getByValSize(), DL, VT: MVT::i32);
4440 SDValue Cpy =
4441 DAG.getMemcpy(Chain, dl: DL, Dst: DstAddr, Src: Arg, Size: SizeNode,
4442 Alignment: Outs[i].Flags.getNonZeroByValAlign(),
4443 /*isVol = */ false, /*AlwaysInline = */ true,
4444 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: DstInfo,
4445 SrcPtrInfo: MachinePointerInfo(AMDGPUAS::PRIVATE_ADDRESS));
4446
4447 MemOpChains.push_back(Elt: Cpy);
4448 } else {
4449 SDValue Store =
4450 DAG.getStore(Chain, dl: DL, Val: Arg, Ptr: DstAddr, PtrInfo: DstInfo, Alignment);
4451 MemOpChains.push_back(Elt: Store);
4452 }
4453 }
4454 }
4455
4456 if (!MemOpChains.empty())
4457 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
4458
4459 SDValue ReadFirstLaneID =
4460 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
4461
4462 SDValue TokenGlue;
4463 if (CLI.ConvergenceControlToken) {
4464 TokenGlue = DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL, VT: MVT::Glue,
4465 Operand: CLI.ConvergenceControlToken);
4466 }
4467
4468 // Build a sequence of copy-to-reg nodes chained together with token chain
4469 // and flag operands which copy the outgoing args into the appropriate regs.
4470 SDValue InGlue;
4471
4472 unsigned ArgIdx = 0;
4473 for (auto [Reg, Val] : RegsToPass) {
4474 if (ArgIdx++ >= NumSpecialInputs &&
4475 (IsChainCallConv || !Val->isDivergent()) && TRI->isSGPRPhysReg(Reg)) {
4476 // For chain calls, the inreg arguments are required to be
4477 // uniform. Speculatively Insert a readfirstlane in case we cannot prove
4478 // they are uniform.
4479 //
4480 // For other calls, if an inreg arguments is known to be uniform,
4481 // speculatively insert a readfirstlane in case it is in a VGPR.
4482 //
4483 // FIXME: We need to execute this in a waterfall loop if it is a divergent
4484 // value, so let that continue to produce invalid code.
4485
4486 SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Val});
4487 if (TokenGlue)
4488 ReadfirstlaneArgs.push_back(Elt: TokenGlue);
4489 Val = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Val.getValueType(),
4490 Ops: ReadfirstlaneArgs);
4491 }
4492
4493 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N: Val, Glue: InGlue);
4494 InGlue = Chain.getValue(R: 1);
4495 }
4496
4497 // We don't usually want to end the call-sequence here because we would tidy
4498 // the frame up *after* the call, however in the ABI-changing tail-call case
4499 // we've carefully laid out the parameters so that when sp is reset they'll be
4500 // in the correct location.
4501 if (IsTailCall && !IsSibCall) {
4502 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue: InGlue, DL);
4503 InGlue = Chain.getValue(R: 1);
4504 }
4505
4506 std::vector<SDValue> Ops({Chain});
4507
4508 // Add a redundant copy of the callee global which will not be legalized, as
4509 // we need direct access to the callee later.
4510 if (GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
4511 const GlobalValue *GV = GSD->getGlobal();
4512 Ops.push_back(x: Callee);
4513 Ops.push_back(x: DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i64));
4514 } else {
4515 if (IsTailCall) {
4516 // isEligibleForTailCallOptimization considered whether the call target is
4517 // divergent, but we may still end up with a uniform value in a VGPR.
4518 // Insert a readfirstlane just in case.
4519 SDValue ReadFirstLaneID =
4520 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
4521
4522 SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Callee});
4523 if (TokenGlue)
4524 ReadfirstlaneArgs.push_back(Elt: TokenGlue); // Wire up convergence token.
4525 Callee = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Callee.getValueType(),
4526 Ops: ReadfirstlaneArgs);
4527 }
4528
4529 Ops.push_back(x: Callee);
4530 Ops.push_back(x: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i64));
4531 }
4532
4533 if (IsTailCall) {
4534 // Each tail call may have to adjust the stack by a different amount, so
4535 // this information must travel along with the operation for eventual
4536 // consumption by emitEpilogue.
4537 Ops.push_back(x: DAG.getTargetConstant(Val: FPDiff, DL, VT: MVT::i32));
4538 }
4539
4540 if (IsChainCallConv)
4541 llvm::append_range(C&: Ops, R&: ChainCallSpecialArgs);
4542
4543 // Add argument registers to the end of the list so that they are known live
4544 // into the call.
4545 for (auto &[Reg, Val] : RegsToPass)
4546 Ops.push_back(x: DAG.getRegister(Reg, VT: Val.getValueType()));
4547
4548 // Add a register mask operand representing the call-preserved registers.
4549 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
4550 assert(Mask && "Missing call preserved mask for calling convention");
4551 Ops.push_back(x: DAG.getRegisterMask(RegMask: Mask));
4552
4553 if (SDValue Token = CLI.ConvergenceControlToken) {
4554 SmallVector<SDValue, 2> GlueOps;
4555 GlueOps.push_back(Elt: Token);
4556 if (InGlue)
4557 GlueOps.push_back(Elt: InGlue);
4558
4559 InGlue = SDValue(DAG.getMachineNode(Opcode: TargetOpcode::CONVERGENCECTRL_GLUE, dl: DL,
4560 VT: MVT::Glue, Ops: GlueOps),
4561 0);
4562 }
4563
4564 if (InGlue)
4565 Ops.push_back(x: InGlue);
4566
4567 // If we're doing a tall call, use a TC_RETURN here rather than an
4568 // actual call instruction.
4569 if (IsTailCall) {
4570 MFI.setHasTailCall();
4571 unsigned OPC = AMDGPUISD::TC_RETURN;
4572 switch (CallConv) {
4573 case CallingConv::AMDGPU_Gfx:
4574 OPC = AMDGPUISD::TC_RETURN_GFX;
4575 break;
4576 case CallingConv::AMDGPU_CS_Chain:
4577 case CallingConv::AMDGPU_CS_ChainPreserve:
4578 OPC = UsesDynamicVGPRs ? AMDGPUISD::TC_RETURN_CHAIN_DVGPR
4579 : AMDGPUISD::TC_RETURN_CHAIN;
4580 break;
4581 }
4582
4583 // If the caller is a whole wave function, we need to use a special opcode
4584 // so we can patch up EXEC.
4585 if (Info->isWholeWaveFunction())
4586 OPC = AMDGPUISD::TC_RETURN_GFX_WholeWave;
4587
4588 return DAG.getNode(Opcode: OPC, DL, VT: MVT::Other, Ops);
4589 }
4590
4591 // Returns a chain and a flag for retval copy to use.
4592 SDValue Call = DAG.getNode(Opcode: AMDGPUISD::CALL, DL, ResultTys: {MVT::Other, MVT::Glue}, Ops);
4593 Chain = Call.getValue(R: 0);
4594 InGlue = Call.getValue(R: 1);
4595
4596 uint64_t CalleePopBytes = NumBytes;
4597 Chain = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: CalleePopBytes, Glue: InGlue, DL);
4598 if (!Ins.empty())
4599 InGlue = Chain.getValue(R: 1);
4600
4601 // Handle result values, copying them out of physregs into vregs that we
4602 // return.
4603 return LowerCallResult(Chain, InGlue, CallConv, IsVarArg, Ins, DL, DAG,
4604 InVals, /*IsThisReturn=*/false, ThisVal: SDValue());
4605}
4606
4607// This is similar to the default implementation in ExpandDYNAMIC_STACKALLOC,
4608// except for:
4609// 1. Stack growth direction(default: downwards, AMDGPU: upwards), and
4610// 2. Scale size where, scale = wave-reduction(alloca-size) * wave-size
4611SDValue SITargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
4612 SelectionDAG &DAG) const {
4613 const MachineFunction &MF = DAG.getMachineFunction();
4614 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
4615
4616 SDLoc dl(Op);
4617 EVT VT = Op.getValueType();
4618 SDValue Chain = Op.getOperand(i: 0);
4619 Register SPReg = Info->getStackPtrOffsetReg();
4620
4621 // Chain the dynamic stack allocation so that it doesn't modify the stack
4622 // pointer when other instructions are using the stack.
4623 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
4624
4625 SDValue Size = Op.getOperand(i: 1);
4626 SDValue BaseAddr = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
4627 Align Alignment = cast<ConstantSDNode>(Val: Op.getOperand(i: 2))->getAlignValue();
4628
4629 const TargetFrameLowering *TFL = Subtarget->getFrameLowering();
4630 assert(TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp &&
4631 "Stack grows upwards for AMDGPU");
4632
4633 Chain = BaseAddr.getValue(R: 1);
4634 Align StackAlign = TFL->getStackAlign();
4635 if (Alignment > StackAlign) {
4636 uint64_t ScaledAlignment = Alignment.value()
4637 << Subtarget->getWavefrontSizeLog2();
4638 uint64_t StackAlignMask = ScaledAlignment - 1;
4639 SDValue TmpAddr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr,
4640 N2: DAG.getConstant(Val: StackAlignMask, DL: dl, VT));
4641 BaseAddr = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: TmpAddr,
4642 N2: DAG.getSignedConstant(Val: -ScaledAlignment, DL: dl, VT));
4643 }
4644
4645 assert(Size.getValueType() == MVT::i32 && "Size must be 32-bit");
4646 SDValue NewSP;
4647 if (isa<ConstantSDNode>(Val: Size)) {
4648 // For constant sized alloca, scale alloca size by wave-size
4649 SDValue ScaledSize = DAG.getNode(
4650 Opcode: ISD::SHL, DL: dl, VT, N1: Size,
4651 N2: DAG.getConstant(Val: Subtarget->getWavefrontSizeLog2(), DL: dl, VT: MVT::i32));
4652 NewSP = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr, N2: ScaledSize); // Value
4653 } else {
4654 // For dynamic sized alloca, perform wave-wide reduction to get max of
4655 // alloca size(divergent) and then scale it by wave-size
4656 SDValue WaveReduction =
4657 DAG.getTargetConstant(Val: Intrinsic::amdgcn_wave_reduce_umax, DL: dl, VT: MVT::i32);
4658 Size = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::i32, N1: WaveReduction,
4659 N2: Size, N3: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32));
4660 SDValue ScaledSize = DAG.getNode(
4661 Opcode: ISD::SHL, DL: dl, VT, N1: Size,
4662 N2: DAG.getConstant(Val: Subtarget->getWavefrontSizeLog2(), DL: dl, VT: MVT::i32));
4663 NewSP =
4664 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr, N2: ScaledSize); // Value in vgpr.
4665 SDValue ReadFirstLaneID =
4666 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: dl, VT: MVT::i32);
4667 NewSP = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::i32, N1: ReadFirstLaneID,
4668 N2: NewSP);
4669 }
4670
4671 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: NewSP); // Output chain
4672 SDValue CallSeqEnd = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
4673
4674 return DAG.getMergeValues(Ops: {BaseAddr, CallSeqEnd}, dl);
4675}
4676
4677SDValue SITargetLowering::LowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const {
4678 if (Op.getValueType() != MVT::i32)
4679 return Op; // Defer to cannot select error.
4680
4681 Register SP = getStackPointerRegisterToSaveRestore();
4682 SDLoc SL(Op);
4683
4684 SDValue CopyFromSP = DAG.getCopyFromReg(Chain: Op->getOperand(Num: 0), dl: SL, Reg: SP, VT: MVT::i32);
4685
4686 // Convert from wave uniform to swizzled vector address. This should protect
4687 // from any edge cases where the stacksave result isn't directly used with
4688 // stackrestore.
4689 SDValue VectorAddress =
4690 DAG.getNode(Opcode: AMDGPUISD::WAVE_ADDRESS, DL: SL, VT: MVT::i32, Operand: CopyFromSP);
4691 return DAG.getMergeValues(Ops: {VectorAddress, CopyFromSP.getValue(R: 1)}, dl: SL);
4692}
4693
4694SDValue SITargetLowering::lowerGET_ROUNDING(SDValue Op,
4695 SelectionDAG &DAG) const {
4696 SDLoc SL(Op);
4697 assert(Op.getValueType() == MVT::i32);
4698
4699 uint32_t BothRoundHwReg =
4700 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
4701 SDValue GetRoundBothImm = DAG.getTargetConstant(Val: BothRoundHwReg, DL: SL, VT: MVT::i32);
4702
4703 SDValue IntrinID =
4704 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_getreg, DL: SL, VT: MVT::i32);
4705 SDValue GetReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList: Op->getVTList(),
4706 N1: Op.getOperand(i: 0), N2: IntrinID, N3: GetRoundBothImm);
4707
4708 // There are two rounding modes, one for f32 and one for f64/f16. We only
4709 // report in the standard value range if both are the same.
4710 //
4711 // The raw values also differ from the expected FLT_ROUNDS values. Nearest
4712 // ties away from zero is not supported, and the other values are rotated by
4713 // 1.
4714 //
4715 // If the two rounding modes are not the same, report a target defined value.
4716
4717 // Mode register rounding mode fields:
4718 //
4719 // [1:0] Single-precision round mode.
4720 // [3:2] Double/Half-precision round mode.
4721 //
4722 // 0=nearest even; 1= +infinity; 2= -infinity, 3= toward zero.
4723 //
4724 // Hardware Spec
4725 // Toward-0 3 0
4726 // Nearest Even 0 1
4727 // +Inf 1 2
4728 // -Inf 2 3
4729 // NearestAway0 N/A 4
4730 //
4731 // We have to handle 16 permutations of a 4-bit value, so we create a 64-bit
4732 // table we can index by the raw hardware mode.
4733 //
4734 // (trunc (FltRoundConversionTable >> MODE.fp_round)) & 0xf
4735
4736 SDValue BitTable =
4737 DAG.getConstant(Val: AMDGPU::FltRoundConversionTable, DL: SL, VT: MVT::i64);
4738
4739 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4740 SDValue RoundModeTimesNumBits =
4741 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: GetReg, N2: Two);
4742
4743 // TODO: We could possibly avoid a 64-bit shift and use a simpler table if we
4744 // knew only one mode was demanded.
4745 SDValue TableValue =
4746 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i64, N1: BitTable, N2: RoundModeTimesNumBits);
4747 SDValue TruncTable = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: TableValue);
4748
4749 SDValue EntryMask = DAG.getConstant(Val: 0xf, DL: SL, VT: MVT::i32);
4750 SDValue TableEntry =
4751 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: TruncTable, N2: EntryMask);
4752
4753 // There's a gap in the 4-bit encoded table and actual enum values, so offset
4754 // if it's an extended value.
4755 SDValue Four = DAG.getConstant(Val: 4, DL: SL, VT: MVT::i32);
4756 SDValue IsStandardValue =
4757 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: TableEntry, RHS: Four, Cond: ISD::SETULT);
4758 SDValue EnumOffset = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: TableEntry, N2: Four);
4759 SDValue Result = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i32, N1: IsStandardValue,
4760 N2: TableEntry, N3: EnumOffset);
4761
4762 return DAG.getMergeValues(Ops: {Result, GetReg.getValue(R: 1)}, dl: SL);
4763}
4764
4765SDValue SITargetLowering::lowerSET_ROUNDING(SDValue Op,
4766 SelectionDAG &DAG) const {
4767 SDLoc SL(Op);
4768
4769 SDValue NewMode = Op.getOperand(i: 1);
4770 assert(NewMode.getValueType() == MVT::i32);
4771
4772 // Index a table of 4-bit entries mapping from the C FLT_ROUNDS values to the
4773 // hardware MODE.fp_round values.
4774 if (auto *ConstMode = dyn_cast<ConstantSDNode>(Val&: NewMode)) {
4775 uint32_t ClampedVal = std::min(
4776 a: static_cast<uint32_t>(ConstMode->getZExtValue()),
4777 b: static_cast<uint32_t>(AMDGPU::TowardZeroF32_TowardNegativeF64));
4778 NewMode = DAG.getConstant(
4779 Val: AMDGPU::decodeFltRoundToHWConversionTable(FltRounds: ClampedVal), DL: SL, VT: MVT::i32);
4780 } else {
4781 // If we know the input can only be one of the supported standard modes in
4782 // the range 0-3, we can use a simplified mapping to hardware values.
4783 KnownBits KB = DAG.computeKnownBits(Op: NewMode);
4784 const bool UseReducedTable = KB.countMinLeadingZeros() >= 30;
4785 // The supported standard values are 0-3. The extended values start at 8. We
4786 // need to offset by 4 if the value is in the extended range.
4787
4788 if (UseReducedTable) {
4789 // Truncate to the low 32-bits.
4790 SDValue BitTable = DAG.getConstant(
4791 Val: AMDGPU::FltRoundToHWConversionTable & 0xffff, DL: SL, VT: MVT::i32);
4792
4793 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4794 SDValue RoundModeTimesNumBits =
4795 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: NewMode, N2: Two);
4796
4797 NewMode =
4798 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: BitTable, N2: RoundModeTimesNumBits);
4799
4800 // TODO: SimplifyDemandedBits on the setreg source here can likely reduce
4801 // the table extracted bits into inline immediates.
4802 } else {
4803 // table_index = umin(value, value - 4)
4804 // MODE.fp_round = (bit_table >> (table_index << 2)) & 0xf
4805 SDValue BitTable =
4806 DAG.getConstant(Val: AMDGPU::FltRoundToHWConversionTable, DL: SL, VT: MVT::i64);
4807
4808 SDValue Four = DAG.getConstant(Val: 4, DL: SL, VT: MVT::i32);
4809 SDValue OffsetEnum = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: NewMode, N2: Four);
4810 SDValue IndexVal =
4811 DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewMode, N2: OffsetEnum);
4812
4813 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4814 SDValue RoundModeTimesNumBits =
4815 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: IndexVal, N2: Two);
4816
4817 SDValue TableValue =
4818 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i64, N1: BitTable, N2: RoundModeTimesNumBits);
4819 SDValue TruncTable = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: TableValue);
4820
4821 // No need to mask out the high bits since the setreg will ignore them
4822 // anyway.
4823 NewMode = TruncTable;
4824 }
4825
4826 // Insert a readfirstlane in case the value is a VGPR. We could do this
4827 // earlier and keep more operations scalar, but that interferes with
4828 // combining the source.
4829 SDValue ReadFirstLaneID =
4830 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: SL, VT: MVT::i32);
4831 NewMode = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
4832 N1: ReadFirstLaneID, N2: NewMode);
4833 }
4834
4835 // N.B. The setreg will be later folded into s_round_mode on supported
4836 // targets.
4837 SDValue IntrinID =
4838 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_setreg, DL: SL, VT: MVT::i32);
4839 uint32_t BothRoundHwReg =
4840 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
4841 SDValue RoundBothImm = DAG.getTargetConstant(Val: BothRoundHwReg, DL: SL, VT: MVT::i32);
4842
4843 SDValue SetReg =
4844 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VTList: Op->getVTList(), N1: Op.getOperand(i: 0),
4845 N2: IntrinID, N3: RoundBothImm, N4: NewMode);
4846
4847 return SetReg;
4848}
4849
4850SDValue SITargetLowering::lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const {
4851 if (Op->isDivergent() &&
4852 (!Subtarget->hasVmemPrefInsts() || !Op.getConstantOperandVal(i: 4)))
4853 // Cannot do I$ prefetch with divergent pointer.
4854 return SDValue();
4855
4856 switch (cast<MemSDNode>(Val&: Op)->getAddressSpace()) {
4857 case AMDGPUAS::FLAT_ADDRESS:
4858 case AMDGPUAS::GLOBAL_ADDRESS:
4859 case AMDGPUAS::CONSTANT_ADDRESS:
4860 break;
4861 case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
4862 if (Subtarget->hasSafeSmemPrefetch())
4863 break;
4864 [[fallthrough]];
4865 default:
4866 return SDValue();
4867 }
4868
4869 // I$ prefetch
4870 if (!Subtarget->hasSafeSmemPrefetch() && !Op.getConstantOperandVal(i: 4))
4871 return SDValue();
4872
4873 return Op;
4874}
4875
4876// Work around DAG legality rules only based on the result type.
4877SDValue SITargetLowering::lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
4878 bool IsStrict = Op.getOpcode() == ISD::STRICT_FP_EXTEND;
4879 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
4880 EVT SrcVT = Src.getValueType();
4881
4882 if (SrcVT.getScalarType() != MVT::bf16)
4883 return Op;
4884
4885 SDLoc SL(Op);
4886 SDValue BitCast =
4887 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: SrcVT.changeTypeToInteger(), Operand: Src);
4888
4889 EVT DstVT = Op.getValueType();
4890 if (IsStrict)
4891 llvm_unreachable("Need STRICT_BF16_TO_FP");
4892
4893 return DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SL, VT: DstVT, Operand: BitCast);
4894}
4895
4896SDValue SITargetLowering::lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const {
4897 SDLoc SL(Op);
4898 if (Op.getValueType() != MVT::i64)
4899 return Op;
4900
4901 uint32_t ModeHwReg =
4902 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 23);
4903 SDValue ModeHwRegImm = DAG.getTargetConstant(Val: ModeHwReg, DL: SL, VT: MVT::i32);
4904 uint32_t TrapHwReg =
4905 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: 0, Values: 5);
4906 SDValue TrapHwRegImm = DAG.getTargetConstant(Val: TrapHwReg, DL: SL, VT: MVT::i32);
4907
4908 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
4909 SDValue IntrinID =
4910 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_getreg, DL: SL, VT: MVT::i32);
4911 SDValue GetModeReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList,
4912 N1: Op.getOperand(i: 0), N2: IntrinID, N3: ModeHwRegImm);
4913 SDValue GetTrapReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList,
4914 N1: Op.getOperand(i: 0), N2: IntrinID, N3: TrapHwRegImm);
4915 SDValue TokenReg =
4916 DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: GetModeReg.getValue(R: 1),
4917 N2: GetTrapReg.getValue(R: 1));
4918
4919 SDValue CvtPtr =
4920 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: GetModeReg, N2: GetTrapReg);
4921 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
4922
4923 return DAG.getMergeValues(Ops: {Result, TokenReg}, dl: SL);
4924}
4925
4926SDValue SITargetLowering::lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const {
4927 SDLoc SL(Op);
4928 if (Op.getOperand(i: 1).getValueType() != MVT::i64)
4929 return Op;
4930
4931 SDValue Input = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
4932 SDValue NewModeReg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Input,
4933 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
4934 SDValue NewTrapReg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Input,
4935 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
4936
4937 SDValue ReadFirstLaneID =
4938 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: SL, VT: MVT::i32);
4939 NewModeReg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
4940 N1: ReadFirstLaneID, N2: NewModeReg);
4941 NewTrapReg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
4942 N1: ReadFirstLaneID, N2: NewTrapReg);
4943
4944 unsigned ModeHwReg =
4945 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 23);
4946 SDValue ModeHwRegImm = DAG.getTargetConstant(Val: ModeHwReg, DL: SL, VT: MVT::i32);
4947 unsigned TrapHwReg =
4948 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: 0, Values: 5);
4949 SDValue TrapHwRegImm = DAG.getTargetConstant(Val: TrapHwReg, DL: SL, VT: MVT::i32);
4950
4951 SDValue IntrinID =
4952 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_setreg, DL: SL, VT: MVT::i32);
4953 SDValue SetModeReg =
4954 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VT: MVT::Other, N1: Op.getOperand(i: 0),
4955 N2: IntrinID, N3: ModeHwRegImm, N4: NewModeReg);
4956 SDValue SetTrapReg =
4957 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VT: MVT::Other, N1: Op.getOperand(i: 0),
4958 N2: IntrinID, N3: TrapHwRegImm, N4: NewTrapReg);
4959 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: SetTrapReg, N2: SetModeReg);
4960}
4961
4962Register SITargetLowering::getRegisterByName(const char *RegName, LLT VT,
4963 const MachineFunction &MF) const {
4964 const Function &Fn = MF.getFunction();
4965
4966 Register Reg = StringSwitch<Register>(RegName)
4967 .Case(S: "m0", Value: AMDGPU::M0)
4968 .Case(S: "exec", Value: AMDGPU::EXEC)
4969 .Case(S: "exec_lo", Value: AMDGPU::EXEC_LO)
4970 .Case(S: "exec_hi", Value: AMDGPU::EXEC_HI)
4971 .Case(S: "flat_scratch", Value: AMDGPU::FLAT_SCR)
4972 .Case(S: "flat_scratch_lo", Value: AMDGPU::FLAT_SCR_LO)
4973 .Case(S: "flat_scratch_hi", Value: AMDGPU::FLAT_SCR_HI)
4974 .Default(Value: Register());
4975 if (!Reg)
4976 return Reg;
4977
4978 if (!Subtarget->hasFlatScrRegister() &&
4979 Subtarget->getRegisterInfo()->regsOverlap(RegA: Reg, RegB: AMDGPU::FLAT_SCR)) {
4980 Fn.getContext().emitError(ErrorStr: Twine("invalid register \"" + StringRef(RegName) +
4981 "\" for subtarget."));
4982 }
4983
4984 switch (Reg) {
4985 case AMDGPU::M0:
4986 case AMDGPU::EXEC_LO:
4987 case AMDGPU::EXEC_HI:
4988 case AMDGPU::FLAT_SCR_LO:
4989 case AMDGPU::FLAT_SCR_HI:
4990 if (VT.getSizeInBits() == 32)
4991 return Reg;
4992 break;
4993 case AMDGPU::EXEC:
4994 case AMDGPU::FLAT_SCR:
4995 if (VT.getSizeInBits() == 64)
4996 return Reg;
4997 break;
4998 default:
4999 llvm_unreachable("missing register type checking");
5000 }
5001
5002 report_fatal_error(
5003 reason: Twine("invalid type for register \"" + StringRef(RegName) + "\"."));
5004}
5005
5006// If kill is not the last instruction, split the block so kill is always a
5007// proper terminator.
5008MachineBasicBlock *
5009SITargetLowering::splitKillBlock(MachineInstr &MI,
5010 MachineBasicBlock *BB) const {
5011 MachineBasicBlock *SplitBB = BB->splitAt(SplitInst&: MI, /*UpdateLiveIns=*/true);
5012 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5013 MI.setDesc(TII->getKillTerminatorFromPseudo(Opcode: MI.getOpcode()));
5014 return SplitBB;
5015}
5016
5017// Split block \p MBB at \p MI, as to insert a loop. If \p InstInLoop is true,
5018// \p MI will be the only instruction in the loop body block. Otherwise, it will
5019// be the first instruction in the remainder block.
5020//
5021/// \returns { LoopBody, Remainder }
5022static std::pair<MachineBasicBlock *, MachineBasicBlock *>
5023splitBlockForLoop(MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop) {
5024 MachineFunction *MF = MBB.getParent();
5025 MachineBasicBlock::iterator I(&MI);
5026
5027 // To insert the loop we need to split the block. Move everything after this
5028 // point to a new block, and insert a new empty block between the two.
5029 MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
5030 MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
5031 MachineFunction::iterator MBBI(MBB);
5032 ++MBBI;
5033
5034 MF->insert(MBBI, MBB: LoopBB);
5035 MF->insert(MBBI, MBB: RemainderBB);
5036
5037 LoopBB->addSuccessor(Succ: LoopBB);
5038 LoopBB->addSuccessor(Succ: RemainderBB);
5039
5040 // Move the rest of the block into a new block.
5041 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
5042
5043 if (InstInLoop) {
5044 auto Next = std::next(x: I);
5045
5046 // Move instruction to loop body.
5047 LoopBB->splice(Where: LoopBB->begin(), Other: &MBB, From: I, To: Next);
5048
5049 // Move the rest of the block.
5050 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: Next, To: MBB.end());
5051 } else {
5052 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: I, To: MBB.end());
5053 }
5054
5055 MBB.addSuccessor(Succ: LoopBB);
5056
5057 return std::pair(LoopBB, RemainderBB);
5058}
5059
5060/// Insert \p MI into a BUNDLE with an S_WAITCNT 0 immediately following it.
5061void SITargetLowering::bundleInstWithWaitcnt(MachineInstr &MI) const {
5062 MachineBasicBlock *MBB = MI.getParent();
5063 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5064 auto I = MI.getIterator();
5065 auto E = std::next(x: I);
5066
5067 // clang-format off
5068 BuildMI(BB&: *MBB, I: E, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_WAITCNT))
5069 .addImm(Val: 0);
5070 // clang-format on
5071
5072 MIBundleBuilder Bundler(*MBB, I, E);
5073 finalizeBundle(MBB&: *MBB, FirstMI: Bundler.begin());
5074}
5075
5076MachineBasicBlock *
5077SITargetLowering::emitGWSMemViolTestLoop(MachineInstr &MI,
5078 MachineBasicBlock *BB) const {
5079 const DebugLoc &DL = MI.getDebugLoc();
5080
5081 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
5082
5083 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5084
5085 // Apparently kill flags are only valid if the def is in the same block?
5086 if (MachineOperand *Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::data0))
5087 Src->setIsKill(false);
5088
5089 auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, MBB&: *BB, InstInLoop: true);
5090
5091 MachineBasicBlock::iterator I = LoopBB->end();
5092
5093 const unsigned EncodedReg = AMDGPU::Hwreg::HwregEncoding::encode(
5094 Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: AMDGPU::Hwreg::OFFSET_MEM_VIOL, Values: 1);
5095
5096 // Clear TRAP_STS.MEM_VIOL
5097 BuildMI(BB&: *LoopBB, I: LoopBB->begin(), MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SETREG_IMM32_B32))
5098 .addImm(Val: 0)
5099 .addImm(Val: EncodedReg);
5100
5101 bundleInstWithWaitcnt(MI);
5102
5103 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5104
5105 // Load and check TRAP_STS.MEM_VIOL
5106 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: Reg)
5107 .addImm(Val: EncodedReg);
5108
5109 // FIXME: Do we need to use an isel pseudo that may clobber scc?
5110 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
5111 .addReg(RegNo: Reg, Flags: RegState::Kill)
5112 .addImm(Val: 0);
5113 // clang-format off
5114 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
5115 .addMBB(MBB: LoopBB);
5116 // clang-format on
5117
5118 return RemainderBB;
5119}
5120
5121// Do a v_movrels_b32 or v_movreld_b32 for each unique value of \p IdxReg in the
5122// wavefront. If the value is uniform and just happens to be in a VGPR, this
5123// will only do one iteration. In the worst case, this will loop 64 times.
5124//
5125// TODO: Just use v_readlane_b32 if we know the VGPR has a uniform value.
5126static MachineBasicBlock::iterator
5127emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI,
5128 MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
5129 const DebugLoc &DL, const MachineOperand &Idx,
5130 unsigned InitReg, unsigned ResultReg, unsigned PhiReg,
5131 unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode,
5132 Register &SGPRIdxReg) {
5133
5134 MachineFunction *MF = OrigBB.getParent();
5135 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5136 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5137 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
5138 MachineBasicBlock::iterator I = LoopBB.begin();
5139
5140 const TargetRegisterClass *BoolRC = TRI->getBoolRC();
5141 Register PhiExec = MRI.createVirtualRegister(RegClass: BoolRC);
5142 Register NewExec = MRI.createVirtualRegister(RegClass: BoolRC);
5143 Register CurrentIdxReg =
5144 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5145 Register CondReg = MRI.createVirtualRegister(RegClass: BoolRC);
5146
5147 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::PHI), DestReg: PhiReg)
5148 .addReg(RegNo: InitReg)
5149 .addMBB(MBB: &OrigBB)
5150 .addReg(RegNo: ResultReg)
5151 .addMBB(MBB: &LoopBB);
5152
5153 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::PHI), DestReg: PhiExec)
5154 .addReg(RegNo: InitSaveExecReg)
5155 .addMBB(MBB: &OrigBB)
5156 .addReg(RegNo: NewExec)
5157 .addMBB(MBB: &LoopBB);
5158
5159 // Read the next variant <- also loop target.
5160 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurrentIdxReg)
5161 .addReg(RegNo: Idx.getReg(), Flags: getUndefRegState(B: Idx.isUndef()));
5162
5163 // Compare the just read M0 value to all possible Idx values.
5164 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: CondReg)
5165 .addReg(RegNo: CurrentIdxReg)
5166 .addReg(RegNo: Idx.getReg(), Flags: {}, SubReg: Idx.getSubReg());
5167
5168 // Update EXEC, save the original EXEC value to VCC.
5169 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.AndSaveExecOpc), DestReg: NewExec)
5170 .addReg(RegNo: CondReg, Flags: RegState::Kill);
5171
5172 MRI.setSimpleHint(VReg: NewExec, PrefReg: CondReg);
5173
5174 if (UseGPRIdxMode) {
5175 if (Offset == 0) {
5176 SGPRIdxReg = CurrentIdxReg;
5177 } else {
5178 SGPRIdxReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
5179 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: SGPRIdxReg)
5180 .addReg(RegNo: CurrentIdxReg, Flags: RegState::Kill)
5181 .addImm(Val: Offset);
5182 }
5183 } else {
5184 // Move index from VCC into M0
5185 if (Offset == 0) {
5186 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
5187 .addReg(RegNo: CurrentIdxReg, Flags: RegState::Kill);
5188 } else {
5189 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: AMDGPU::M0)
5190 .addReg(RegNo: CurrentIdxReg, Flags: RegState::Kill)
5191 .addImm(Val: Offset);
5192 }
5193 }
5194
5195 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
5196 MachineInstr *InsertPt =
5197 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.XorTermOpc), DestReg: LMC.ExecReg)
5198 .addReg(RegNo: LMC.ExecReg)
5199 .addReg(RegNo: NewExec);
5200
5201 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
5202 // s_cbranch_scc0?
5203
5204 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
5205 // clang-format off
5206 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_EXECNZ))
5207 .addMBB(MBB: &LoopBB);
5208 // clang-format on
5209
5210 return InsertPt->getIterator();
5211}
5212
5213// This has slightly sub-optimal regalloc when the source vector is killed by
5214// the read. The register allocator does not understand that the kill is
5215// per-workitem, so is kept alive for the whole loop so we end up not re-using a
5216// subregister from it, using 1 more VGPR than necessary. This was saved when
5217// this was expanded after register allocation.
5218static MachineBasicBlock::iterator
5219loadM0FromVGPR(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI,
5220 unsigned InitResultReg, unsigned PhiReg, int Offset,
5221 bool UseGPRIdxMode, Register &SGPRIdxReg) {
5222 MachineFunction *MF = MBB.getParent();
5223 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5224 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5225 MachineRegisterInfo &MRI = MF->getRegInfo();
5226 const DebugLoc &DL = MI.getDebugLoc();
5227 MachineBasicBlock::iterator I(&MI);
5228
5229 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
5230 Register DstReg = MI.getOperand(i: 0).getReg();
5231 Register SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
5232 Register TmpExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
5233 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
5234
5235 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: TmpExec);
5236
5237 // Save the EXEC mask
5238 // clang-format off
5239 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: SaveExec)
5240 .addReg(RegNo: LMC.ExecReg);
5241 // clang-format on
5242
5243 auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, MBB, InstInLoop: false);
5244
5245 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5246
5247 auto InsPt = emitLoadM0FromVGPRLoop(TII, MRI, OrigBB&: MBB, LoopBB&: *LoopBB, DL, Idx: *Idx,
5248 InitReg: InitResultReg, ResultReg: DstReg, PhiReg, InitSaveExecReg: TmpExec,
5249 Offset, UseGPRIdxMode, SGPRIdxReg);
5250
5251 MachineBasicBlock *LandingPad = MF->CreateMachineBasicBlock();
5252 MachineFunction::iterator MBBI(LoopBB);
5253 ++MBBI;
5254 MF->insert(MBBI, MBB: LandingPad);
5255 LoopBB->removeSuccessor(Succ: RemainderBB);
5256 LandingPad->addSuccessor(Succ: RemainderBB);
5257 LoopBB->addSuccessor(Succ: LandingPad);
5258 MachineBasicBlock::iterator First = LandingPad->begin();
5259 // clang-format off
5260 BuildMI(BB&: *LandingPad, I: First, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
5261 .addReg(RegNo: SaveExec);
5262 // clang-format on
5263
5264 return InsPt;
5265}
5266
5267// Returns subreg index, offset
5268static std::pair<unsigned, int>
5269computeIndirectRegAndOffset(const SIRegisterInfo &TRI,
5270 const TargetRegisterClass *SuperRC, unsigned VecReg,
5271 int Offset) {
5272 int NumElts = TRI.getRegSizeInBits(RC: *SuperRC) / 32;
5273
5274 // Skip out of bounds offsets, or else we would end up using an undefined
5275 // register.
5276 if (Offset >= NumElts || Offset < 0)
5277 return std::pair(AMDGPU::sub0, Offset);
5278
5279 return std::pair(SIRegisterInfo::getSubRegFromChannel(Channel: Offset), 0);
5280}
5281
5282static void setM0ToIndexFromSGPR(const SIInstrInfo *TII,
5283 MachineRegisterInfo &MRI, MachineInstr &MI,
5284 int Offset) {
5285 MachineBasicBlock *MBB = MI.getParent();
5286 const DebugLoc &DL = MI.getDebugLoc();
5287 MachineBasicBlock::iterator I(&MI);
5288
5289 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5290
5291 assert(Idx->getReg() != AMDGPU::NoRegister);
5292
5293 if (Offset == 0) {
5294 // clang-format off
5295 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
5296 .add(MO: *Idx);
5297 // clang-format on
5298 } else {
5299 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: AMDGPU::M0)
5300 .add(MO: *Idx)
5301 .addImm(Val: Offset);
5302 }
5303}
5304
5305static Register getIndirectSGPRIdx(const SIInstrInfo *TII,
5306 MachineRegisterInfo &MRI, MachineInstr &MI,
5307 int Offset) {
5308 MachineBasicBlock *MBB = MI.getParent();
5309 const DebugLoc &DL = MI.getDebugLoc();
5310 MachineBasicBlock::iterator I(&MI);
5311
5312 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5313
5314 if (Offset == 0)
5315 return Idx->getReg();
5316
5317 Register Tmp = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5318 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: Tmp)
5319 .add(MO: *Idx)
5320 .addImm(Val: Offset);
5321 return Tmp;
5322}
5323
5324static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
5325 MachineBasicBlock &MBB,
5326 const GCNSubtarget &ST) {
5327 const SIInstrInfo *TII = ST.getInstrInfo();
5328 const SIRegisterInfo &TRI = TII->getRegisterInfo();
5329 MachineFunction *MF = MBB.getParent();
5330 MachineRegisterInfo &MRI = MF->getRegInfo();
5331
5332 Register Dst = MI.getOperand(i: 0).getReg();
5333 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5334 Register SrcReg = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src)->getReg();
5335 int Offset = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::offset)->getImm();
5336
5337 const TargetRegisterClass *VecRC = MRI.getRegClass(Reg: SrcReg);
5338 const TargetRegisterClass *IdxRC = MRI.getRegClass(Reg: Idx->getReg());
5339
5340 unsigned SubReg;
5341 std::tie(args&: SubReg, args&: Offset) =
5342 computeIndirectRegAndOffset(TRI, SuperRC: VecRC, VecReg: SrcReg, Offset);
5343
5344 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
5345
5346 // Check for a SGPR index.
5347 if (TII->getRegisterInfo().isSGPRClass(RC: IdxRC)) {
5348 MachineBasicBlock::iterator I(&MI);
5349 const DebugLoc &DL = MI.getDebugLoc();
5350
5351 if (UseGPRIdxMode) {
5352 // TODO: Look at the uses to avoid the copy. This may require rescheduling
5353 // to avoid interfering with other uses, so probably requires a new
5354 // optimization pass.
5355 Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
5356
5357 const MCInstrDesc &GPRIDXDesc =
5358 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: true);
5359 BuildMI(BB&: MBB, I, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5360 .addReg(RegNo: SrcReg)
5361 .addReg(RegNo: Idx)
5362 .addImm(Val: SubReg);
5363 } else {
5364 setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
5365
5366 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: Dst)
5367 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
5368 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5369 }
5370
5371 MI.eraseFromParent();
5372
5373 return &MBB;
5374 }
5375
5376 // Control flow needs to be inserted if indexing with a VGPR.
5377 const DebugLoc &DL = MI.getDebugLoc();
5378 MachineBasicBlock::iterator I(&MI);
5379
5380 Register PhiReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5381 Register InitReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5382
5383 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: InitReg);
5384
5385 Register SGPRIdxReg;
5386 auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitResultReg: InitReg, PhiReg, Offset,
5387 UseGPRIdxMode, SGPRIdxReg);
5388
5389 MachineBasicBlock *LoopBB = InsPt->getParent();
5390
5391 if (UseGPRIdxMode) {
5392 const MCInstrDesc &GPRIDXDesc =
5393 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: true);
5394
5395 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5396 .addReg(RegNo: SrcReg)
5397 .addReg(RegNo: SGPRIdxReg)
5398 .addImm(Val: SubReg);
5399 } else {
5400 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: Dst)
5401 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
5402 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5403 }
5404
5405 MI.eraseFromParent();
5406
5407 return LoopBB;
5408}
5409
5410static MachineBasicBlock *emitIndirectDst(MachineInstr &MI,
5411 MachineBasicBlock &MBB,
5412 const GCNSubtarget &ST) {
5413 const SIInstrInfo *TII = ST.getInstrInfo();
5414 const SIRegisterInfo &TRI = TII->getRegisterInfo();
5415 MachineFunction *MF = MBB.getParent();
5416 MachineRegisterInfo &MRI = MF->getRegInfo();
5417
5418 Register Dst = MI.getOperand(i: 0).getReg();
5419 const MachineOperand *SrcVec = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src);
5420 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5421 const MachineOperand *Val = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::val);
5422 int Offset = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::offset)->getImm();
5423 const TargetRegisterClass *VecRC = MRI.getRegClass(Reg: SrcVec->getReg());
5424 const TargetRegisterClass *IdxRC = MRI.getRegClass(Reg: Idx->getReg());
5425
5426 // This can be an immediate, but will be folded later.
5427 assert(Val->getReg());
5428
5429 unsigned SubReg;
5430 std::tie(args&: SubReg, args&: Offset) =
5431 computeIndirectRegAndOffset(TRI, SuperRC: VecRC, VecReg: SrcVec->getReg(), Offset);
5432 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
5433
5434 if (Idx->getReg() == AMDGPU::NoRegister) {
5435 MachineBasicBlock::iterator I(&MI);
5436 const DebugLoc &DL = MI.getDebugLoc();
5437
5438 assert(Offset == 0);
5439
5440 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: Dst)
5441 .add(MO: *SrcVec)
5442 .add(MO: *Val)
5443 .addImm(Val: SubReg);
5444
5445 MI.eraseFromParent();
5446 return &MBB;
5447 }
5448
5449 // Check for a SGPR index.
5450 if (TII->getRegisterInfo().isSGPRClass(RC: IdxRC)) {
5451 MachineBasicBlock::iterator I(&MI);
5452 const DebugLoc &DL = MI.getDebugLoc();
5453
5454 if (UseGPRIdxMode) {
5455 Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
5456
5457 const MCInstrDesc &GPRIDXDesc =
5458 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
5459 BuildMI(BB&: MBB, I, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5460 .addReg(RegNo: SrcVec->getReg())
5461 .add(MO: *Val)
5462 .addReg(RegNo: Idx)
5463 .addImm(Val: SubReg);
5464 } else {
5465 setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
5466
5467 const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
5468 VecSize: TRI.getRegSizeInBits(RC: *VecRC), EltSize: 32, IsSGPR: false);
5469 BuildMI(BB&: MBB, I, MIMD: DL, MCID: MovRelDesc, DestReg: Dst)
5470 .addReg(RegNo: SrcVec->getReg())
5471 .add(MO: *Val)
5472 .addImm(Val: SubReg);
5473 }
5474 MI.eraseFromParent();
5475 return &MBB;
5476 }
5477
5478 // Control flow needs to be inserted if indexing with a VGPR.
5479 if (Val->isReg())
5480 MRI.clearKillFlags(Reg: Val->getReg());
5481
5482 const DebugLoc &DL = MI.getDebugLoc();
5483
5484 Register PhiReg = MRI.createVirtualRegister(RegClass: VecRC);
5485
5486 Register SGPRIdxReg;
5487 auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitResultReg: SrcVec->getReg(), PhiReg, Offset,
5488 UseGPRIdxMode, SGPRIdxReg);
5489 MachineBasicBlock *LoopBB = InsPt->getParent();
5490
5491 if (UseGPRIdxMode) {
5492 const MCInstrDesc &GPRIDXDesc =
5493 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
5494
5495 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5496 .addReg(RegNo: PhiReg)
5497 .add(MO: *Val)
5498 .addReg(RegNo: SGPRIdxReg)
5499 .addImm(Val: SubReg);
5500 } else {
5501 const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
5502 VecSize: TRI.getRegSizeInBits(RC: *VecRC), EltSize: 32, IsSGPR: false);
5503 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: MovRelDesc, DestReg: Dst)
5504 .addReg(RegNo: PhiReg)
5505 .add(MO: *Val)
5506 .addImm(Val: SubReg);
5507 }
5508
5509 MI.eraseFromParent();
5510 return LoopBB;
5511}
5512
5513static MachineBasicBlock *Expand64BitScalarArithmetic(MachineInstr &MI,
5514 MachineBasicBlock *BB) {
5515 // For targets older than GFX12, we emit a sequence of 32-bit operations.
5516 // For GFX12, we emit s_add_u64 and s_sub_u64.
5517 MachineFunction *MF = BB->getParent();
5518 const SIInstrInfo *TII = MF->getSubtarget<GCNSubtarget>().getInstrInfo();
5519 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5520 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
5521 const DebugLoc &DL = MI.getDebugLoc();
5522 MachineOperand &Dest = MI.getOperand(i: 0);
5523 MachineOperand &Src0 = MI.getOperand(i: 1);
5524 MachineOperand &Src1 = MI.getOperand(i: 2);
5525 bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
5526 if (ST.hasScalarAddSub64()) {
5527 unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64;
5528 // clang-format off
5529 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest.getReg())
5530 .add(MO: Src0)
5531 .add(MO: Src1);
5532 // clang-format on
5533 } else {
5534 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5535 const TargetRegisterClass *BoolRC = TRI->getBoolRC();
5536
5537 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5538 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5539
5540 MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
5541 MI, MRI, SuperReg: Src0, SuperRC: BoolRC, SubIdx: AMDGPU::sub0, SubRC: &AMDGPU::SReg_32RegClass);
5542 MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
5543 MI, MRI, SuperReg: Src0, SuperRC: BoolRC, SubIdx: AMDGPU::sub1, SubRC: &AMDGPU::SReg_32RegClass);
5544
5545 MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
5546 MI, MRI, SuperReg: Src1, SuperRC: BoolRC, SubIdx: AMDGPU::sub0, SubRC: &AMDGPU::SReg_32RegClass);
5547 MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
5548 MI, MRI, SuperReg: Src1, SuperRC: BoolRC, SubIdx: AMDGPU::sub1, SubRC: &AMDGPU::SReg_32RegClass);
5549
5550 unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
5551 unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
5552 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoOpc), DestReg: DestSub0).add(MO: Src0Sub0).add(MO: Src1Sub0);
5553 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: HiOpc), DestReg: DestSub1).add(MO: Src0Sub1).add(MO: Src1Sub1);
5554 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest.getReg())
5555 .addReg(RegNo: DestSub0)
5556 .addImm(Val: AMDGPU::sub0)
5557 .addReg(RegNo: DestSub1)
5558 .addImm(Val: AMDGPU::sub1);
5559 }
5560 MI.eraseFromParent();
5561 return BB;
5562}
5563
5564static uint32_t getIdentityValueFor32BitWaveReduction(unsigned Opc) {
5565 switch (Opc) {
5566 case AMDGPU::S_MIN_U32:
5567 return std::numeric_limits<uint32_t>::max();
5568 case AMDGPU::S_MIN_I32:
5569 return std::numeric_limits<int32_t>::max();
5570 case AMDGPU::S_MAX_U32:
5571 return std::numeric_limits<uint32_t>::min();
5572 case AMDGPU::S_MAX_I32:
5573 return std::numeric_limits<int32_t>::min();
5574 case AMDGPU::V_ADD_F32_e64: // -0.0
5575 return 0x80000000;
5576 case AMDGPU::V_SUB_F32_e64: // +0.0
5577 return 0x0;
5578 case AMDGPU::S_ADD_I32:
5579 case AMDGPU::S_SUB_I32:
5580 case AMDGPU::S_OR_B32:
5581 case AMDGPU::S_XOR_B32:
5582 return std::numeric_limits<uint32_t>::min();
5583 case AMDGPU::S_AND_B32:
5584 return std::numeric_limits<uint32_t>::max();
5585 case AMDGPU::V_MIN_F32_e64:
5586 case AMDGPU::V_MAX_F32_e64:
5587 return 0x7fc00000; // qNAN
5588 default:
5589 llvm_unreachable(
5590 "Unexpected opcode in getIdentityValueFor32BitWaveReduction");
5591 }
5592}
5593
5594static uint64_t getIdentityValueFor64BitWaveReduction(unsigned Opc) {
5595 switch (Opc) {
5596 case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
5597 return std::numeric_limits<uint64_t>::max();
5598 case AMDGPU::V_CMP_LT_I64_e64: // min.i64
5599 return std::numeric_limits<int64_t>::max();
5600 case AMDGPU::V_CMP_GT_U64_e64: // umax.u64
5601 return std::numeric_limits<uint64_t>::min();
5602 case AMDGPU::V_CMP_GT_I64_e64: // max.i64
5603 return std::numeric_limits<int64_t>::min();
5604 case AMDGPU::V_MIN_F64_e64:
5605 case AMDGPU::V_MAX_F64_e64:
5606 case AMDGPU::V_MIN_NUM_F64_e64:
5607 case AMDGPU::V_MAX_NUM_F64_e64:
5608 return 0x7FF8000000000000; // qNAN
5609 case AMDGPU::S_ADD_U64_PSEUDO:
5610 case AMDGPU::S_SUB_U64_PSEUDO:
5611 case AMDGPU::S_OR_B64:
5612 case AMDGPU::S_XOR_B64:
5613 return std::numeric_limits<uint64_t>::min();
5614 case AMDGPU::S_AND_B64:
5615 return std::numeric_limits<uint64_t>::max();
5616 case AMDGPU::V_ADD_F64_e64:
5617 case AMDGPU::V_ADD_F64_pseudo_e64:
5618 return 0x8000000000000000; // -0.0
5619 default:
5620 llvm_unreachable(
5621 "Unexpected opcode in getIdentityValueFor64BitWaveReduction");
5622 }
5623}
5624
5625static bool is32bitWaveReduceOperation(unsigned Opc) {
5626 return Opc == AMDGPU::S_MIN_U32 || Opc == AMDGPU::S_MIN_I32 ||
5627 Opc == AMDGPU::S_MAX_U32 || Opc == AMDGPU::S_MAX_I32 ||
5628 Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32 ||
5629 Opc == AMDGPU::S_AND_B32 || Opc == AMDGPU::S_OR_B32 ||
5630 Opc == AMDGPU::S_XOR_B32 || Opc == AMDGPU::V_MIN_F32_e64 ||
5631 Opc == AMDGPU::V_MAX_F32_e64 || Opc == AMDGPU::V_ADD_F32_e64 ||
5632 Opc == AMDGPU::V_SUB_F32_e64;
5633}
5634
5635static bool isFloatingPointWaveReduceOperation(unsigned Opc) {
5636 return Opc == AMDGPU::V_MIN_F32_e64 || Opc == AMDGPU::V_MAX_F32_e64 ||
5637 Opc == AMDGPU::V_ADD_F32_e64 || Opc == AMDGPU::V_SUB_F32_e64 ||
5638 Opc == AMDGPU::V_MIN_F64_e64 || Opc == AMDGPU::V_MAX_F64_e64 ||
5639 Opc == AMDGPU::V_MIN_NUM_F64_e64 || Opc == AMDGPU::V_MAX_NUM_F64_e64 ||
5640 Opc == AMDGPU::V_ADD_F64_e64 || Opc == AMDGPU::V_ADD_F64_pseudo_e64;
5641}
5642
5643static unsigned getDPPOpcForWaveReduction(unsigned Opc,
5644 const GCNSubtarget &ST) {
5645 switch (Opc) {
5646 case AMDGPU::S_MIN_U32:
5647 return AMDGPU::V_MIN_U32_dpp;
5648 case AMDGPU::S_MIN_I32:
5649 return AMDGPU::V_MIN_I32_dpp;
5650 case AMDGPU::S_MAX_U32:
5651 return AMDGPU::V_MAX_U32_dpp;
5652 case AMDGPU::S_MAX_I32:
5653 return AMDGPU::V_MAX_I32_dpp;
5654 case AMDGPU::S_ADD_I32:
5655 case AMDGPU::S_SUB_I32:
5656 return ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_dpp
5657 : AMDGPU::V_ADD_CO_U32_dpp;
5658 case AMDGPU::S_AND_B32:
5659 return AMDGPU::V_AND_B32_dpp;
5660 case AMDGPU::S_OR_B32:
5661 return AMDGPU::V_OR_B32_dpp;
5662 case AMDGPU::S_XOR_B32:
5663 return AMDGPU::V_XOR_B32_dpp;
5664 case AMDGPU::V_ADD_F32_e64:
5665 case AMDGPU::V_SUB_F32_e64:
5666 return AMDGPU::V_ADD_F32_dpp;
5667 case AMDGPU::V_MIN_F32_e64:
5668 return AMDGPU::V_MIN_F32_dpp;
5669 case AMDGPU::V_MAX_F32_e64:
5670 return AMDGPU::V_MAX_F32_dpp;
5671 default:
5672 llvm_unreachable("unhandled lane op");
5673 }
5674}
5675
5676static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
5677 MachineBasicBlock &BB,
5678 const GCNSubtarget &ST,
5679 unsigned Opc) {
5680 MachineRegisterInfo &MRI = BB.getParent()->getRegInfo();
5681 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5682 const DebugLoc &DL = MI.getDebugLoc();
5683 const SIInstrInfo *TII = ST.getInstrInfo();
5684
5685 // Reduction operations depend on whether the input operand is SGPR or VGPR.
5686 Register SrcReg = MI.getOperand(i: 1).getReg();
5687 bool isSGPR = TRI->isSGPRClass(RC: MRI.getRegClass(Reg: SrcReg));
5688 Register DstReg = MI.getOperand(i: 0).getReg();
5689 unsigned Stratergy = static_cast<unsigned>(MI.getOperand(i: 2).getImm());
5690 enum WAVE_REDUCE_STRATEGY : unsigned { DEFAULT = 0, ITERATIVE = 1, DPP = 2 };
5691 MachineBasicBlock *RetBB = nullptr;
5692 if (isSGPR) {
5693 switch (Opc) {
5694 case AMDGPU::S_MIN_U32:
5695 case AMDGPU::S_MIN_I32:
5696 case AMDGPU::V_MIN_F32_e64:
5697 case AMDGPU::S_MAX_U32:
5698 case AMDGPU::S_MAX_I32:
5699 case AMDGPU::V_MAX_F32_e64:
5700 case AMDGPU::S_AND_B32:
5701 case AMDGPU::S_OR_B32: {
5702 // Idempotent operations.
5703 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstReg).addReg(RegNo: SrcReg);
5704 RetBB = &BB;
5705 break;
5706 }
5707 case AMDGPU::V_CMP_LT_U64_e64: // umin
5708 case AMDGPU::V_CMP_LT_I64_e64: // min
5709 case AMDGPU::V_CMP_GT_U64_e64: // umax
5710 case AMDGPU::V_CMP_GT_I64_e64: // max
5711 case AMDGPU::V_MIN_F64_e64:
5712 case AMDGPU::V_MIN_NUM_F64_e64:
5713 case AMDGPU::V_MAX_F64_e64:
5714 case AMDGPU::V_MAX_NUM_F64_e64:
5715 case AMDGPU::S_AND_B64:
5716 case AMDGPU::S_OR_B64: {
5717 // Idempotent operations.
5718 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B64), DestReg: DstReg).addReg(RegNo: SrcReg);
5719 RetBB = &BB;
5720 break;
5721 }
5722 case AMDGPU::S_XOR_B32:
5723 case AMDGPU::S_XOR_B64:
5724 case AMDGPU::S_ADD_I32:
5725 case AMDGPU::S_ADD_U64_PSEUDO:
5726 case AMDGPU::V_ADD_F32_e64:
5727 case AMDGPU::V_ADD_F64_e64:
5728 case AMDGPU::V_ADD_F64_pseudo_e64:
5729 case AMDGPU::S_SUB_I32:
5730 case AMDGPU::S_SUB_U64_PSEUDO:
5731 case AMDGPU::V_SUB_F32_e64: {
5732 const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
5733 const TargetRegisterClass *DstRegClass = MRI.getRegClass(Reg: DstReg);
5734 Register ExecMask = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
5735 Register NumActiveLanes =
5736 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5737
5738 bool IsWave32 = ST.isWave32();
5739 unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
5740 MCRegister ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
5741 unsigned BitCountOpc =
5742 IsWave32 ? AMDGPU::S_BCNT1_I32_B32 : AMDGPU::S_BCNT1_I32_B64;
5743
5744 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MovOpc), DestReg: ExecMask).addReg(RegNo: ExecReg);
5745
5746 auto NewAccumulator =
5747 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BitCountOpc), DestReg: NumActiveLanes)
5748 .addReg(RegNo: ExecMask);
5749
5750 switch (Opc) {
5751 case AMDGPU::S_XOR_B32:
5752 case AMDGPU::S_XOR_B64: {
5753 // Performing an XOR operation on a uniform value
5754 // depends on the parity of the number of active lanes.
5755 // For even parity, the result will be 0, for odd
5756 // parity the result will be the same as the input value.
5757 Register ParityRegister =
5758 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5759
5760 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_AND_B32), DestReg: ParityRegister)
5761 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
5762 .addImm(Val: 1)
5763 .setOperandDead(3); // Dead scc
5764 if (Opc == AMDGPU::S_XOR_B32) {
5765 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
5766 .addReg(RegNo: SrcReg)
5767 .addReg(RegNo: ParityRegister);
5768 } else {
5769 Register DestSub0 =
5770 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5771 Register DestSub1 =
5772 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5773
5774 const TargetRegisterClass *SrcRC = MRI.getRegClass(Reg: SrcReg);
5775 const TargetRegisterClass *SrcSubRC =
5776 TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
5777
5778 MachineOperand Op1L = TII->buildExtractSubRegOrImm(
5779 MI, MRI, SuperReg: MI.getOperand(i: 1), SuperRC: SrcRC, SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
5780 MachineOperand Op1H = TII->buildExtractSubRegOrImm(
5781 MI, MRI, SuperReg: MI.getOperand(i: 1), SuperRC: SrcRC, SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
5782
5783 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub0)
5784 .add(MO: Op1L)
5785 .addReg(RegNo: ParityRegister);
5786
5787 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub1)
5788 .add(MO: Op1H)
5789 .addReg(RegNo: ParityRegister);
5790
5791 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: DstReg)
5792 .addReg(RegNo: DestSub0)
5793 .addImm(Val: AMDGPU::sub0)
5794 .addReg(RegNo: DestSub1)
5795 .addImm(Val: AMDGPU::sub1);
5796 }
5797 break;
5798 }
5799 case AMDGPU::S_SUB_I32: {
5800 Register NegatedVal = MRI.createVirtualRegister(RegClass: DstRegClass);
5801
5802 // Take the negation of the source operand.
5803 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedVal)
5804 .addImm(Val: 0)
5805 .addReg(RegNo: SrcReg);
5806 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
5807 .addReg(RegNo: NegatedVal)
5808 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg());
5809 break;
5810 }
5811 case AMDGPU::S_ADD_I32: {
5812 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
5813 .addReg(RegNo: SrcReg)
5814 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg());
5815 break;
5816 }
5817 case AMDGPU::S_ADD_U64_PSEUDO:
5818 case AMDGPU::S_SUB_U64_PSEUDO: {
5819 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5820 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5821 Register Op1H_Op0L_Reg =
5822 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5823 Register Op1L_Op0H_Reg =
5824 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5825 Register CarryReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5826 Register AddReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5827 Register NegatedValLo =
5828 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5829 Register NegatedValHi =
5830 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5831
5832 const TargetRegisterClass *Src1RC = MRI.getRegClass(Reg: SrcReg);
5833 const TargetRegisterClass *Src1SubRC =
5834 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub0);
5835
5836 MachineOperand Op1L = TII->buildExtractSubRegOrImm(
5837 MI, MRI, SuperReg: MI.getOperand(i: 1), SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
5838 MachineOperand Op1H = TII->buildExtractSubRegOrImm(
5839 MI, MRI, SuperReg: MI.getOperand(i: 1), SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
5840
5841 if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
5842 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedValLo)
5843 .addImm(Val: 0)
5844 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
5845 .setOperandDead(3); // Dead scc
5846 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ASHR_I32), DestReg: NegatedValHi)
5847 .addReg(RegNo: NegatedValLo)
5848 .addImm(Val: 31)
5849 .setOperandDead(3); // Dead scc
5850 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: Op1L_Op0H_Reg)
5851 .add(MO: Op1L)
5852 .addReg(RegNo: NegatedValHi);
5853 }
5854 Register LowOpcode = Opc == AMDGPU::S_SUB_U64_PSEUDO
5855 ? NegatedValLo
5856 : NewAccumulator->getOperand(i: 0).getReg();
5857 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub0)
5858 .add(MO: Op1L)
5859 .addReg(RegNo: LowOpcode);
5860 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_HI_U32), DestReg: CarryReg)
5861 .add(MO: Op1L)
5862 .addReg(RegNo: LowOpcode);
5863 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: Op1H_Op0L_Reg)
5864 .add(MO: Op1H)
5865 .addReg(RegNo: LowOpcode);
5866
5867 Register HiVal = Opc == AMDGPU::S_SUB_U64_PSEUDO ? AddReg : DestSub1;
5868 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: HiVal)
5869 .addReg(RegNo: CarryReg)
5870 .addReg(RegNo: Op1H_Op0L_Reg)
5871 .setOperandDead(3); // Dead scc
5872
5873 if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
5874 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: DestSub1)
5875 .addReg(RegNo: HiVal)
5876 .addReg(RegNo: Op1L_Op0H_Reg)
5877 .setOperandDead(3); // Dead scc
5878 }
5879 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: DstReg)
5880 .addReg(RegNo: DestSub0)
5881 .addImm(Val: AMDGPU::sub0)
5882 .addReg(RegNo: DestSub1)
5883 .addImm(Val: AMDGPU::sub1);
5884 break;
5885 }
5886 case AMDGPU::V_ADD_F32_e64:
5887 case AMDGPU::V_ADD_F64_e64:
5888 case AMDGPU::V_ADD_F64_pseudo_e64:
5889 case AMDGPU::V_SUB_F32_e64: {
5890 bool is32BitOpc = is32bitWaveReduceOperation(Opc);
5891 const TargetRegisterClass *VregRC = TII->getRegClass(MCID: TII->get(Opcode: Opc), OpNum: 0);
5892 Register ActiveLanesVreg = MRI.createVirtualRegister(RegClass: VregRC);
5893 Register DstVreg = MRI.createVirtualRegister(RegClass: VregRC);
5894 // Get number of active lanes as a float val.
5895 BuildMI(BB, I&: MI, MIMD: DL,
5896 MCID: TII->get(Opcode: is32BitOpc ? AMDGPU::V_CVT_F32_I32_e64
5897 : AMDGPU::V_CVT_F64_I32_e64),
5898 DestReg: ActiveLanesVreg)
5899 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
5900 .addImm(Val: 0) // clamp
5901 .addImm(Val: 0); // output-modifier
5902
5903 // Take negation of input for SUB reduction
5904 unsigned srcMod =
5905 (Opc == AMDGPU::V_SUB_F32_e64 ||
5906 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64)
5907 ? SISrcMods::NEG
5908 : SISrcMods::NONE;
5909 unsigned MulOpc = is32BitOpc ? AMDGPU::V_MUL_F32_e64
5910 : ST.getGeneration() >= AMDGPUSubtarget::GFX12
5911 ? AMDGPU::V_MUL_F64_pseudo_e64
5912 : AMDGPU::V_MUL_F64_e64;
5913 auto DestVregInst = BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MulOpc),
5914 DestReg: DstVreg)
5915 .addImm(Val: srcMod) // src0 modifier
5916 .addReg(RegNo: SrcReg)
5917 .addImm(Val: SISrcMods::NONE) // src1 modifier
5918 .addReg(RegNo: ActiveLanesVreg)
5919 .addImm(Val: SISrcMods::NONE) // clamp
5920 .addImm(Val: SISrcMods::NONE); // output-mod
5921 if (is32BitOpc) {
5922 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
5923 .addReg(RegNo: DstVreg);
5924 } else {
5925 Register LaneValueLoReg =
5926 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5927 Register LaneValueHiReg =
5928 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5929 const TargetRegisterClass *VregSubRC =
5930 TRI->getSubRegisterClass(VregRC, AMDGPU::sub0);
5931 MachineOperand Op1L =
5932 TII->buildExtractSubRegOrImm(MI, MRI, SuperReg: DestVregInst->getOperand(i: 0),
5933 SuperRC: VregRC, SubIdx: AMDGPU::sub0, SubRC: VregSubRC);
5934 MachineOperand Op1H =
5935 TII->buildExtractSubRegOrImm(MI, MRI, SuperReg: DestVregInst->getOperand(i: 0),
5936 SuperRC: VregRC, SubIdx: AMDGPU::sub1, SubRC: VregSubRC);
5937 // lane value input should be in an sgpr
5938 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
5939 DestReg: LaneValueLoReg)
5940 .add(MO: Op1L);
5941 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
5942 DestReg: LaneValueHiReg)
5943 .add(MO: Op1H);
5944 NewAccumulator =
5945 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: DstReg)
5946 .addReg(RegNo: LaneValueLoReg)
5947 .addImm(Val: AMDGPU::sub0)
5948 .addReg(RegNo: LaneValueHiReg)
5949 .addImm(Val: AMDGPU::sub1);
5950 }
5951 }
5952 }
5953 RetBB = &BB;
5954 }
5955 }
5956 } else {
5957 MachineBasicBlock::iterator I = BB.end();
5958 Register SrcReg = MI.getOperand(i: 1).getReg();
5959 bool is32BitOpc = is32bitWaveReduceOperation(Opc);
5960 bool isFPOp = isFloatingPointWaveReduceOperation(Opc);
5961 // Create virtual registers required for lowering.
5962 const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
5963 const TargetRegisterClass *DstRegClass = MRI.getRegClass(Reg: DstReg);
5964 const TargetRegisterClass *SrcRegClass = MRI.getRegClass(Reg: SrcReg);
5965 bool IsWave32 = ST.isWave32();
5966 unsigned MovOpcForExec = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
5967 unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
5968 if (Stratergy == WAVE_REDUCE_STRATEGY::ITERATIVE ||
5969 !ST.hasDPP()) { // If target doesn't support DPP operations, default to
5970 // iterative stratergy
5971
5972 // To reduce the VGPR using iterative approach, we need to iterate
5973 // over all the active lanes. Lowering consists of ComputeLoop,
5974 // which iterate over only active lanes. We use copy of EXEC register
5975 // as induction variable and every active lane modifies it using bitset0
5976 // so that we will get the next active lane for next iteration.
5977
5978 // Create Control flow for loop
5979 // Split MI's Machine Basic block into For loop
5980 auto [ComputeLoop, ComputeEnd] = splitBlockForLoop(MI, MBB&: BB, InstInLoop: true);
5981
5982 Register LoopIterator = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
5983 Register IdentityValReg = MRI.createVirtualRegister(RegClass: DstRegClass);
5984 Register AccumulatorReg = MRI.createVirtualRegister(RegClass: DstRegClass);
5985 Register ActiveBitsReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
5986 Register NewActiveBitsReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
5987 Register FF1Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5988 Register LaneValueReg = MRI.createVirtualRegister(RegClass: DstRegClass);
5989
5990 // Create initial values of induction variable from Exec, Accumulator and
5991 // insert branch instr to newly created ComputeBlock
5992 BuildMI(BB, I, MIMD: DL, MCID: TII->get(Opcode: MovOpcForExec), DestReg: LoopIterator).addReg(RegNo: ExecReg);
5993 if (is32BitOpc) {
5994 uint32_t IdentityValue = getIdentityValueFor32BitWaveReduction(Opc);
5995 BuildMI(BB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: IdentityValReg)
5996 .addImm(Val: IdentityValue);
5997 } else {
5998 uint64_t IdentityValue =
5999 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
6000 ? 0x0 // +0.0 for double sub reduction
6001 : getIdentityValueFor64BitWaveReduction(Opc);
6002 BuildMI(BB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B64_IMM_PSEUDO),
6003 DestReg: IdentityValReg)
6004 .addImm(Val: IdentityValue);
6005 }
6006 // clang-format off
6007 BuildMI(BB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_BRANCH))
6008 .addMBB(MBB: ComputeLoop);
6009 // clang-format on
6010
6011 // Start constructing ComputeLoop
6012 I = ComputeLoop->begin();
6013 auto Accumulator =
6014 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::PHI), DestReg: AccumulatorReg)
6015 .addReg(RegNo: IdentityValReg)
6016 .addMBB(MBB: &BB);
6017 auto ActiveBits =
6018 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::PHI), DestReg: ActiveBitsReg)
6019 .addReg(RegNo: LoopIterator)
6020 .addMBB(MBB: &BB);
6021
6022 I = ComputeLoop->end();
6023 MachineInstr *NewAccumulator;
6024 // Perform the computations
6025 unsigned SFFOpc =
6026 IsWave32 ? AMDGPU::S_FF1_I32_B32 : AMDGPU::S_FF1_I32_B64;
6027 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: SFFOpc), DestReg: FF1Reg)
6028 .addReg(RegNo: ActiveBitsReg);
6029 if (is32BitOpc) {
6030 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6031 DestReg: LaneValueReg)
6032 .addReg(RegNo: SrcReg)
6033 .addReg(RegNo: FF1Reg);
6034 if (isFPOp) {
6035 Register LaneValVreg =
6036 MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: SrcReg));
6037 Register DstVreg = MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: SrcReg));
6038 // Get the Lane Value in VGPR to avoid the Constant Bus Restriction
6039 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32),
6040 DestReg: LaneValVreg)
6041 .addReg(RegNo: LaneValueReg);
6042 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstVreg)
6043 .addImm(Val: 0) // src0 modifier
6044 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg())
6045 .addImm(Val: 0) // src1 modifier
6046 .addReg(RegNo: LaneValVreg)
6047 .addImm(Val: 0) // clamp
6048 .addImm(Val: 0); // omod
6049 NewAccumulator =
6050 BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6051 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
6052 .addReg(RegNo: DstVreg);
6053 } else {
6054 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstReg)
6055 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg())
6056 .addReg(RegNo: LaneValueReg);
6057 }
6058 } else {
6059 Register LaneValueLoReg =
6060 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6061 Register LaneValueHiReg =
6062 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6063 Register LaneValReg =
6064 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6065 const TargetRegisterClass *SrcRC = MRI.getRegClass(Reg: SrcReg);
6066 const TargetRegisterClass *SrcSubRC =
6067 TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
6068 MachineOperand Op1L = TII->buildExtractSubRegOrImm(
6069 MI, MRI, SuperReg: MI.getOperand(i: 1), SuperRC: SrcRC, SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
6070 MachineOperand Op1H = TII->buildExtractSubRegOrImm(
6071 MI, MRI, SuperReg: MI.getOperand(i: 1), SuperRC: SrcRC, SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
6072 // lane value input should be in an sgpr
6073 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6074 DestReg: LaneValueLoReg)
6075 .add(MO: Op1L)
6076 .addReg(RegNo: FF1Reg);
6077 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6078 DestReg: LaneValueHiReg)
6079 .add(MO: Op1H)
6080 .addReg(RegNo: FF1Reg);
6081 auto LaneValue =
6082 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE),
6083 DestReg: LaneValReg)
6084 .addReg(RegNo: LaneValueLoReg)
6085 .addImm(Val: AMDGPU::sub0)
6086 .addReg(RegNo: LaneValueHiReg)
6087 .addImm(Val: AMDGPU::sub1);
6088 switch (Opc) {
6089 case AMDGPU::S_OR_B64:
6090 case AMDGPU::S_AND_B64:
6091 case AMDGPU::S_XOR_B64: {
6092 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstReg)
6093 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg())
6094 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6095 .setOperandDead(3); // Dead scc
6096 break;
6097 }
6098 case AMDGPU::V_CMP_GT_I64_e64:
6099 case AMDGPU::V_CMP_GT_U64_e64:
6100 case AMDGPU::V_CMP_LT_I64_e64:
6101 case AMDGPU::V_CMP_LT_U64_e64: {
6102 Register LaneMaskReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6103 Register ComparisonResultReg =
6104 MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6105 int SrcIdx =
6106 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src);
6107 const TargetRegisterClass *VregClass =
6108 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: SrcIdx));
6109 const TargetRegisterClass *VSubRegClass =
6110 TRI->getSubRegisterClass(VregClass, AMDGPU::sub0);
6111 Register AccumulatorVReg = MRI.createVirtualRegister(RegClass: VregClass);
6112 MachineOperand SrcReg0Sub0 = TII->buildExtractSubRegOrImm(
6113 MI, MRI, SuperReg: Accumulator->getOperand(i: 0), SuperRC: VregClass, SubIdx: AMDGPU::sub0,
6114 SubRC: VSubRegClass);
6115 MachineOperand SrcReg0Sub1 = TII->buildExtractSubRegOrImm(
6116 MI, MRI, SuperReg: Accumulator->getOperand(i: 0), SuperRC: VregClass, SubIdx: AMDGPU::sub1,
6117 SubRC: VSubRegClass);
6118 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE),
6119 DestReg: AccumulatorVReg)
6120 .add(MO: SrcReg0Sub0)
6121 .addImm(Val: AMDGPU::sub0)
6122 .add(MO: SrcReg0Sub1)
6123 .addImm(Val: AMDGPU::sub1);
6124 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: LaneMaskReg)
6125 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6126 .addReg(RegNo: AccumulatorVReg);
6127
6128 unsigned AndOpc = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
6129 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AndOpc), DestReg: ComparisonResultReg)
6130 .addReg(RegNo: LaneMaskReg)
6131 .addReg(RegNo: ActiveBitsReg);
6132
6133 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6134 MCID: TII->get(Opcode: AMDGPU::S_CSELECT_B64), DestReg: DstReg)
6135 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6136 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg());
6137 break;
6138 }
6139 case AMDGPU::V_MIN_F64_e64:
6140 case AMDGPU::V_MIN_NUM_F64_e64:
6141 case AMDGPU::V_MAX_F64_e64:
6142 case AMDGPU::V_MAX_NUM_F64_e64:
6143 case AMDGPU::V_ADD_F64_e64:
6144 case AMDGPU::V_ADD_F64_pseudo_e64: {
6145 int SrcIdx =
6146 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src);
6147 const TargetRegisterClass *VregRC =
6148 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: SrcIdx));
6149 const TargetRegisterClass *VregSubRC =
6150 TRI->getSubRegisterClass(VregRC, AMDGPU::sub0);
6151 Register AccumulatorVReg = MRI.createVirtualRegister(RegClass: VregRC);
6152 Register DstVreg = MRI.createVirtualRegister(RegClass: VregRC);
6153 Register LaneValLo =
6154 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6155 Register LaneValHi =
6156 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6157 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AccumulatorVReg)
6158 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg());
6159 unsigned Modifier =
6160 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
6161 ? SISrcMods::NEG
6162 : SISrcMods::NONE;
6163 auto DstVregInst =
6164 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstVreg)
6165 .addImm(Val: Modifier) // src0 modifiers
6166 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6167 .addImm(Val: SISrcMods::NONE) // src1 modifiers
6168 .addReg(RegNo: AccumulatorVReg)
6169 .addImm(Val: SISrcMods::NONE) // clamp
6170 .addImm(Val: SISrcMods::NONE); // omod
6171 auto ReadLaneLo =
6172 BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6173 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: LaneValLo);
6174 auto ReadLaneHi =
6175 BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6176 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: LaneValHi);
6177 MachineBasicBlock::iterator Iters = *ReadLaneLo;
6178 MachineOperand Op1L = TII->buildExtractSubRegOrImm(
6179 MI: Iters, MRI, SuperReg: DstVregInst->getOperand(i: 0), SuperRC: VregRC, SubIdx: AMDGPU::sub0,
6180 SubRC: VregSubRC);
6181 MachineOperand Op1H = TII->buildExtractSubRegOrImm(
6182 MI: Iters, MRI, SuperReg: DstVregInst->getOperand(i: 0), SuperRC: VregRC, SubIdx: AMDGPU::sub1,
6183 SubRC: VregSubRC);
6184 ReadLaneLo.add(MO: Op1L);
6185 ReadLaneHi.add(MO: Op1H);
6186 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6187 MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: DstReg)
6188 .addReg(RegNo: LaneValLo)
6189 .addImm(Val: AMDGPU::sub0)
6190 .addReg(RegNo: LaneValHi)
6191 .addImm(Val: AMDGPU::sub1);
6192 break;
6193 }
6194 case AMDGPU::S_ADD_U64_PSEUDO:
6195 case AMDGPU::S_SUB_U64_PSEUDO: {
6196 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstReg)
6197 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg())
6198 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg());
6199 ComputeLoop =
6200 Expand64BitScalarArithmetic(MI&: *NewAccumulator, BB: ComputeLoop);
6201 break;
6202 }
6203 }
6204 }
6205 // Manipulate the iterator to get the next active lane
6206 unsigned BITSETOpc =
6207 IsWave32 ? AMDGPU::S_BITSET0_B32 : AMDGPU::S_BITSET0_B64;
6208 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: BITSETOpc), DestReg: NewActiveBitsReg)
6209 .addReg(RegNo: FF1Reg)
6210 .addReg(RegNo: ActiveBitsReg);
6211
6212 // Add phi nodes
6213 Accumulator.addReg(RegNo: DstReg).addMBB(MBB: ComputeLoop);
6214 ActiveBits.addReg(RegNo: NewActiveBitsReg).addMBB(MBB: ComputeLoop);
6215
6216 // Creating branching
6217 unsigned CMPOpc = IsWave32 ? AMDGPU::S_CMP_LG_U32 : AMDGPU::S_CMP_LG_U64;
6218 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: CMPOpc))
6219 .addReg(RegNo: NewActiveBitsReg)
6220 .addImm(Val: 0);
6221 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
6222 .addMBB(MBB: ComputeLoop);
6223
6224 RetBB = ComputeEnd;
6225 } else {
6226 assert(ST.hasDPP() && "Sub Target does not support DPP Operations");
6227
6228 bool IsFPOp = isFloatingPointWaveReduceOperation(Opc);
6229 Register SrcWithIdentity = MRI.createVirtualRegister(RegClass: SrcRegClass);
6230 Register IdentityVGPR = MRI.createVirtualRegister(RegClass: SrcRegClass);
6231 Register IdentitySGPR = MRI.createVirtualRegister(RegClass: DstRegClass);
6232 Register DPPRowShr1 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6233 Register DPPRowShr2 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6234 Register DPPRowShr4 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6235 Register DPPRowShr8 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6236 Register RowBcast15 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6237 Register ReducedValSGPR = MRI.createVirtualRegister(RegClass: DstRegClass);
6238 Register NegatedReducedVal = MRI.createVirtualRegister(RegClass: DstRegClass);
6239 Register RowBcast31 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6240 Register UndefExec = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6241 Register FinalDPPResult;
6242 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefExec);
6243
6244 uint32_t IdentityValue = getIdentityValueFor32BitWaveReduction(Opc);
6245 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: IdentitySGPR)
6246 .addImm(Val: IdentityValue);
6247 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: IdentityVGPR)
6248 .addReg(RegNo: IdentitySGPR);
6249
6250 // Set inactive lanes to the identity value.
6251 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_SET_INACTIVE_B32), DestReg: SrcWithIdentity)
6252 .addImm(Val: 0) // src0 modifiers
6253 .addReg(RegNo: SrcReg) // src0
6254 .addImm(Val: 0) // src1 modifiers
6255 .addReg(RegNo: IdentityVGPR) // identity value for inactive lanes
6256 .addReg(RegNo: UndefExec); // bool i1
6257
6258 unsigned DPPOpc = getDPPOpcForWaveReduction(Opc, ST);
6259 auto BuildDPPMachineInstr = [&](Register Dst, Register Src,
6260 unsigned DPPCtrl) {
6261 auto DPPInstr =
6262 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: DPPOpc), DestReg: Dst).addReg(RegNo: Src); // old
6263 if (IsFPOp)
6264 DPPInstr.addImm(Val: SISrcMods::NONE); // src0 modifier
6265 DPPInstr.addReg(RegNo: Src); // src0
6266 if (IsFPOp)
6267 DPPInstr.addImm(Val: SISrcMods::NONE); // src1 modifier
6268 DPPInstr
6269 .addReg(RegNo: Src) // src1
6270 .addImm(Val: DPPCtrl) // dpp-ctrl
6271 .addImm(Val: 0xf) // row-mask
6272 .addImm(Val: 0xf) // bank-mask
6273 .addImm(Val: 0); // bound-control
6274 };
6275 auto BuildClampInstr = [&](Register Dst, Register Src0, Register Src1) {
6276 unsigned ClampOpc = Opc;
6277 if (!IsFPOp) {
6278 if (Opc == AMDGPU::S_SUB_I32)
6279 ClampOpc = AMDGPU::S_ADD_I32;
6280 ClampOpc = TII->getVALUOp(Opc: ClampOpc);
6281 }
6282 auto ClampInstr = BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: ClampOpc), DestReg: Dst);
6283 if (IsFPOp)
6284 ClampInstr.addImm(Val: SISrcMods::NONE); // src0 mod
6285 ClampInstr.addReg(RegNo: Src0); // src0
6286 if (IsFPOp)
6287 ClampInstr.addImm(Val: SISrcMods::NONE); // src1 mod
6288 ClampInstr.addReg(RegNo: Src1); // src1
6289 if (TII->hasIntClamp(MI: *ClampInstr) || TII->hasFPClamp(MI: *ClampInstr))
6290 ClampInstr.addImm(Val: 0); // clamp
6291 if (IsFPOp)
6292 ClampInstr.addImm(Val: 0); // omod
6293 };
6294 // DPP reduction
6295 BuildDPPMachineInstr(DPPRowShr1, SrcWithIdentity,
6296 AMDGPU::DPP::ROW_SHR_FIRST);
6297
6298 BuildDPPMachineInstr(DPPRowShr2, DPPRowShr1,
6299 (AMDGPU::DPP::ROW_SHR_FIRST + 1));
6300
6301 BuildDPPMachineInstr(DPPRowShr4, DPPRowShr2,
6302 (AMDGPU::DPP::ROW_SHR_FIRST + 3));
6303
6304 BuildDPPMachineInstr(DPPRowShr8, DPPRowShr4,
6305 (AMDGPU::DPP::ROW_SHR_FIRST + 7));
6306
6307 if (ST.hasDPPBroadcasts()) {
6308 BuildDPPMachineInstr(RowBcast15, DPPRowShr8, AMDGPU::DPP::BCAST15);
6309 } else {
6310 // magic constant: 0x1E0
6311 // To Set BIT_MODE : bit 15 = 0
6312 // XOR mask : bit [14:10] = 0
6313 // OR mask : bit [9:5] = 15
6314 // AND mask : bit [4:0] = 0
6315 Register SwizzledValue =
6316 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6317 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_SWIZZLE_B32), DestReg: SwizzledValue)
6318 .addReg(RegNo: DPPRowShr8) // addr
6319 .addImm(Val: 0x1E0) // swizzle offset (i16)
6320 .addImm(Val: 0x0); // gds (i1)
6321 BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue);
6322 }
6323 FinalDPPResult = RowBcast15;
6324 if (!IsWave32) {
6325 if (ST.hasDPPBroadcasts()) {
6326 BuildDPPMachineInstr(RowBcast31, RowBcast15, AMDGPU::DPP::BCAST31);
6327 } else {
6328 Register ShiftedThreadID =
6329 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6330 Register PermuteByteOffset =
6331 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6332 Register PermutedValue =
6333 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6334 Register Lane32Offset = MRI.createVirtualRegister(RegClass: DstRegClass);
6335 Register WordSizeConst = MRI.createVirtualRegister(RegClass: DstRegClass);
6336 Register ThreadIDRegLo =
6337 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6338 Register ThreadIDReg =
6339 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6340 // Get the thread ID.
6341 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MBCNT_LO_U32_B32_e64),
6342 DestReg: ThreadIDRegLo)
6343 .addImm(Val: -1)
6344 .addImm(Val: 0);
6345 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MBCNT_HI_U32_B32_e64),
6346 DestReg: ThreadIDReg)
6347 .addImm(Val: -1)
6348 .addReg(RegNo: ThreadIDRegLo);
6349 // shift each lane over by 32 positions, so value in 31st lane is
6350 // present in 63rd lane.
6351 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: Lane32Offset)
6352 .addImm(Val: 0x20);
6353 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ADD_U32_e64), DestReg: ShiftedThreadID)
6354 .addReg(RegNo: ThreadIDReg)
6355 .addReg(RegNo: Lane32Offset)
6356 .addImm(Val: 0); // clamp
6357 // multiply by reg size.
6358 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: WordSizeConst)
6359 .addImm(Val: 0x4);
6360 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MUL_LO_U32_e64),
6361 DestReg: PermuteByteOffset)
6362 .addReg(RegNo: WordSizeConst)
6363 .addReg(RegNo: ShiftedThreadID);
6364 // Permute the lanes
6365 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_PERMUTE_B32), DestReg: PermutedValue)
6366 .addReg(RegNo: PermuteByteOffset) // addr
6367 .addReg(RegNo: RowBcast15) // data
6368 .addImm(Val: 0); // offset
6369 BuildClampInstr(RowBcast31, RowBcast15, PermutedValue);
6370 }
6371 FinalDPPResult = RowBcast31;
6372 }
6373 if (Opc == AMDGPU::V_SUB_F32_e64) {
6374 Register NegatedValVGPR =
6375 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6376 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_SUB_F32_e64), DestReg: NegatedValVGPR)
6377 .addImm(Val: SISrcMods::NONE) // src0 mods
6378 .addReg(RegNo: IdentityVGPR) // src0
6379 .addImm(Val: SISrcMods::NONE) // src1 mods
6380 .addReg(RegNo: IsWave32 ? RowBcast15 : RowBcast31) // src1
6381 .addImm(Val: SISrcMods::NONE) // clamp
6382 .addImm(Val: SISrcMods::NONE); // omod
6383 FinalDPPResult = NegatedValVGPR;
6384 }
6385 // The final reduced value is in the last lane.
6386 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32), DestReg: ReducedValSGPR)
6387 .addReg(RegNo: FinalDPPResult)
6388 .addImm(Val: ST.getWavefrontSize() - 1);
6389 if (Opc == AMDGPU::S_SUB_I32)
6390 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedReducedVal)
6391 .addImm(Val: 0)
6392 .addReg(RegNo: ReducedValSGPR);
6393 // Mark the final result as a whole-wave-mode calculation.
6394 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::STRICT_WWM), DestReg: DstReg)
6395 .addReg(RegNo: Opc == AMDGPU::S_SUB_I32 ? NegatedReducedVal
6396 : ReducedValSGPR);
6397 RetBB = &BB;
6398 }
6399 }
6400 MI.eraseFromParent();
6401 return RetBB;
6402}
6403
6404MachineBasicBlock *
6405SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
6406 MachineBasicBlock *BB) const {
6407 MachineFunction *MF = BB->getParent();
6408 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
6409 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
6410 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
6411 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
6412 MachineRegisterInfo &MRI = MF->getRegInfo();
6413 const DebugLoc &DL = MI.getDebugLoc();
6414
6415 switch (MI.getOpcode()) {
6416 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U32:
6417 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MIN_U32);
6418 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U64:
6419 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_LT_U64_e64);
6420 case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I32:
6421 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MIN_I32);
6422 case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I64:
6423 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_LT_I64_e64);
6424 case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F32:
6425 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_MIN_F32_e64);
6426 case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F64:
6427 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6428 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6429 ? AMDGPU::V_MIN_NUM_F64_e64
6430 : AMDGPU::V_MIN_F64_e64);
6431 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U32:
6432 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MAX_U32);
6433 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U64:
6434 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_GT_U64_e64);
6435 case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I32:
6436 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MAX_I32);
6437 case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I64:
6438 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_GT_I64_e64);
6439 case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F32:
6440 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_MAX_F32_e64);
6441 case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F64:
6442 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6443 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6444 ? AMDGPU::V_MAX_NUM_F64_e64
6445 : AMDGPU::V_MAX_F64_e64);
6446 case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_I32:
6447 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_ADD_I32);
6448 case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_U64:
6449 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_ADD_U64_PSEUDO);
6450 case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F32:
6451 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_ADD_F32_e64);
6452 case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F64:
6453 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6454 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6455 ? AMDGPU::V_ADD_F64_pseudo_e64
6456 : AMDGPU::V_ADD_F64_e64);
6457 case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_I32:
6458 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_SUB_I32);
6459 case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_U64:
6460 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_SUB_U64_PSEUDO);
6461 case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32:
6462 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_SUB_F32_e64);
6463 case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64:
6464 // There is no S/V_SUB_F64 opcode. Double type subtraction is expanded as
6465 // fadd + neg, by setting the NEG bit in the instruction.
6466 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6467 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6468 ? AMDGPU::V_ADD_F64_pseudo_e64
6469 : AMDGPU::V_ADD_F64_e64);
6470 case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B32:
6471 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_AND_B32);
6472 case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B64:
6473 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_AND_B64);
6474 case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B32:
6475 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_OR_B32);
6476 case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B64:
6477 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_OR_B64);
6478 case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B32:
6479 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_XOR_B32);
6480 case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B64:
6481 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_XOR_B64);
6482 case AMDGPU::S_UADDO_PSEUDO:
6483 case AMDGPU::S_USUBO_PSEUDO: {
6484 MachineOperand &Dest0 = MI.getOperand(i: 0);
6485 MachineOperand &Dest1 = MI.getOperand(i: 1);
6486 MachineOperand &Src0 = MI.getOperand(i: 2);
6487 MachineOperand &Src1 = MI.getOperand(i: 3);
6488
6489 unsigned Opc = (MI.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
6490 ? AMDGPU::S_ADD_U32
6491 : AMDGPU::S_SUB_U32;
6492 // clang-format off
6493 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest0.getReg())
6494 .add(MO: Src0)
6495 .add(MO: Src1);
6496 // clang-format on
6497
6498 unsigned SelOpc =
6499 Subtarget->isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
6500 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SelOpc), DestReg: Dest1.getReg()).addImm(Val: -1).addImm(Val: 0);
6501
6502 MI.eraseFromParent();
6503 return BB;
6504 }
6505 case AMDGPU::S_ADD_U64_PSEUDO:
6506 case AMDGPU::S_SUB_U64_PSEUDO: {
6507 return Expand64BitScalarArithmetic(MI, BB);
6508 }
6509 case AMDGPU::V_ADD_U64_PSEUDO:
6510 case AMDGPU::V_SUB_U64_PSEUDO: {
6511 bool IsAdd = (MI.getOpcode() == AMDGPU::V_ADD_U64_PSEUDO);
6512
6513 MachineOperand &Dest = MI.getOperand(i: 0);
6514 MachineOperand &Src0 = MI.getOperand(i: 1);
6515 MachineOperand &Src1 = MI.getOperand(i: 2);
6516
6517 if (ST.hasAddSubU64Insts()) {
6518 auto I = BuildMI(BB&: *BB, I&: MI, MIMD: DL,
6519 MCID: TII->get(Opcode: IsAdd ? AMDGPU::V_ADD_U64_e64
6520 : AMDGPU::V_SUB_U64_e64),
6521 DestReg: Dest.getReg())
6522 .add(MO: Src0)
6523 .add(MO: Src1)
6524 .addImm(Val: 0); // clamp
6525 TII->legalizeOperands(MI&: *I);
6526 MI.eraseFromParent();
6527 return BB;
6528 }
6529
6530 if (IsAdd && ST.hasLshlAddU64Inst()) {
6531 auto Add = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_LSHL_ADD_U64_e64),
6532 DestReg: Dest.getReg())
6533 .add(MO: Src0)
6534 .addImm(Val: 0)
6535 .add(MO: Src1);
6536 TII->legalizeOperands(MI&: *Add);
6537 MI.eraseFromParent();
6538 return BB;
6539 }
6540
6541 const auto *CarryRC = TRI->getWaveMaskRegClass();
6542
6543 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6544 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6545
6546 Register CarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
6547 Register DeadCarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
6548
6549 const TargetRegisterClass *Src0RC = Src0.isReg()
6550 ? MRI.getRegClass(Reg: Src0.getReg())
6551 : &AMDGPU::VReg_64RegClass;
6552 const TargetRegisterClass *Src1RC = Src1.isReg()
6553 ? MRI.getRegClass(Reg: Src1.getReg())
6554 : &AMDGPU::VReg_64RegClass;
6555
6556 const TargetRegisterClass *Src0SubRC =
6557 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
6558 const TargetRegisterClass *Src1SubRC =
6559 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
6560
6561 MachineOperand SrcReg0Sub0 = TII->buildExtractSubRegOrImm(
6562 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
6563 MachineOperand SrcReg1Sub0 = TII->buildExtractSubRegOrImm(
6564 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
6565
6566 MachineOperand SrcReg0Sub1 = TII->buildExtractSubRegOrImm(
6567 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
6568 MachineOperand SrcReg1Sub1 = TII->buildExtractSubRegOrImm(
6569 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
6570
6571 unsigned LoOpc =
6572 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
6573 MachineInstr *LoHalf = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoOpc), DestReg: DestSub0)
6574 .addReg(RegNo: CarryReg, Flags: RegState::Define)
6575 .add(MO: SrcReg0Sub0)
6576 .add(MO: SrcReg1Sub0)
6577 .addImm(Val: 0); // clamp bit
6578
6579 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
6580 MachineInstr *HiHalf =
6581 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: HiOpc), DestReg: DestSub1)
6582 .addReg(RegNo: DeadCarryReg, Flags: RegState::Define | RegState::Dead)
6583 .add(MO: SrcReg0Sub1)
6584 .add(MO: SrcReg1Sub1)
6585 .addReg(RegNo: CarryReg, Flags: RegState::Kill)
6586 .addImm(Val: 0); // clamp bit
6587
6588 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest.getReg())
6589 .addReg(RegNo: DestSub0)
6590 .addImm(Val: AMDGPU::sub0)
6591 .addReg(RegNo: DestSub1)
6592 .addImm(Val: AMDGPU::sub1);
6593 TII->legalizeOperands(MI&: *LoHalf);
6594 TII->legalizeOperands(MI&: *HiHalf);
6595 MI.eraseFromParent();
6596 return BB;
6597 }
6598 case AMDGPU::S_ADD_CO_PSEUDO:
6599 case AMDGPU::S_SUB_CO_PSEUDO: {
6600 // This pseudo has a chance to be selected
6601 // only from uniform add/subcarry node. All the VGPR operands
6602 // therefore assumed to be splat vectors.
6603 MachineBasicBlock::iterator MII = MI;
6604 MachineOperand &Dest = MI.getOperand(i: 0);
6605 MachineOperand &CarryDest = MI.getOperand(i: 1);
6606 MachineOperand &Src0 = MI.getOperand(i: 2);
6607 MachineOperand &Src1 = MI.getOperand(i: 3);
6608 MachineOperand &Src2 = MI.getOperand(i: 4);
6609 if (Src0.isReg() && TRI->isVectorRegister(MRI, Reg: Src0.getReg())) {
6610 Register RegOp0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6611 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp0)
6612 .addReg(RegNo: Src0.getReg());
6613 Src0.setReg(RegOp0);
6614 }
6615 if (Src1.isReg() && TRI->isVectorRegister(MRI, Reg: Src1.getReg())) {
6616 Register RegOp1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6617 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp1)
6618 .addReg(RegNo: Src1.getReg());
6619 Src1.setReg(RegOp1);
6620 }
6621 Register RegOp2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6622 if (TRI->isVectorRegister(MRI, Reg: Src2.getReg())) {
6623 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp2)
6624 .addReg(RegNo: Src2.getReg());
6625 Src2.setReg(RegOp2);
6626 }
6627
6628 if (ST.isWave64()) {
6629 if (ST.hasScalarCompareEq64()) {
6630 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U64))
6631 .addReg(RegNo: Src2.getReg())
6632 .addImm(Val: 0);
6633 } else {
6634 const TargetRegisterClass *Src2RC = MRI.getRegClass(Reg: Src2.getReg());
6635 const TargetRegisterClass *SubRC =
6636 TRI->getSubRegisterClass(Src2RC, AMDGPU::sub0);
6637 MachineOperand Src2Sub0 = TII->buildExtractSubRegOrImm(
6638 MI: MII, MRI, SuperReg: Src2, SuperRC: Src2RC, SubIdx: AMDGPU::sub0, SubRC);
6639 MachineOperand Src2Sub1 = TII->buildExtractSubRegOrImm(
6640 MI: MII, MRI, SuperReg: Src2, SuperRC: Src2RC, SubIdx: AMDGPU::sub1, SubRC);
6641 Register Src2_32 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6642
6643 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_OR_B32), DestReg: Src2_32)
6644 .add(MO: Src2Sub0)
6645 .add(MO: Src2Sub1);
6646
6647 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
6648 .addReg(RegNo: Src2_32, Flags: RegState::Kill)
6649 .addImm(Val: 0);
6650 }
6651 } else {
6652 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
6653 .addReg(RegNo: Src2.getReg())
6654 .addImm(Val: 0);
6655 }
6656
6657 unsigned Opc = MI.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO
6658 ? AMDGPU::S_ADDC_U32
6659 : AMDGPU::S_SUBB_U32;
6660
6661 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest.getReg()).add(MO: Src0).add(MO: Src1);
6662
6663 unsigned SelOpc =
6664 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
6665
6666 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: SelOpc), DestReg: CarryDest.getReg())
6667 .addImm(Val: -1)
6668 .addImm(Val: 0);
6669
6670 MI.eraseFromParent();
6671 return BB;
6672 }
6673 case AMDGPU::SI_INIT_M0: {
6674 MachineOperand &M0Init = MI.getOperand(i: 0);
6675 BuildMI(BB&: *BB, I: MI.getIterator(), MIMD: MI.getDebugLoc(),
6676 MCID: TII->get(Opcode: M0Init.isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32),
6677 DestReg: AMDGPU::M0)
6678 .add(MO: M0Init);
6679 MI.eraseFromParent();
6680 return BB;
6681 }
6682 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM: {
6683 // Set SCC to true, in case the barrier instruction gets converted to a NOP.
6684 BuildMI(BB&: *BB, I: MI.getIterator(), MIMD: MI.getDebugLoc(),
6685 MCID: TII->get(Opcode: AMDGPU::S_CMP_EQ_U32))
6686 .addImm(Val: 0)
6687 .addImm(Val: 0);
6688 return BB;
6689 }
6690 case AMDGPU::GET_GROUPSTATICSIZE: {
6691 assert(getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA ||
6692 getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL);
6693 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32))
6694 .add(MO: MI.getOperand(i: 0))
6695 .addImm(Val: MFI->getLDSSize());
6696 MI.eraseFromParent();
6697 return BB;
6698 }
6699 case AMDGPU::GET_SHADERCYCLESHILO: {
6700 assert(MF->getSubtarget<GCNSubtarget>().hasShaderCyclesHiLoRegisters());
6701 // The algorithm is:
6702 //
6703 // hi1 = getreg(SHADER_CYCLES_HI)
6704 // lo1 = getreg(SHADER_CYCLES_LO)
6705 // hi2 = getreg(SHADER_CYCLES_HI)
6706 //
6707 // If hi1 == hi2 then there was no overflow and the result is hi2:lo1.
6708 // Otherwise there was overflow and the result is hi2:0. In both cases the
6709 // result should represent the actual time at some point during the sequence
6710 // of three getregs.
6711 using namespace AMDGPU::Hwreg;
6712 Register RegHi1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6713 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegHi1)
6714 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES_HI, Values: 0, Values: 32));
6715 Register RegLo1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6716 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegLo1)
6717 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES, Values: 0, Values: 32));
6718 Register RegHi2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6719 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegHi2)
6720 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES_HI, Values: 0, Values: 32));
6721 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_EQ_U32))
6722 .addReg(RegNo: RegHi1)
6723 .addReg(RegNo: RegHi2);
6724 Register RegLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6725 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: RegLo)
6726 .addReg(RegNo: RegLo1)
6727 .addImm(Val: 0);
6728 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE))
6729 .add(MO: MI.getOperand(i: 0))
6730 .addReg(RegNo: RegLo)
6731 .addImm(Val: AMDGPU::sub0)
6732 .addReg(RegNo: RegHi2)
6733 .addImm(Val: AMDGPU::sub1);
6734 MI.eraseFromParent();
6735 return BB;
6736 }
6737 case AMDGPU::SI_INDIRECT_SRC_V1:
6738 case AMDGPU::SI_INDIRECT_SRC_V2:
6739 case AMDGPU::SI_INDIRECT_SRC_V3:
6740 case AMDGPU::SI_INDIRECT_SRC_V4:
6741 case AMDGPU::SI_INDIRECT_SRC_V5:
6742 case AMDGPU::SI_INDIRECT_SRC_V6:
6743 case AMDGPU::SI_INDIRECT_SRC_V7:
6744 case AMDGPU::SI_INDIRECT_SRC_V8:
6745 case AMDGPU::SI_INDIRECT_SRC_V9:
6746 case AMDGPU::SI_INDIRECT_SRC_V10:
6747 case AMDGPU::SI_INDIRECT_SRC_V11:
6748 case AMDGPU::SI_INDIRECT_SRC_V12:
6749 case AMDGPU::SI_INDIRECT_SRC_V16:
6750 case AMDGPU::SI_INDIRECT_SRC_V32:
6751 return emitIndirectSrc(MI, MBB&: *BB, ST: *getSubtarget());
6752 case AMDGPU::SI_INDIRECT_DST_V1:
6753 case AMDGPU::SI_INDIRECT_DST_V2:
6754 case AMDGPU::SI_INDIRECT_DST_V3:
6755 case AMDGPU::SI_INDIRECT_DST_V4:
6756 case AMDGPU::SI_INDIRECT_DST_V5:
6757 case AMDGPU::SI_INDIRECT_DST_V6:
6758 case AMDGPU::SI_INDIRECT_DST_V7:
6759 case AMDGPU::SI_INDIRECT_DST_V8:
6760 case AMDGPU::SI_INDIRECT_DST_V9:
6761 case AMDGPU::SI_INDIRECT_DST_V10:
6762 case AMDGPU::SI_INDIRECT_DST_V11:
6763 case AMDGPU::SI_INDIRECT_DST_V12:
6764 case AMDGPU::SI_INDIRECT_DST_V16:
6765 case AMDGPU::SI_INDIRECT_DST_V32:
6766 return emitIndirectDst(MI, MBB&: *BB, ST: *getSubtarget());
6767 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
6768 case AMDGPU::SI_KILL_I1_PSEUDO:
6769 return splitKillBlock(MI, BB);
6770 case AMDGPU::V_CNDMASK_B64_PSEUDO: {
6771 Register Dst = MI.getOperand(i: 0).getReg();
6772 const MachineOperand &Src0 = MI.getOperand(i: 1);
6773 const MachineOperand &Src1 = MI.getOperand(i: 2);
6774 Register SrcCond = MI.getOperand(i: 3).getReg();
6775
6776 Register DstLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6777 Register DstHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6778 const auto *CondRC = TRI->getWaveMaskRegClass();
6779 Register SrcCondCopy = MRI.createVirtualRegister(RegClass: CondRC);
6780
6781 const TargetRegisterClass *Src0RC = Src0.isReg()
6782 ? MRI.getRegClass(Reg: Src0.getReg())
6783 : &AMDGPU::VReg_64RegClass;
6784 const TargetRegisterClass *Src1RC = Src1.isReg()
6785 ? MRI.getRegClass(Reg: Src1.getReg())
6786 : &AMDGPU::VReg_64RegClass;
6787
6788 const TargetRegisterClass *Src0SubRC =
6789 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
6790 const TargetRegisterClass *Src1SubRC =
6791 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
6792
6793 MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
6794 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
6795 MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
6796 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
6797
6798 MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
6799 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
6800 MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
6801 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
6802
6803 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: SrcCondCopy).addReg(RegNo: SrcCond);
6804 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstLo)
6805 .addImm(Val: 0)
6806 .add(MO: Src0Sub0)
6807 .addImm(Val: 0)
6808 .add(MO: Src1Sub0)
6809 .addReg(RegNo: SrcCondCopy);
6810 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstHi)
6811 .addImm(Val: 0)
6812 .add(MO: Src0Sub1)
6813 .addImm(Val: 0)
6814 .add(MO: Src1Sub1)
6815 .addReg(RegNo: SrcCondCopy);
6816
6817 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
6818 .addReg(RegNo: DstLo)
6819 .addImm(Val: AMDGPU::sub0)
6820 .addReg(RegNo: DstHi)
6821 .addImm(Val: AMDGPU::sub1);
6822 MI.eraseFromParent();
6823 return BB;
6824 }
6825 case AMDGPU::SI_BR_UNDEF: {
6826 MachineInstr *Br = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
6827 .add(MO: MI.getOperand(i: 0));
6828 Br->getOperand(i: 1).setIsUndef(); // read undef SCC
6829 MI.eraseFromParent();
6830 return BB;
6831 }
6832 case AMDGPU::ADJCALLSTACKUP:
6833 case AMDGPU::ADJCALLSTACKDOWN: {
6834 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
6835 MachineInstrBuilder MIB(*MF, &MI);
6836 MIB.addReg(RegNo: Info->getStackPtrOffsetReg(), Flags: RegState::ImplicitDefine)
6837 .addReg(RegNo: Info->getStackPtrOffsetReg(), Flags: RegState::Implicit);
6838 return BB;
6839 }
6840 case AMDGPU::SI_CALL_ISEL: {
6841 unsigned ReturnAddrReg = TII->getRegisterInfo().getReturnAddressReg(MF: *MF);
6842
6843 MachineInstrBuilder MIB;
6844 MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_CALL), DestReg: ReturnAddrReg);
6845
6846 for (const MachineOperand &MO : MI.operands())
6847 MIB.add(MO);
6848
6849 MIB.cloneMemRefs(OtherMI: MI);
6850 MI.eraseFromParent();
6851 return BB;
6852 }
6853 case AMDGPU::V_ADD_CO_U32_e32:
6854 case AMDGPU::V_SUB_CO_U32_e32:
6855 case AMDGPU::V_SUBREV_CO_U32_e32: {
6856 // TODO: Define distinct V_*_I32_Pseudo instructions instead.
6857 unsigned Opc = MI.getOpcode();
6858
6859 bool NeedClampOperand = false;
6860 if (TII->pseudoToMCOpcode(Opcode: Opc) == -1) {
6861 Opc = AMDGPU::getVOPe64(Opcode: Opc);
6862 NeedClampOperand = true;
6863 }
6864
6865 auto I = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: MI.getOperand(i: 0).getReg());
6866 if (TII->isVOP3(MI: *I)) {
6867 I.addReg(RegNo: TRI->getVCC(), Flags: RegState::Define);
6868 }
6869 I.add(MO: MI.getOperand(i: 1)).add(MO: MI.getOperand(i: 2));
6870 if (NeedClampOperand)
6871 I.addImm(Val: 0); // clamp bit for e64 encoding
6872
6873 TII->legalizeOperands(MI&: *I);
6874
6875 MI.eraseFromParent();
6876 return BB;
6877 }
6878 case AMDGPU::V_ADDC_U32_e32:
6879 case AMDGPU::V_SUBB_U32_e32:
6880 case AMDGPU::V_SUBBREV_U32_e32:
6881 // These instructions have an implicit use of vcc which counts towards the
6882 // constant bus limit.
6883 TII->legalizeOperands(MI);
6884 return BB;
6885 case AMDGPU::DS_GWS_INIT:
6886 case AMDGPU::DS_GWS_SEMA_BR:
6887 case AMDGPU::DS_GWS_BARRIER:
6888 case AMDGPU::DS_GWS_SEMA_V:
6889 case AMDGPU::DS_GWS_SEMA_P:
6890 case AMDGPU::DS_GWS_SEMA_RELEASE_ALL:
6891 // A s_waitcnt 0 is required to be the instruction immediately following.
6892 if (getSubtarget()->hasGWSAutoReplay()) {
6893 bundleInstWithWaitcnt(MI);
6894 return BB;
6895 }
6896
6897 return emitGWSMemViolTestLoop(MI, BB);
6898 case AMDGPU::S_SETREG_B32: {
6899 // Try to optimize cases that only set the denormal mode or rounding mode.
6900 //
6901 // If the s_setreg_b32 fully sets all of the bits in the rounding mode or
6902 // denormal mode to a constant, we can use s_round_mode or s_denorm_mode
6903 // instead.
6904 //
6905 // FIXME: This could be predicates on the immediate, but tablegen doesn't
6906 // allow you to have a no side effect instruction in the output of a
6907 // sideeffecting pattern.
6908 auto [ID, Offset, Width] =
6909 AMDGPU::Hwreg::HwregEncoding::decode(Encoded: MI.getOperand(i: 1).getImm());
6910 if (ID != AMDGPU::Hwreg::ID_MODE)
6911 return BB;
6912
6913 const unsigned WidthMask = maskTrailingOnes<unsigned>(N: Width);
6914 const unsigned SetMask = WidthMask << Offset;
6915
6916 if (getSubtarget()->hasDenormModeInst()) {
6917 unsigned SetDenormOp = 0;
6918 unsigned SetRoundOp = 0;
6919
6920 // The dedicated instructions can only set the whole denorm or round mode
6921 // at once, not a subset of bits in either.
6922 if (SetMask ==
6923 (AMDGPU::Hwreg::FP_ROUND_MASK | AMDGPU::Hwreg::FP_DENORM_MASK)) {
6924 // If this fully sets both the round and denorm mode, emit the two
6925 // dedicated instructions for these.
6926 SetRoundOp = AMDGPU::S_ROUND_MODE;
6927 SetDenormOp = AMDGPU::S_DENORM_MODE;
6928 } else if (SetMask == AMDGPU::Hwreg::FP_ROUND_MASK) {
6929 SetRoundOp = AMDGPU::S_ROUND_MODE;
6930 } else if (SetMask == AMDGPU::Hwreg::FP_DENORM_MASK) {
6931 SetDenormOp = AMDGPU::S_DENORM_MODE;
6932 }
6933
6934 if (SetRoundOp || SetDenormOp) {
6935 MachineInstr *Def = MRI.getVRegDef(Reg: MI.getOperand(i: 0).getReg());
6936 if (Def && Def->isMoveImmediate() && Def->getOperand(i: 1).isImm()) {
6937 unsigned ImmVal = Def->getOperand(i: 1).getImm();
6938 if (SetRoundOp) {
6939 BuildMI(BB&: *BB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SetRoundOp))
6940 .addImm(Val: ImmVal & 0xf);
6941
6942 // If we also have the denorm mode, get just the denorm mode bits.
6943 ImmVal >>= 4;
6944 }
6945
6946 if (SetDenormOp) {
6947 BuildMI(BB&: *BB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SetDenormOp))
6948 .addImm(Val: ImmVal & 0xf);
6949 }
6950
6951 MI.eraseFromParent();
6952 return BB;
6953 }
6954 }
6955 }
6956
6957 // If only FP bits are touched, used the no side effects pseudo.
6958 if ((SetMask & (AMDGPU::Hwreg::FP_ROUND_MASK |
6959 AMDGPU::Hwreg::FP_DENORM_MASK)) == SetMask)
6960 MI.setDesc(TII->get(Opcode: AMDGPU::S_SETREG_B32_mode));
6961
6962 return BB;
6963 }
6964 case AMDGPU::S_INVERSE_BALLOT_U32:
6965 case AMDGPU::S_INVERSE_BALLOT_U64:
6966 // These opcodes only exist to let SIFixSGPRCopies insert a readfirstlane if
6967 // necessary. After that they are equivalent to a COPY.
6968 MI.setDesc(TII->get(Opcode: AMDGPU::COPY));
6969 return BB;
6970 case AMDGPU::ENDPGM_TRAP: {
6971 if (BB->succ_empty() && std::next(x: MI.getIterator()) == BB->end()) {
6972 MI.setDesc(TII->get(Opcode: AMDGPU::S_ENDPGM));
6973 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
6974 return BB;
6975 }
6976
6977 // We need a block split to make the real endpgm a terminator. We also don't
6978 // want to break phis in successor blocks, so we can't just delete to the
6979 // end of the block.
6980
6981 MachineBasicBlock *SplitBB = BB->splitAt(SplitInst&: MI, UpdateLiveIns: false /*UpdateLiveIns*/);
6982 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
6983 MF->push_back(MBB: TrapBB);
6984 // clang-format off
6985 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ENDPGM))
6986 .addImm(Val: 0);
6987 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_EXECNZ))
6988 .addMBB(MBB: TrapBB);
6989 // clang-format on
6990
6991 BB->addSuccessor(Succ: TrapBB);
6992 MI.eraseFromParent();
6993 return SplitBB;
6994 }
6995 case AMDGPU::SIMULATED_TRAP: {
6996 assert(Subtarget->hasPrivEnabledTrap2NopBug());
6997 MachineBasicBlock *SplitBB =
6998 TII->insertSimulatedTrap(MRI, MBB&: *BB, MI, DL: MI.getDebugLoc());
6999 MI.eraseFromParent();
7000 return SplitBB;
7001 }
7002 case AMDGPU::SI_TCRETURN_GFX_WholeWave:
7003 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN: {
7004 assert(MFI->isWholeWaveFunction());
7005
7006 // During ISel, it's difficult to propagate the original EXEC mask to use as
7007 // an input to SI_WHOLE_WAVE_FUNC_RETURN. Set it up here instead.
7008 MachineInstr *Setup = TII->getWholeWaveFunctionSetup(MF&: *BB->getParent());
7009 assert(Setup && "Couldn't find SI_SETUP_WHOLE_WAVE_FUNC");
7010 Register OriginalExec = Setup->getOperand(i: 0).getReg();
7011 MF->getRegInfo().clearKillFlags(Reg: OriginalExec);
7012 MI.getOperand(i: 0).setReg(OriginalExec);
7013 return BB;
7014 }
7015 default:
7016 if (TII->isImage(MI) || TII->isMUBUF(MI)) {
7017 if (!MI.mayStore())
7018 AddMemOpInit(MI);
7019 return BB;
7020 }
7021 return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, MBB: BB);
7022 }
7023}
7024
7025bool SITargetLowering::enableAggressiveFMAFusion(EVT VT) const {
7026 // This currently forces unfolding various combinations of fsub into fma with
7027 // free fneg'd operands. As long as we have fast FMA (controlled by
7028 // isFMAFasterThanFMulAndFAdd), we should perform these.
7029
7030 // When fma is quarter rate, for f64 where add / sub are at best half rate,
7031 // most of these combines appear to be cycle neutral but save on instruction
7032 // count / code size.
7033 return true;
7034}
7035
7036bool SITargetLowering::enableAggressiveFMAFusion(LLT Ty) const { return true; }
7037
7038EVT SITargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx,
7039 EVT VT) const {
7040 if (!VT.isVector()) {
7041 return MVT::i1;
7042 }
7043 return EVT::getVectorVT(Context&: Ctx, VT: MVT::i1, NumElements: VT.getVectorNumElements());
7044}
7045
7046MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT VT) const {
7047 // TODO: Should i16 be used always if legal? For now it would force VALU
7048 // shifts.
7049 return (VT == MVT::i16) ? MVT::i16 : MVT::i32;
7050}
7051
7052LLT SITargetLowering::getPreferredShiftAmountTy(LLT Ty) const {
7053 return (Ty.getScalarSizeInBits() <= 16 && Subtarget->has16BitInsts())
7054 ? Ty.changeElementSize(NewEltSize: 16)
7055 : Ty.changeElementSize(NewEltSize: 32);
7056}
7057
7058// Answering this is somewhat tricky and depends on the specific device which
7059// have different rates for fma or all f64 operations.
7060//
7061// v_fma_f64 and v_mul_f64 always take the same number of cycles as each other
7062// regardless of which device (although the number of cycles differs between
7063// devices), so it is always profitable for f64.
7064//
7065// v_fma_f32 takes 4 or 16 cycles depending on the device, so it is profitable
7066// only on full rate devices. Normally, we should prefer selecting v_mad_f32
7067// which we can always do even without fused FP ops since it returns the same
7068// result as the separate operations and since it is always full
7069// rate. Therefore, we lie and report that it is not faster for f32. v_mad_f32
7070// however does not support denormals, so we do report fma as faster if we have
7071// a fast fma device and require denormals.
7072//
7073bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
7074 EVT VT) const {
7075 VT = VT.getScalarType();
7076
7077 switch (VT.getSimpleVT().SimpleTy) {
7078 case MVT::f32: {
7079 // If mad is not available this depends only on if f32 fma is full rate.
7080 if (!Subtarget->hasMadMacF32Insts())
7081 return Subtarget->hasFastFMAF32();
7082
7083 // Otherwise f32 mad is always full rate and returns the same result as
7084 // the separate operations so should be preferred over fma.
7085 // However does not support denormals.
7086 if (!denormalModeIsFlushAllF32(MF))
7087 return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts();
7088
7089 // If the subtarget has v_fmac_f32, that's just as good as v_mac_f32.
7090 return Subtarget->hasFastFMAF32() && Subtarget->hasDLInsts();
7091 }
7092 case MVT::f64:
7093 return true;
7094 case MVT::f16:
7095 case MVT::bf16:
7096 return Subtarget->has16BitInsts() && !denormalModeIsFlushAllF64F16(MF);
7097 default:
7098 break;
7099 }
7100
7101 return false;
7102}
7103
7104bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
7105 LLT Ty) const {
7106 switch (Ty.getScalarSizeInBits()) {
7107 case 16:
7108 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f16);
7109 case 32:
7110 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f32);
7111 case 64:
7112 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f64);
7113 default:
7114 break;
7115 }
7116
7117 return false;
7118}
7119
7120bool SITargetLowering::isFMADLegal(const MachineInstr &MI, LLT Ty) const {
7121 if (!Ty.isScalar())
7122 return false;
7123
7124 if (Ty.getScalarSizeInBits() == 16)
7125 return Subtarget->hasMadF16() && denormalModeIsFlushAllF64F16(MF: *MI.getMF());
7126 if (Ty.getScalarSizeInBits() == 32)
7127 return Subtarget->hasMadMacF32Insts() &&
7128 denormalModeIsFlushAllF32(MF: *MI.getMF());
7129
7130 return false;
7131}
7132
7133bool SITargetLowering::isFMADLegal(const SelectionDAG &DAG,
7134 const SDNode *N) const {
7135 // TODO: Check future ftz flag
7136 // v_mad_f32/v_mac_f32 do not support denormals.
7137 EVT VT = N->getValueType(ResNo: 0);
7138 if (VT == MVT::f32)
7139 return Subtarget->hasMadMacF32Insts() &&
7140 denormalModeIsFlushAllF32(MF: DAG.getMachineFunction());
7141 if (VT == MVT::f16) {
7142 return Subtarget->hasMadF16() &&
7143 denormalModeIsFlushAllF64F16(MF: DAG.getMachineFunction());
7144 }
7145
7146 return false;
7147}
7148
7149//===----------------------------------------------------------------------===//
7150// Custom DAG Lowering Operations
7151//===----------------------------------------------------------------------===//
7152
7153// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
7154// wider vector type is legal.
7155SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op,
7156 SelectionDAG &DAG) const {
7157 unsigned Opc = Op.getOpcode();
7158 EVT VT = Op.getValueType();
7159 assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16 ||
7160 VT == MVT::v4f32 || VT == MVT::v8i16 || VT == MVT::v8f16 ||
7161 VT == MVT::v8bf16 || VT == MVT::v16i16 || VT == MVT::v16f16 ||
7162 VT == MVT::v16bf16 || VT == MVT::v8f32 || VT == MVT::v16f32 ||
7163 VT == MVT::v32f32 || VT == MVT::v32i16 || VT == MVT::v32f16 ||
7164 VT == MVT::v32bf16);
7165
7166 auto [Lo, Hi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
7167
7168 SDLoc SL(Op);
7169 SDValue OpLo = DAG.getNode(Opcode: Opc, DL: SL, VT: Lo.getValueType(), Operand: Lo, Flags: Op->getFlags());
7170 SDValue OpHi = DAG.getNode(Opcode: Opc, DL: SL, VT: Hi.getValueType(), Operand: Hi, Flags: Op->getFlags());
7171
7172 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7173}
7174
7175// Enable lowering of ROTR for vxi32 types. This is a workaround for a
7176// regression whereby extra unnecessary instructions were added to codegen
7177// for rotr operations, casued by legalising v2i32 or. This resulted in extra
7178// instructions to extract the result from the vector.
7179SDValue SITargetLowering::lowerROTR(SDValue Op, SelectionDAG &DAG) const {
7180 [[maybe_unused]] EVT VT = Op.getValueType();
7181
7182 assert((VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v8i32 ||
7183 VT == MVT::v16i32) &&
7184 "Unexpected ValueType.");
7185
7186 return DAG.UnrollVectorOp(N: Op.getNode());
7187}
7188
7189// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
7190// wider vector type is legal.
7191SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op,
7192 SelectionDAG &DAG) const {
7193 unsigned Opc = Op.getOpcode();
7194 EVT VT = Op.getValueType();
7195 assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16 ||
7196 VT == MVT::v4f32 || VT == MVT::v8i16 || VT == MVT::v8f16 ||
7197 VT == MVT::v8bf16 || VT == MVT::v16i16 || VT == MVT::v16f16 ||
7198 VT == MVT::v16bf16 || VT == MVT::v8f32 || VT == MVT::v16f32 ||
7199 VT == MVT::v32f32 || VT == MVT::v32i16 || VT == MVT::v32f16 ||
7200 VT == MVT::v32bf16);
7201
7202 auto [Lo0, Hi0] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
7203 auto [Lo1, Hi1] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 1);
7204
7205 SDLoc SL(Op);
7206
7207 SDValue OpLo =
7208 DAG.getNode(Opcode: Opc, DL: SL, VT: Lo0.getValueType(), N1: Lo0, N2: Lo1, Flags: Op->getFlags());
7209 SDValue OpHi =
7210 DAG.getNode(Opcode: Opc, DL: SL, VT: Hi0.getValueType(), N1: Hi0, N2: Hi1, Flags: Op->getFlags());
7211
7212 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7213}
7214
7215SDValue SITargetLowering::splitTernaryVectorOp(SDValue Op,
7216 SelectionDAG &DAG) const {
7217 unsigned Opc = Op.getOpcode();
7218 EVT VT = Op.getValueType();
7219 assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v8i16 ||
7220 VT == MVT::v8f16 || VT == MVT::v4f32 || VT == MVT::v16i16 ||
7221 VT == MVT::v16f16 || VT == MVT::v8f32 || VT == MVT::v16f32 ||
7222 VT == MVT::v32f32 || VT == MVT::v32f16 || VT == MVT::v32i16 ||
7223 VT == MVT::v4bf16 || VT == MVT::v8bf16 || VT == MVT::v16bf16 ||
7224 VT == MVT::v32bf16);
7225
7226 SDValue Op0 = Op.getOperand(i: 0);
7227 auto [Lo0, Hi0] = Op0.getValueType().isVector()
7228 ? DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0)
7229 : std::pair(Op0, Op0);
7230
7231 auto [Lo1, Hi1] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 1);
7232 auto [Lo2, Hi2] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 2);
7233
7234 SDLoc SL(Op);
7235 auto ResVT = DAG.GetSplitDestVTs(VT);
7236
7237 SDValue OpLo =
7238 DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT.first, N1: Lo0, N2: Lo1, N3: Lo2, Flags: Op->getFlags());
7239 SDValue OpHi =
7240 DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT.second, N1: Hi0, N2: Hi1, N3: Hi2, Flags: Op->getFlags());
7241
7242 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7243}
7244
7245SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
7246 switch (Op.getOpcode()) {
7247 default:
7248 return AMDGPUTargetLowering::LowerOperation(Op, DAG);
7249 case ISD::BRCOND:
7250 return LowerBRCOND(Op, DAG);
7251 case ISD::RETURNADDR:
7252 return LowerRETURNADDR(Op, DAG);
7253 case ISD::SPONENTRY:
7254 return LowerSPONENTRY(Op, DAG);
7255 case ISD::LOAD: {
7256 SDValue Result = LowerLOAD(Op, DAG);
7257 assert((!Result.getNode() || Result.getNode()->getNumValues() == 2) &&
7258 "Load should return a value and a chain");
7259 return Result;
7260 }
7261 case ISD::FSQRT: {
7262 EVT VT = Op.getValueType();
7263 if (VT == MVT::f32)
7264 return lowerFSQRTF32(Op, DAG);
7265 if (VT == MVT::f64)
7266 return lowerFSQRTF64(Op, DAG);
7267 return SDValue();
7268 }
7269 case ISD::FSIN:
7270 case ISD::FCOS:
7271 return LowerTrig(Op, DAG);
7272 case ISD::SELECT:
7273 return LowerSELECT(Op, DAG);
7274 case ISD::FDIV:
7275 return LowerFDIV(Op, DAG);
7276 case ISD::FFREXP:
7277 return LowerFFREXP(Op, DAG);
7278 case ISD::ATOMIC_CMP_SWAP:
7279 return LowerATOMIC_CMP_SWAP(Op, DAG);
7280 case ISD::STORE:
7281 return LowerSTORE(Op, DAG);
7282 case ISD::GlobalAddress: {
7283 MachineFunction &MF = DAG.getMachineFunction();
7284 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
7285 return LowerGlobalAddress(MFI, Op, DAG);
7286 }
7287 case ISD::ExternalSymbol:
7288 return LowerExternalSymbol(Op, DAG);
7289 case ISD::INTRINSIC_WO_CHAIN:
7290 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
7291 case ISD::INTRINSIC_W_CHAIN:
7292 return LowerINTRINSIC_W_CHAIN(Op, DAG);
7293 case ISD::INTRINSIC_VOID:
7294 return LowerINTRINSIC_VOID(Op, DAG);
7295 case ISD::ADDRSPACECAST:
7296 return lowerADDRSPACECAST(Op, DAG);
7297 case ISD::INSERT_SUBVECTOR:
7298 return lowerINSERT_SUBVECTOR(Op, DAG);
7299 case ISD::INSERT_VECTOR_ELT:
7300 return lowerINSERT_VECTOR_ELT(Op, DAG);
7301 case ISD::EXTRACT_VECTOR_ELT:
7302 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
7303 case ISD::VECTOR_SHUFFLE:
7304 return lowerVECTOR_SHUFFLE(Op, DAG);
7305 case ISD::SCALAR_TO_VECTOR:
7306 return lowerSCALAR_TO_VECTOR(Op, DAG);
7307 case ISD::BUILD_VECTOR:
7308 return lowerBUILD_VECTOR(Op, DAG);
7309 case ISD::FP_ROUND:
7310 case ISD::STRICT_FP_ROUND:
7311 return lowerFP_ROUND(Op, DAG);
7312 case ISD::TRAP:
7313 return lowerTRAP(Op, DAG);
7314 case ISD::DEBUGTRAP:
7315 return lowerDEBUGTRAP(Op, DAG);
7316 case ISD::ABS:
7317 case ISD::FABS:
7318 case ISD::FNEG:
7319 case ISD::FCANONICALIZE:
7320 case ISD::BSWAP:
7321 return splitUnaryVectorOp(Op, DAG);
7322 case ISD::FMINNUM:
7323 case ISD::FMAXNUM:
7324 return lowerFMINNUM_FMAXNUM(Op, DAG);
7325 case ISD::FMINIMUMNUM:
7326 case ISD::FMAXIMUMNUM:
7327 return lowerFMINIMUMNUM_FMAXIMUMNUM(Op, DAG);
7328 case ISD::FMINIMUM:
7329 case ISD::FMAXIMUM:
7330 return lowerFMINIMUM_FMAXIMUM(Op, DAG);
7331 case ISD::FLDEXP:
7332 case ISD::STRICT_FLDEXP:
7333 return lowerFLDEXP(Op, DAG);
7334 case ISD::FMA:
7335 return splitTernaryVectorOp(Op, DAG);
7336 case ISD::FP_TO_SINT:
7337 case ISD::FP_TO_UINT:
7338 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX11 &&
7339 Op.getValueType() == MVT::i16 &&
7340 Op.getOperand(i: 0).getValueType() == MVT::f32) {
7341 // Make f32->i16 legal so we can select V_CVT_PK_[IU]16_F32.
7342 return Op;
7343 }
7344 return LowerFP_TO_INT(Op, DAG);
7345 case ISD::SHL:
7346 case ISD::SRA:
7347 case ISD::SRL:
7348 case ISD::ADD:
7349 case ISD::SUB:
7350 case ISD::SMIN:
7351 case ISD::SMAX:
7352 case ISD::UMIN:
7353 case ISD::UMAX:
7354 case ISD::FADD:
7355 case ISD::FMUL:
7356 case ISD::FMINNUM_IEEE:
7357 case ISD::FMAXNUM_IEEE:
7358 case ISD::UADDSAT:
7359 case ISD::USUBSAT:
7360 case ISD::SADDSAT:
7361 case ISD::SSUBSAT:
7362 return splitBinaryVectorOp(Op, DAG);
7363 case ISD::FCOPYSIGN:
7364 return lowerFCOPYSIGN(Op, DAG);
7365 case ISD::MUL:
7366 return lowerMUL(Op, DAG);
7367 case ISD::SMULO:
7368 case ISD::UMULO:
7369 return lowerXMULO(Op, DAG);
7370 case ISD::SMUL_LOHI:
7371 case ISD::UMUL_LOHI:
7372 return lowerXMUL_LOHI(Op, DAG);
7373 case ISD::DYNAMIC_STACKALLOC:
7374 return LowerDYNAMIC_STACKALLOC(Op, DAG);
7375 case ISD::STACKSAVE:
7376 return LowerSTACKSAVE(Op, DAG);
7377 case ISD::GET_ROUNDING:
7378 return lowerGET_ROUNDING(Op, DAG);
7379 case ISD::SET_ROUNDING:
7380 return lowerSET_ROUNDING(Op, DAG);
7381 case ISD::PREFETCH:
7382 return lowerPREFETCH(Op, DAG);
7383 case ISD::FP_EXTEND:
7384 case ISD::STRICT_FP_EXTEND:
7385 return lowerFP_EXTEND(Op, DAG);
7386 case ISD::GET_FPENV:
7387 return lowerGET_FPENV(Op, DAG);
7388 case ISD::SET_FPENV:
7389 return lowerSET_FPENV(Op, DAG);
7390 case ISD::ROTR:
7391 return lowerROTR(Op, DAG);
7392 }
7393 return SDValue();
7394}
7395
7396// Used for D16: Casts the result of an instruction into the right vector,
7397// packs values if loads return unpacked values.
7398static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
7399 const SDLoc &DL, SelectionDAG &DAG,
7400 bool Unpacked) {
7401 if (!LoadVT.isVector())
7402 return Result;
7403
7404 // Cast back to the original packed type or to a larger type that is a
7405 // multiple of 32 bit for D16. Widening the return type is a required for
7406 // legalization.
7407 EVT FittingLoadVT = LoadVT;
7408 if ((LoadVT.getVectorNumElements() % 2) == 1) {
7409 FittingLoadVT =
7410 EVT::getVectorVT(Context&: *DAG.getContext(), VT: LoadVT.getVectorElementType(),
7411 NumElements: LoadVT.getVectorNumElements() + 1);
7412 }
7413
7414 if (Unpacked) { // From v2i32/v4i32 back to v2f16/v4f16.
7415 // Truncate to v2i16/v4i16.
7416 EVT IntLoadVT = FittingLoadVT.changeTypeToInteger();
7417
7418 // Workaround legalizer not scalarizing truncate after vector op
7419 // legalization but not creating intermediate vector trunc.
7420 SmallVector<SDValue, 4> Elts;
7421 DAG.ExtractVectorElements(Op: Result, Args&: Elts);
7422 for (SDValue &Elt : Elts)
7423 Elt = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Elt);
7424
7425 // Pad illegal v1i16/v3fi6 to v4i16
7426 if ((LoadVT.getVectorNumElements() % 2) == 1)
7427 Elts.push_back(Elt: DAG.getPOISON(VT: MVT::i16));
7428
7429 Result = DAG.getBuildVector(VT: IntLoadVT, DL, Ops: Elts);
7430
7431 // Bitcast to original type (v2f16/v4f16).
7432 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FittingLoadVT, Operand: Result);
7433 }
7434
7435 // Cast back to the original packed type.
7436 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FittingLoadVT, Operand: Result);
7437}
7438
7439SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, MemSDNode *M,
7440 SelectionDAG &DAG,
7441 ArrayRef<SDValue> Ops,
7442 bool IsIntrinsic) const {
7443 SDLoc DL(M);
7444
7445 bool Unpacked = Subtarget->hasUnpackedD16VMem();
7446 EVT LoadVT = M->getValueType(ResNo: 0);
7447
7448 EVT EquivLoadVT = LoadVT;
7449 if (LoadVT.isVector()) {
7450 if (Unpacked) {
7451 EquivLoadVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
7452 NumElements: LoadVT.getVectorNumElements());
7453 } else if ((LoadVT.getVectorNumElements() % 2) == 1) {
7454 // Widen v3f16 to legal type
7455 EquivLoadVT =
7456 EVT::getVectorVT(Context&: *DAG.getContext(), VT: LoadVT.getVectorElementType(),
7457 NumElements: LoadVT.getVectorNumElements() + 1);
7458 }
7459 }
7460
7461 // Change from v4f16/v2f16 to EquivLoadVT.
7462 SDVTList VTList = DAG.getVTList(VT1: EquivLoadVT, VT2: MVT::Other);
7463
7464 SDValue Load = DAG.getMemIntrinsicNode(
7465 Opcode: IsIntrinsic ? (unsigned)ISD::INTRINSIC_W_CHAIN : Opcode, dl: DL, VTList, Ops,
7466 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
7467
7468 SDValue Adjusted = adjustLoadValueTypeImpl(Result: Load, LoadVT, DL, DAG, Unpacked);
7469
7470 return DAG.getMergeValues(Ops: {Adjusted, Load.getValue(R: 1)}, dl: DL);
7471}
7472
7473SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
7474 SelectionDAG &DAG,
7475 ArrayRef<SDValue> Ops) const {
7476 SDLoc DL(M);
7477 EVT LoadVT = M->getValueType(ResNo: 0);
7478 EVT EltType = LoadVT.getScalarType();
7479 EVT IntVT = LoadVT.changeTypeToInteger();
7480
7481 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
7482
7483 assert(M->getNumValues() == 2 || M->getNumValues() == 3);
7484 bool IsTFE = M->getNumValues() == 3;
7485
7486 unsigned Opc = IsFormat ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_TFE
7487 : AMDGPUISD::BUFFER_LOAD_FORMAT)
7488 : IsTFE ? AMDGPUISD::BUFFER_LOAD_TFE
7489 : AMDGPUISD::BUFFER_LOAD;
7490
7491 if (IsD16) {
7492 return adjustLoadValueType(Opcode: AMDGPUISD::BUFFER_LOAD_FORMAT_D16, M, DAG, Ops);
7493 }
7494
7495 // Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics
7496 if (!IsD16 && !LoadVT.isVector() && EltType.getSizeInBits() < 32)
7497 return handleByteShortBufferLoads(DAG, LoadVT, DL, Ops, MMO: M->getMemOperand(),
7498 IsTFE);
7499
7500 if (isTypeLegal(VT: LoadVT)) {
7501 return getMemIntrinsicNode(Opcode: Opc, DL, VTList: M->getVTList(), Ops, MemVT: IntVT,
7502 MMO: M->getMemOperand(), DAG);
7503 }
7504
7505 EVT CastVT = getEquivalentMemType(Context&: *DAG.getContext(), VT: LoadVT);
7506 SDVTList VTList = DAG.getVTList(VT1: CastVT, VT2: MVT::Other);
7507 SDValue MemNode = getMemIntrinsicNode(Opcode: Opc, DL, VTList, Ops, MemVT: CastVT,
7508 MMO: M->getMemOperand(), DAG);
7509 return DAG.getMergeValues(
7510 Ops: {DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: MemNode), MemNode.getValue(R: 1)},
7511 dl: DL);
7512}
7513
7514static SDValue lowerICMPIntrinsic(const SITargetLowering &TLI, SDNode *N,
7515 SelectionDAG &DAG) {
7516 EVT VT = N->getValueType(ResNo: 0);
7517 unsigned CondCode = N->getConstantOperandVal(Num: 3);
7518 if (!ICmpInst::isIntPredicate(P: static_cast<ICmpInst::Predicate>(CondCode)))
7519 return DAG.getPOISON(VT);
7520
7521 ICmpInst::Predicate IcInput = static_cast<ICmpInst::Predicate>(CondCode);
7522
7523 SDValue LHS = N->getOperand(Num: 1);
7524 SDValue RHS = N->getOperand(Num: 2);
7525
7526 SDLoc DL(N);
7527
7528 EVT CmpVT = LHS.getValueType();
7529 if (CmpVT == MVT::i16 && !TLI.isTypeLegal(VT: MVT::i16)) {
7530 unsigned PromoteOp =
7531 ICmpInst::isSigned(Pred: IcInput) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
7532 LHS = DAG.getNode(Opcode: PromoteOp, DL, VT: MVT::i32, Operand: LHS);
7533 RHS = DAG.getNode(Opcode: PromoteOp, DL, VT: MVT::i32, Operand: RHS);
7534 }
7535
7536 ISD::CondCode CCOpcode = getICmpCondCode(Pred: IcInput);
7537
7538 unsigned WavefrontSize = TLI.getSubtarget()->getWavefrontSize();
7539 EVT CCVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: WavefrontSize);
7540
7541 SDValue SetCC = DAG.getNode(Opcode: AMDGPUISD::SETCC, DL, VT: CCVT, N1: LHS, N2: RHS,
7542 N3: DAG.getCondCode(Cond: CCOpcode));
7543 if (VT.bitsEq(VT: CCVT))
7544 return SetCC;
7545 return DAG.getZExtOrTrunc(Op: SetCC, DL, VT);
7546}
7547
7548static SDValue lowerFCMPIntrinsic(const SITargetLowering &TLI, SDNode *N,
7549 SelectionDAG &DAG) {
7550 EVT VT = N->getValueType(ResNo: 0);
7551
7552 unsigned CondCode = N->getConstantOperandVal(Num: 3);
7553 if (!FCmpInst::isFPPredicate(P: static_cast<FCmpInst::Predicate>(CondCode)))
7554 return DAG.getPOISON(VT);
7555
7556 SDValue Src0 = N->getOperand(Num: 1);
7557 SDValue Src1 = N->getOperand(Num: 2);
7558 EVT CmpVT = Src0.getValueType();
7559 SDLoc SL(N);
7560
7561 if (CmpVT == MVT::f16 && !TLI.isTypeLegal(VT: CmpVT)) {
7562 Src0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Src0);
7563 Src1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Src1);
7564 }
7565
7566 FCmpInst::Predicate IcInput = static_cast<FCmpInst::Predicate>(CondCode);
7567 ISD::CondCode CCOpcode = getFCmpCondCode(Pred: IcInput);
7568 unsigned WavefrontSize = TLI.getSubtarget()->getWavefrontSize();
7569 EVT CCVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: WavefrontSize);
7570 SDValue SetCC = DAG.getNode(Opcode: AMDGPUISD::SETCC, DL: SL, VT: CCVT, N1: Src0, N2: Src1,
7571 N3: DAG.getCondCode(Cond: CCOpcode));
7572 if (VT.bitsEq(VT: CCVT))
7573 return SetCC;
7574 return DAG.getZExtOrTrunc(Op: SetCC, DL: SL, VT);
7575}
7576
7577static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N,
7578 SelectionDAG &DAG) {
7579 EVT VT = N->getValueType(ResNo: 0);
7580 SDValue Src = N->getOperand(Num: 1);
7581 SDLoc SL(N);
7582
7583 if (Src.getOpcode() == ISD::SETCC) {
7584 SDValue Op0 = Src.getOperand(i: 0);
7585 SDValue Op1 = Src.getOperand(i: 1);
7586 // Need to expand bfloat to float for comparison (setcc).
7587 if (Op0.getValueType() == MVT::bf16) {
7588 Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op0);
7589 Op1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op1);
7590 }
7591 // (ballot (ISD::SETCC ...)) -> (AMDGPUISD::SETCC ...)
7592 return DAG.getNode(Opcode: AMDGPUISD::SETCC, DL: SL, VT, N1: Op0, N2: Op1, N3: Src.getOperand(i: 2));
7593 }
7594 if (const ConstantSDNode *Arg = dyn_cast<ConstantSDNode>(Val&: Src)) {
7595 // (ballot 0) -> 0
7596 if (Arg->isZero())
7597 return DAG.getConstant(Val: 0, DL: SL, VT);
7598
7599 // (ballot 1) -> EXEC/EXEC_LO
7600 if (Arg->isOne()) {
7601 Register Exec;
7602 if (VT.getScalarSizeInBits() == 32)
7603 Exec = AMDGPU::EXEC_LO;
7604 else if (VT.getScalarSizeInBits() == 64)
7605 Exec = AMDGPU::EXEC;
7606 else
7607 return SDValue();
7608
7609 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: Exec, VT);
7610 }
7611 }
7612
7613 // (ballot (i1 $src)) -> (AMDGPUISD::SETCC (i32 (zext $src)) (i32 0)
7614 // ISD::SETNE)
7615 return DAG.getNode(
7616 Opcode: AMDGPUISD::SETCC, DL: SL, VT, N1: DAG.getZExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32),
7617 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), N3: DAG.getCondCode(Cond: ISD::SETNE));
7618}
7619
7620static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N,
7621 SelectionDAG &DAG) {
7622 EVT VT = N->getValueType(ResNo: 0);
7623 unsigned ValSize = VT.getSizeInBits();
7624 unsigned IID = N->getConstantOperandVal(Num: 0);
7625 bool IsPermLane16 = IID == Intrinsic::amdgcn_permlane16 ||
7626 IID == Intrinsic::amdgcn_permlanex16;
7627 bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
7628 IID == Intrinsic::amdgcn_set_inactive_chain_arg;
7629 SDLoc SL(N);
7630 MVT IntVT = MVT::getIntegerVT(BitWidth: ValSize);
7631 const GCNSubtarget *ST = TLI.getSubtarget();
7632 unsigned SplitSize = 32;
7633 if (IID == Intrinsic::amdgcn_update_dpp && (ValSize % 64 == 0) &&
7634 ST->hasDPALU_DPP() &&
7635 AMDGPU::isLegalDPALU_DPPControl(ST: *ST, DC: N->getConstantOperandVal(Num: 3)))
7636 SplitSize = 64;
7637
7638 auto createLaneOp = [&DAG, &SL, N, IID](SDValue Src0, SDValue Src1,
7639 SDValue Src2, MVT ValT) -> SDValue {
7640 SmallVector<SDValue, 8> Operands;
7641 switch (IID) {
7642 case Intrinsic::amdgcn_permlane16:
7643 case Intrinsic::amdgcn_permlanex16:
7644 case Intrinsic::amdgcn_update_dpp:
7645 Operands.push_back(Elt: N->getOperand(Num: 6));
7646 Operands.push_back(Elt: N->getOperand(Num: 5));
7647 Operands.push_back(Elt: N->getOperand(Num: 4));
7648 [[fallthrough]];
7649 case Intrinsic::amdgcn_writelane:
7650 Operands.push_back(Elt: Src2);
7651 [[fallthrough]];
7652 case Intrinsic::amdgcn_readlane:
7653 case Intrinsic::amdgcn_set_inactive:
7654 case Intrinsic::amdgcn_set_inactive_chain_arg:
7655 case Intrinsic::amdgcn_mov_dpp8:
7656 Operands.push_back(Elt: Src1);
7657 [[fallthrough]];
7658 case Intrinsic::amdgcn_readfirstlane:
7659 case Intrinsic::amdgcn_permlane64:
7660 Operands.push_back(Elt: Src0);
7661 break;
7662 default:
7663 llvm_unreachable("unhandled lane op");
7664 }
7665
7666 Operands.push_back(Elt: DAG.getTargetConstant(Val: IID, DL: SL, VT: MVT::i32));
7667 std::reverse(first: Operands.begin(), last: Operands.end());
7668
7669 if (SDNode *GL = N->getGluedNode()) {
7670 assert(GL->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
7671 GL = GL->getOperand(Num: 0).getNode();
7672 Operands.push_back(Elt: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL: SL, VT: MVT::Glue,
7673 Operand: SDValue(GL, 0)));
7674 }
7675
7676 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: ValT, Ops: Operands);
7677 };
7678
7679 SDValue Src0 = N->getOperand(Num: 1);
7680 SDValue Src1, Src2;
7681 if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
7682 IID == Intrinsic::amdgcn_mov_dpp8 ||
7683 IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) {
7684 Src1 = N->getOperand(Num: 2);
7685 if (IID == Intrinsic::amdgcn_writelane ||
7686 IID == Intrinsic::amdgcn_update_dpp || IsPermLane16)
7687 Src2 = N->getOperand(Num: 3);
7688 }
7689
7690 if (ValSize == SplitSize) {
7691 // Already legal
7692 return SDValue();
7693 }
7694
7695 if (ValSize < 32) {
7696 bool IsFloat = VT.isFloatingPoint();
7697 Src0 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src0) : Src0,
7698 DL: SL, VT: MVT::i32);
7699
7700 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) {
7701 Src1 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src1) : Src1,
7702 DL: SL, VT: MVT::i32);
7703 }
7704
7705 if (IID == Intrinsic::amdgcn_writelane) {
7706 Src2 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src2) : Src2,
7707 DL: SL, VT: MVT::i32);
7708 }
7709
7710 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, MVT::i32);
7711 SDValue Trunc = DAG.getAnyExtOrTrunc(Op: LaneOp, DL: SL, VT: IntVT);
7712 return IsFloat ? DAG.getBitcast(VT, V: Trunc) : Trunc;
7713 }
7714
7715 if (ValSize % SplitSize != 0)
7716 return SDValue();
7717
7718 auto unrollLaneOp = [&DAG, &SL](SDNode *N) -> SDValue {
7719 EVT VT = N->getValueType(ResNo: 0);
7720 unsigned NE = VT.getVectorNumElements();
7721 EVT EltVT = VT.getVectorElementType();
7722 SmallVector<SDValue, 8> Scalars;
7723 unsigned NumOperands = N->getNumOperands();
7724 SmallVector<SDValue, 4> Operands(NumOperands);
7725 SDNode *GL = N->getGluedNode();
7726
7727 // only handle convergencectrl_glue
7728 assert(!GL || GL->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
7729
7730 for (unsigned i = 0; i != NE; ++i) {
7731 for (unsigned j = 0, e = GL ? NumOperands - 1 : NumOperands; j != e;
7732 ++j) {
7733 SDValue Operand = N->getOperand(Num: j);
7734 EVT OperandVT = Operand.getValueType();
7735 if (OperandVT.isVector()) {
7736 // A vector operand; extract a single element.
7737 EVT OperandEltVT = OperandVT.getVectorElementType();
7738 Operands[j] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: OperandEltVT,
7739 N1: Operand, N2: DAG.getVectorIdxConstant(Val: i, DL: SL));
7740 } else {
7741 // A scalar operand; just use it as is.
7742 Operands[j] = Operand;
7743 }
7744 }
7745
7746 if (GL)
7747 Operands[NumOperands - 1] =
7748 DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL: SL, VT: MVT::Glue,
7749 Operand: SDValue(GL->getOperand(Num: 0).getNode(), 0));
7750
7751 Scalars.push_back(Elt: DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: EltVT, Ops: Operands));
7752 }
7753
7754 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NE);
7755 return DAG.getBuildVector(VT: VecVT, DL: SL, Ops: Scalars);
7756 };
7757
7758 if (VT.isVector()) {
7759 switch (MVT::SimpleValueType EltTy =
7760 VT.getVectorElementType().getSimpleVT().SimpleTy) {
7761 case MVT::i32:
7762 case MVT::f32:
7763 if (SplitSize == 32) {
7764 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VT.getSimpleVT());
7765 return unrollLaneOp(LaneOp.getNode());
7766 }
7767 [[fallthrough]];
7768 case MVT::i16:
7769 case MVT::f16:
7770 case MVT::bf16: {
7771 unsigned SubVecNumElt =
7772 SplitSize / VT.getVectorElementType().getSizeInBits();
7773 MVT SubVecVT = MVT::getVectorVT(VT: EltTy, NumElements: SubVecNumElt);
7774 SmallVector<SDValue, 4> Pieces;
7775 SDValue Src0SubVec, Src1SubVec, Src2SubVec;
7776 for (unsigned i = 0, EltIdx = 0; i < ValSize / SplitSize; i++) {
7777 Src0SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src0,
7778 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
7779
7780 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive ||
7781 IsPermLane16)
7782 Src1SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src1,
7783 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
7784
7785 if (IID == Intrinsic::amdgcn_writelane)
7786 Src2SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src2,
7787 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
7788
7789 Pieces.push_back(
7790 Elt: IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16
7791 ? createLaneOp(Src0SubVec, Src1SubVec, Src2, SubVecVT)
7792 : createLaneOp(Src0SubVec, Src1, Src2SubVec, SubVecVT));
7793 EltIdx += SubVecNumElt;
7794 }
7795 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT, Ops: Pieces);
7796 }
7797 default:
7798 // Handle all other cases by bitcasting to i32 vectors
7799 break;
7800 }
7801 }
7802
7803 MVT VecVT =
7804 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplitSize), NumElements: ValSize / SplitSize);
7805 Src0 = DAG.getBitcast(VT: VecVT, V: Src0);
7806
7807 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
7808 Src1 = DAG.getBitcast(VT: VecVT, V: Src1);
7809
7810 if (IID == Intrinsic::amdgcn_writelane)
7811 Src2 = DAG.getBitcast(VT: VecVT, V: Src2);
7812
7813 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VecVT);
7814 SDValue UnrolledLaneOp = unrollLaneOp(LaneOp.getNode());
7815 return DAG.getBitcast(VT, V: UnrolledLaneOp);
7816}
7817
7818static SDValue lowerWaveShuffle(const SITargetLowering &TLI, SDNode *N,
7819 SelectionDAG &DAG) {
7820 EVT VT = N->getValueType(ResNo: 0);
7821
7822 if (VT.getSizeInBits() != 32)
7823 return SDValue();
7824
7825 SDLoc SL(N);
7826
7827 SDValue Value = N->getOperand(Num: 1);
7828 SDValue Index = N->getOperand(Num: 2);
7829
7830 // ds_bpermute requires index to be multiplied by 4
7831 SDValue ShiftAmount = DAG.getShiftAmountConstant(Val: 2, VT: MVT::i32, DL: SL);
7832 SDValue ShiftedIndex =
7833 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: Index.getValueType(), N1: Index, N2: ShiftAmount);
7834
7835 // Intrinsics will require i32 to operate on
7836 SDValue ValueI32 = DAG.getBitcast(VT: MVT::i32, V: Value);
7837
7838 auto MakeIntrinsic = [&DAG, &SL](unsigned IID, MVT RetVT,
7839 SmallVector<SDValue> IntrinArgs) -> SDValue {
7840 SmallVector<SDValue> Operands(1);
7841 Operands[0] = DAG.getTargetConstant(Val: IID, DL: SL, VT: MVT::i32);
7842 Operands.append(RHS: IntrinArgs);
7843 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: RetVT, Ops: Operands);
7844 };
7845
7846 // If we can bpermute across the whole wave, then just do that
7847 if (TLI.getSubtarget()->supportsWaveWideBPermute()) {
7848 SDValue BPermute = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
7849 {ShiftedIndex, ValueI32});
7850 return DAG.getBitcast(VT, V: BPermute);
7851 }
7852
7853 assert(TLI.getSubtarget()->isWave64());
7854
7855 // Otherwise, we need to make use of whole wave mode
7856 SDValue PoisonVal = DAG.getPOISON(VT: ValueI32->getValueType(ResNo: 0));
7857
7858 // Set inactive lanes to poison
7859 SDValue WWMValue = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
7860 {ValueI32, PoisonVal});
7861 SDValue WWMIndex = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
7862 {ShiftedIndex, PoisonVal});
7863
7864 SDValue Swapped =
7865 MakeIntrinsic(Intrinsic::amdgcn_permlane64, MVT::i32, {WWMValue});
7866
7867 // Get permutation of each half, then we'll select which one to use
7868 SDValue BPermSameHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
7869 {WWMIndex, WWMValue});
7870 SDValue BPermOtherHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute,
7871 MVT::i32, {WWMIndex, Swapped});
7872 SDValue BPermOtherHalfWWM =
7873 MakeIntrinsic(Intrinsic::amdgcn_wwm, MVT::i32, {BPermOtherHalf});
7874
7875 // Select which side to take the permute from
7876 SDValue ThreadIDMask = DAG.getAllOnesConstant(DL: SL, VT: MVT::i32);
7877 // We can get away with only using mbcnt_lo here since we're only
7878 // trying to detect which side of 32 each lane is on, and mbcnt_lo
7879 // returns 32 for lanes 32-63.
7880 SDValue ThreadID =
7881 MakeIntrinsic(Intrinsic::amdgcn_mbcnt_lo, MVT::i32,
7882 {ThreadIDMask, DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32)});
7883
7884 SDValue SameOrOtherHalf =
7885 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32,
7886 N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: ThreadID, N2: Index),
7887 N2: DAG.getTargetConstant(Val: 32, DL: SL, VT: MVT::i32));
7888 SDValue UseSameHalf =
7889 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: SameOrOtherHalf,
7890 RHS: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond: ISD::SETEQ);
7891 SDValue Result = DAG.getSelect(DL: SL, VT: MVT::i32, Cond: UseSameHalf, LHS: BPermSameHalf,
7892 RHS: BPermOtherHalfWWM);
7893 return DAG.getBitcast(VT, V: Result);
7894}
7895
7896void SITargetLowering::ReplaceNodeResults(SDNode *N,
7897 SmallVectorImpl<SDValue> &Results,
7898 SelectionDAG &DAG) const {
7899 switch (N->getOpcode()) {
7900 case ISD::INSERT_VECTOR_ELT: {
7901 if (SDValue Res = lowerINSERT_VECTOR_ELT(Op: SDValue(N, 0), DAG))
7902 Results.push_back(Elt: Res);
7903 return;
7904 }
7905 case ISD::EXTRACT_VECTOR_ELT: {
7906 if (SDValue Res = lowerEXTRACT_VECTOR_ELT(Op: SDValue(N, 0), DAG))
7907 Results.push_back(Elt: Res);
7908 return;
7909 }
7910 case ISD::INTRINSIC_WO_CHAIN: {
7911 unsigned IID = N->getConstantOperandVal(Num: 0);
7912 switch (IID) {
7913 case Intrinsic::amdgcn_make_buffer_rsrc:
7914 Results.push_back(Elt: lowerPointerAsRsrcIntrin(Op: N, DAG));
7915 return;
7916 case Intrinsic::amdgcn_cvt_pkrtz: {
7917 SDValue Src0 = N->getOperand(Num: 1);
7918 SDValue Src1 = N->getOperand(Num: 2);
7919 SDLoc SL(N);
7920 SDValue Cvt =
7921 DAG.getNode(Opcode: AMDGPUISD::CVT_PKRTZ_F16_F32, DL: SL, VT: MVT::i32, N1: Src0, N2: Src1);
7922 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Cvt));
7923 return;
7924 }
7925 case Intrinsic::amdgcn_cvt_pknorm_i16:
7926 case Intrinsic::amdgcn_cvt_pknorm_u16:
7927 case Intrinsic::amdgcn_cvt_pk_i16:
7928 case Intrinsic::amdgcn_cvt_pk_u16: {
7929 SDValue Src0 = N->getOperand(Num: 1);
7930 SDValue Src1 = N->getOperand(Num: 2);
7931 SDLoc SL(N);
7932 unsigned Opcode;
7933
7934 if (IID == Intrinsic::amdgcn_cvt_pknorm_i16)
7935 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
7936 else if (IID == Intrinsic::amdgcn_cvt_pknorm_u16)
7937 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
7938 else if (IID == Intrinsic::amdgcn_cvt_pk_i16)
7939 Opcode = AMDGPUISD::CVT_PK_I16_I32;
7940 else
7941 Opcode = AMDGPUISD::CVT_PK_U16_U32;
7942
7943 EVT VT = N->getValueType(ResNo: 0);
7944 if (isTypeLegal(VT))
7945 Results.push_back(Elt: DAG.getNode(Opcode, DL: SL, VT, N1: Src0, N2: Src1));
7946 else {
7947 SDValue Cvt = DAG.getNode(Opcode, DL: SL, VT: MVT::i32, N1: Src0, N2: Src1);
7948 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: Cvt));
7949 }
7950 return;
7951 }
7952 case Intrinsic::amdgcn_s_buffer_load: {
7953 // Lower llvm.amdgcn.s.buffer.load.(i8, u8) intrinsics. First, we generate
7954 // s_buffer_load_u8 for signed and unsigned load instructions. Next, DAG
7955 // combiner tries to merge the s_buffer_load_u8 with a sext instruction
7956 // (performSignExtendInRegCombine()) and it replaces s_buffer_load_u8 with
7957 // s_buffer_load_i8.
7958 if (!Subtarget->hasScalarSubwordLoads())
7959 return;
7960 SDValue Op = SDValue(N, 0);
7961 SDValue Rsrc = Op.getOperand(i: 1);
7962 SDValue Offset = Op.getOperand(i: 2);
7963 SDValue CachePolicy = Op.getOperand(i: 3);
7964 EVT VT = Op.getValueType();
7965 assert(VT == MVT::i8 && "Expected 8-bit s_buffer_load intrinsics.\n");
7966 SDLoc DL(Op);
7967 MachineFunction &MF = DAG.getMachineFunction();
7968 const DataLayout &DataLayout = DAG.getDataLayout();
7969 Align Alignment =
7970 DataLayout.getABITypeAlign(Ty: VT.getTypeForEVT(Context&: *DAG.getContext()));
7971 MachineMemOperand *MMO = MF.getMachineMemOperand(
7972 PtrInfo: MachinePointerInfo(),
7973 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
7974 MachineMemOperand::MOInvariant,
7975 Size: VT.getStoreSize(), BaseAlignment: Alignment);
7976 SDValue LoadVal;
7977 if (!Offset->isDivergent()) {
7978 SDValue Ops[] = {Rsrc, // source register
7979 Offset, CachePolicy};
7980 SDValue BufferLoad =
7981 DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::SBUFFER_LOAD_UBYTE, dl: DL,
7982 VTList: DAG.getVTList(VT: MVT::i32), Ops, MemVT: VT, MMO);
7983 LoadVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: BufferLoad);
7984 } else {
7985 SDValue Ops[] = {
7986 DAG.getEntryNode(), // Chain
7987 Rsrc, // rsrc
7988 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
7989 {}, // voffset
7990 {}, // soffset
7991 {}, // offset
7992 CachePolicy, // cachepolicy
7993 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
7994 };
7995 setBufferOffsets(CombinedOffset: Offset, DAG, Offsets: &Ops[3], Alignment: Align(4));
7996 LoadVal = handleByteShortBufferLoads(DAG, LoadVT: VT, DL, Ops, MMO);
7997 }
7998 Results.push_back(Elt: LoadVal);
7999 return;
8000 }
8001 case Intrinsic::amdgcn_dead: {
8002 for (unsigned I = 0, E = N->getNumValues(); I < E; ++I)
8003 Results.push_back(Elt: DAG.getPOISON(VT: N->getValueType(ResNo: I)));
8004 return;
8005 }
8006 }
8007 break;
8008 }
8009 case ISD::INTRINSIC_W_CHAIN: {
8010 if (SDValue Res = LowerINTRINSIC_W_CHAIN(Op: SDValue(N, 0), DAG)) {
8011 if (Res.getOpcode() == ISD::MERGE_VALUES) {
8012 // FIXME: Hacky
8013 for (unsigned I = 0; I < Res.getNumOperands(); I++) {
8014 Results.push_back(Elt: Res.getOperand(i: I));
8015 }
8016 } else {
8017 Results.push_back(Elt: Res);
8018 Results.push_back(Elt: Res.getValue(R: 1));
8019 }
8020 return;
8021 }
8022
8023 break;
8024 }
8025 case ISD::SELECT: {
8026 SDLoc SL(N);
8027 EVT VT = N->getValueType(ResNo: 0);
8028 EVT NewVT = getEquivalentMemType(Context&: *DAG.getContext(), VT);
8029 SDValue LHS = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: N->getOperand(Num: 1));
8030 SDValue RHS = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: N->getOperand(Num: 2));
8031
8032 EVT SelectVT = NewVT;
8033 if (NewVT.bitsLT(VT: MVT::i32)) {
8034 LHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: LHS);
8035 RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: RHS);
8036 SelectVT = MVT::i32;
8037 }
8038
8039 SDValue NewSelect =
8040 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: SelectVT, N1: N->getOperand(Num: 0), N2: LHS, N3: RHS);
8041
8042 if (NewVT != SelectVT)
8043 NewSelect = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: NewVT, Operand: NewSelect);
8044 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: NewSelect));
8045 return;
8046 }
8047 case ISD::FNEG: {
8048 if (N->getValueType(ResNo: 0) != MVT::v2f16)
8049 break;
8050
8051 SDLoc SL(N);
8052 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 0));
8053
8054 SDValue Op = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: BC,
8055 N2: DAG.getConstant(Val: 0x80008000, DL: SL, VT: MVT::i32));
8056 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Op));
8057 return;
8058 }
8059 case ISD::FABS: {
8060 if (N->getValueType(ResNo: 0) != MVT::v2f16)
8061 break;
8062
8063 SDLoc SL(N);
8064 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 0));
8065
8066 SDValue Op = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: BC,
8067 N2: DAG.getConstant(Val: 0x7fff7fff, DL: SL, VT: MVT::i32));
8068 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Op));
8069 return;
8070 }
8071 case ISD::FSQRT: {
8072 if (N->getValueType(ResNo: 0) != MVT::f16)
8073 break;
8074 Results.push_back(Elt: lowerFSQRTF16(Op: SDValue(N, 0), DAG));
8075 break;
8076 }
8077 default:
8078 AMDGPUTargetLowering::ReplaceNodeResults(N, Results, DAG);
8079 break;
8080 }
8081}
8082
8083/// Helper function for LowerBRCOND
8084static SDNode *findUser(SDValue Value, unsigned Opcode) {
8085
8086 for (SDUse &U : Value->uses()) {
8087 if (U.get() != Value)
8088 continue;
8089
8090 if (U.getUser()->getOpcode() == Opcode)
8091 return U.getUser();
8092 }
8093 return nullptr;
8094}
8095
8096unsigned SITargetLowering::isCFIntrinsic(const SDNode *Intr) const {
8097 if (Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
8098 switch (Intr->getConstantOperandVal(Num: 1)) {
8099 case Intrinsic::amdgcn_if:
8100 return AMDGPUISD::IF;
8101 case Intrinsic::amdgcn_else:
8102 return AMDGPUISD::ELSE;
8103 case Intrinsic::amdgcn_loop:
8104 return AMDGPUISD::LOOP;
8105 case Intrinsic::amdgcn_end_cf:
8106 llvm_unreachable("should not occur");
8107 default:
8108 return 0;
8109 }
8110 }
8111
8112 // break, if_break, else_break are all only used as inputs to loop, not
8113 // directly as branch conditions.
8114 return 0;
8115}
8116
8117bool SITargetLowering::shouldEmitFixup(const GlobalValue *GV) const {
8118 const Triple &TT = getTargetMachine().getTargetTriple();
8119 return (GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
8120 GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
8121 AMDGPU::shouldEmitConstantsToTextSection(TT);
8122}
8123
8124bool SITargetLowering::shouldEmitGOTReloc(const GlobalValue *GV) const {
8125 if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS())
8126 return false;
8127
8128 // FIXME: Either avoid relying on address space here or change the default
8129 // address space for functions to avoid the explicit check.
8130 return (GV->getValueType()->isFunctionTy() ||
8131 !isNonGlobalAddrSpace(AS: GV->getAddressSpace())) &&
8132 !shouldEmitFixup(GV) && !getTargetMachine().shouldAssumeDSOLocal(GV);
8133}
8134
8135bool SITargetLowering::shouldEmitPCReloc(const GlobalValue *GV) const {
8136 return !shouldEmitFixup(GV) && !shouldEmitGOTReloc(GV);
8137}
8138
8139bool SITargetLowering::shouldUseLDSConstAddress(const GlobalValue *GV) const {
8140 if (!GV->hasExternalLinkage())
8141 return true;
8142
8143 const auto OS = getTargetMachine().getTargetTriple().getOS();
8144 return OS == Triple::AMDHSA || OS == Triple::AMDPAL;
8145}
8146
8147/// This transforms the control flow intrinsics to get the branch destination as
8148/// last parameter, also switches branch target with BR if the need arise
8149SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND, SelectionDAG &DAG) const {
8150 SDLoc DL(BRCOND);
8151
8152 SDNode *Intr = BRCOND.getOperand(i: 1).getNode();
8153 SDValue Target = BRCOND.getOperand(i: 2);
8154 SDNode *BR = nullptr;
8155 SDNode *SetCC = nullptr;
8156
8157 switch (Intr->getOpcode()) {
8158 case ISD::SETCC: {
8159 // As long as we negate the condition everything is fine
8160 SetCC = Intr;
8161 Intr = SetCC->getOperand(Num: 0).getNode();
8162 break;
8163 }
8164 case ISD::XOR: {
8165 // Similar to SETCC, if we have (xor c, -1), we will be fine.
8166 SDValue LHS = Intr->getOperand(Num: 0);
8167 SDValue RHS = Intr->getOperand(Num: 1);
8168 if (auto *C = dyn_cast<ConstantSDNode>(Val&: RHS); C && C->getZExtValue()) {
8169 Intr = LHS.getNode();
8170 break;
8171 }
8172 [[fallthrough]];
8173 }
8174 default: {
8175 // Get the target from BR if we don't negate the condition
8176 BR = findUser(Value: BRCOND, Opcode: ISD::BR);
8177 assert(BR && "brcond missing unconditional branch user");
8178 Target = BR->getOperand(Num: 1);
8179 }
8180 }
8181
8182 unsigned CFNode = isCFIntrinsic(Intr);
8183 if (CFNode == 0) {
8184 // This is a uniform branch so we don't need to legalize.
8185 return BRCOND;
8186 }
8187
8188 bool HaveChain = Intr->getOpcode() == ISD::INTRINSIC_VOID ||
8189 Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN;
8190
8191 assert(!SetCC ||
8192 (SetCC->getConstantOperandVal(1) == 1 &&
8193 cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() ==
8194 ISD::SETNE));
8195
8196 // operands of the new intrinsic call
8197 SmallVector<SDValue, 4> Ops;
8198 if (HaveChain)
8199 Ops.push_back(Elt: BRCOND.getOperand(i: 0));
8200
8201 Ops.append(in_start: Intr->op_begin() + (HaveChain ? 2 : 1), in_end: Intr->op_end());
8202 Ops.push_back(Elt: Target);
8203
8204 ArrayRef<EVT> Res(Intr->value_begin() + 1, Intr->value_end());
8205
8206 // build the new intrinsic call
8207 SDNode *Result = DAG.getNode(Opcode: CFNode, DL, VTList: DAG.getVTList(VTs: Res), Ops).getNode();
8208
8209 if (!HaveChain) {
8210 SDValue Ops[] = {SDValue(Result, 0), BRCOND.getOperand(i: 0)};
8211
8212 Result = DAG.getMergeValues(Ops, dl: DL).getNode();
8213 }
8214
8215 if (BR) {
8216 // Give the branch instruction our target
8217 SDValue Ops[] = {BR->getOperand(Num: 0), BRCOND.getOperand(i: 2)};
8218 SDValue NewBR = DAG.getNode(Opcode: ISD::BR, DL, VTList: BR->getVTList(), Ops);
8219 DAG.ReplaceAllUsesWith(From: BR, To: NewBR.getNode());
8220 }
8221
8222 SDValue Chain = SDValue(Result, Result->getNumValues() - 1);
8223
8224 // Copy the intrinsic results to registers
8225 for (unsigned i = 1, e = Intr->getNumValues() - 1; i != e; ++i) {
8226 SDNode *CopyToReg = findUser(Value: SDValue(Intr, i), Opcode: ISD::CopyToReg);
8227 if (!CopyToReg)
8228 continue;
8229
8230 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: CopyToReg->getOperand(Num: 1),
8231 N: SDValue(Result, i - 1), Glue: SDValue());
8232
8233 DAG.ReplaceAllUsesWith(From: SDValue(CopyToReg, 0), To: CopyToReg->getOperand(Num: 0));
8234 }
8235
8236 // Remove the old intrinsic from the chain
8237 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Intr, Intr->getNumValues() - 1),
8238 To: Intr->getOperand(Num: 0));
8239
8240 return Chain;
8241}
8242
8243SDValue SITargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
8244 MVT VT = Op.getSimpleValueType();
8245 SDLoc DL(Op);
8246 // Checking the depth
8247 if (Op.getConstantOperandVal(i: 0) != 0)
8248 return DAG.getConstant(Val: 0, DL, VT);
8249
8250 MachineFunction &MF = DAG.getMachineFunction();
8251 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8252 // Check for kernel and shader functions
8253 if (Info->isEntryFunction())
8254 return DAG.getConstant(Val: 0, DL, VT);
8255
8256 MachineFrameInfo &MFI = MF.getFrameInfo();
8257 // There is a call to @llvm.returnaddress in this function
8258 MFI.setReturnAddressIsTaken(true);
8259
8260 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
8261 // Get the return address reg and mark it as an implicit live-in
8262 Register Reg = MF.addLiveIn(PReg: TRI->getReturnAddressReg(MF),
8263 RC: getRegClassFor(VT, isDivergent: Op.getNode()->isDivergent()));
8264
8265 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg, VT);
8266}
8267
8268SDValue SITargetLowering::LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const {
8269 MachineFunction &MF = DAG.getMachineFunction();
8270 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
8271
8272 // For functions that set up their own stack, select the GET_STACK_BASE
8273 // pseudo.
8274 if (MFI->isBottomOfStack())
8275 return Op;
8276
8277 // For everything else, create a dummy stack object.
8278 int FI = MF.getFrameInfo().CreateFixedObject(Size: 1, SPOffset: 0, /*IsImmutable=*/false);
8279 return DAG.getFrameIndex(FI, VT: Op.getValueType());
8280}
8281
8282SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG, SDValue Op,
8283 const SDLoc &DL, EVT VT) const {
8284 return Op.getValueType().bitsLE(VT)
8285 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
8286 : DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
8287 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
8288}
8289
8290SDValue SITargetLowering::splitFP_ROUNDVectorOp(SDValue Op,
8291 SelectionDAG &DAG) const {
8292 EVT DstVT = Op.getValueType();
8293 unsigned NumElts = DstVT.getVectorNumElements();
8294 assert(NumElts > 2 && isPowerOf2_32(NumElts));
8295
8296 auto [Lo, Hi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
8297
8298 SDLoc DL(Op);
8299 unsigned Opc = Op.getOpcode();
8300 SDValue Flags = Op.getOperand(i: 1);
8301 EVT HalfDstVT =
8302 EVT::getVectorVT(Context&: *DAG.getContext(), VT: DstVT.getScalarType(), NumElements: NumElts / 2);
8303 SDValue OpLo = DAG.getNode(Opcode: Opc, DL, VT: HalfDstVT, N1: Lo, N2: Flags);
8304 SDValue OpHi = DAG.getNode(Opcode: Opc, DL, VT: HalfDstVT, N1: Hi, N2: Flags);
8305
8306 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: DstVT, N1: OpLo, N2: OpHi);
8307}
8308
8309SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
8310 SDValue Src = Op.getOperand(i: 0);
8311 EVT SrcVT = Src.getValueType();
8312 EVT DstVT = Op.getValueType();
8313
8314 if (DstVT.isVector() && DstVT.getScalarType() == MVT::f16) {
8315 assert(Subtarget->hasCvtPkF16F32Inst() && "support v_cvt_pk_f16_f32");
8316 if (SrcVT.getScalarType() != MVT::f32)
8317 return SDValue();
8318 return SrcVT == MVT::v2f32 ? Op : splitFP_ROUNDVectorOp(Op, DAG);
8319 }
8320
8321 if (SrcVT.getScalarType() != MVT::f64)
8322 return Op;
8323
8324 SDLoc DL(Op);
8325 if (DstVT == MVT::f16) {
8326 // TODO: Handle strictfp
8327 if (Op.getOpcode() != ISD::FP_ROUND)
8328 return Op;
8329
8330 if (!Subtarget->has16BitInsts()) {
8331 SDValue FpToFp16 = DAG.getNode(Opcode: ISD::FP_TO_FP16, DL, VT: MVT::i32, Operand: Src);
8332 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToFp16);
8333 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f16, Operand: Trunc);
8334 }
8335 if (Op->getFlags().hasApproximateFuncs()) {
8336 SDValue Flags = Op.getOperand(i: 1);
8337 SDValue Src32 = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: MVT::f32, N1: Src, N2: Flags);
8338 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: MVT::f16, N1: Src32, N2: Flags);
8339 }
8340 SDValue FpToFp16 = LowerF64ToF16Safe(Src, DL, DAG);
8341 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToFp16);
8342 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f16, Operand: Trunc);
8343 }
8344
8345 assert(DstVT.getScalarType() == MVT::bf16 &&
8346 "custom lower FP_ROUND for f16 or bf16");
8347 assert(Subtarget->hasBF16ConversionInsts() && "f32 -> bf16 is legal");
8348
8349 // Round-inexact-to-odd f64 to f32, then do the final rounding using the
8350 // hardware f32 -> bf16 instruction.
8351 EVT F32VT = SrcVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
8352 SDValue Rod = expandRoundInexactToOdd(ResultVT: F32VT, Op: Src, DL, DAG);
8353 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: DstVT, N1: Rod,
8354 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
8355}
8356
8357SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
8358 SelectionDAG &DAG) const {
8359 EVT VT = Op.getValueType();
8360 const MachineFunction &MF = DAG.getMachineFunction();
8361 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8362 bool IsIEEEMode = Info->getMode().IEEE;
8363
8364 // FIXME: Assert during selection that this is only selected for
8365 // ieee_mode. Currently a combine can produce the ieee version for non-ieee
8366 // mode functions, but this happens to be OK since it's only done in cases
8367 // where there is known no sNaN.
8368 if (IsIEEEMode)
8369 return expandFMINNUM_FMAXNUM(N: Op.getNode(), DAG);
8370
8371 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
8372 VT == MVT::v16bf16)
8373 return splitBinaryVectorOp(Op, DAG);
8374 return Op;
8375}
8376
8377SDValue
8378SITargetLowering::lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op,
8379 SelectionDAG &DAG) const {
8380 EVT VT = Op.getValueType();
8381 const MachineFunction &MF = DAG.getMachineFunction();
8382 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8383 bool IsIEEEMode = Info->getMode().IEEE;
8384
8385 if (IsIEEEMode)
8386 return expandFMINIMUMNUM_FMAXIMUMNUM(N: Op.getNode(), DAG);
8387
8388 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
8389 VT == MVT::v16bf16)
8390 return splitBinaryVectorOp(Op, DAG);
8391 return Op;
8392}
8393
8394SDValue SITargetLowering::lowerFMINIMUM_FMAXIMUM(SDValue Op,
8395 SelectionDAG &DAG) const {
8396 EVT VT = Op.getValueType();
8397 if (VT.isVector())
8398 return splitBinaryVectorOp(Op, DAG);
8399
8400 assert(!Subtarget->hasIEEEMinimumMaximumInsts() &&
8401 !Subtarget->hasMinimum3Maximum3F16() &&
8402 Subtarget->hasMinimum3Maximum3PKF16() && VT == MVT::f16 &&
8403 "should not need to widen f16 minimum/maximum to v2f16");
8404
8405 // Widen f16 operation to v2f16
8406
8407 // fminimum f16:x, f16:y ->
8408 // extract_vector_elt (fminimum (v2f16 (scalar_to_vector x))
8409 // (v2f16 (scalar_to_vector y))), 0
8410 SDLoc SL(Op);
8411 SDValue WideSrc0 =
8412 DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SL, VT: MVT::v2f16, Operand: Op.getOperand(i: 0));
8413 SDValue WideSrc1 =
8414 DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SL, VT: MVT::v2f16, Operand: Op.getOperand(i: 1));
8415
8416 SDValue Widened =
8417 DAG.getNode(Opcode: Op.getOpcode(), DL: SL, VT: MVT::v2f16, N1: WideSrc0, N2: WideSrc1);
8418
8419 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::f16, N1: Widened,
8420 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
8421}
8422
8423SDValue SITargetLowering::lowerFLDEXP(SDValue Op, SelectionDAG &DAG) const {
8424 bool IsStrict = Op.getOpcode() == ISD::STRICT_FLDEXP;
8425 EVT VT = Op.getValueType();
8426 assert(VT == MVT::f16);
8427
8428 SDValue Exp = Op.getOperand(i: IsStrict ? 2 : 1);
8429 EVT ExpVT = Exp.getValueType();
8430 if (ExpVT == MVT::i16)
8431 return Op;
8432
8433 SDLoc DL(Op);
8434
8435 // Correct the exponent type for f16 to i16.
8436 // Clamp the range of the exponent to the instruction's range.
8437
8438 // TODO: This should be a generic narrowing legalization, and can easily be
8439 // for GlobalISel.
8440
8441 SDValue MinExp = DAG.getSignedConstant(Val: minIntN(N: 16), DL, VT: ExpVT);
8442 SDValue ClampMin = DAG.getNode(Opcode: ISD::SMAX, DL, VT: ExpVT, N1: Exp, N2: MinExp);
8443
8444 SDValue MaxExp = DAG.getSignedConstant(Val: maxIntN(N: 16), DL, VT: ExpVT);
8445 SDValue Clamp = DAG.getNode(Opcode: ISD::SMIN, DL, VT: ExpVT, N1: ClampMin, N2: MaxExp);
8446
8447 SDValue TruncExp = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Clamp);
8448
8449 if (IsStrict) {
8450 return DAG.getNode(Opcode: ISD::STRICT_FLDEXP, DL, ResultTys: {VT, MVT::Other},
8451 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), TruncExp});
8452 }
8453
8454 return DAG.getNode(Opcode: ISD::FLDEXP, DL, VT, N1: Op.getOperand(i: 0), N2: TruncExp);
8455}
8456
8457static unsigned getExtOpcodeForPromotedOp(SDValue Op) {
8458 switch (Op->getOpcode()) {
8459 case ISD::SRA:
8460 case ISD::SMIN:
8461 case ISD::SMAX:
8462 return ISD::SIGN_EXTEND;
8463 case ISD::SRL:
8464 case ISD::UMIN:
8465 case ISD::UMAX:
8466 return ISD::ZERO_EXTEND;
8467 case ISD::ADD:
8468 case ISD::SUB:
8469 case ISD::AND:
8470 case ISD::OR:
8471 case ISD::XOR:
8472 case ISD::SHL:
8473 case ISD::SELECT:
8474 case ISD::MUL:
8475 // operation result won't be influenced by garbage high bits.
8476 // TODO: are all of those cases correct, and are there more?
8477 return ISD::ANY_EXTEND;
8478 case ISD::SETCC: {
8479 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
8480 return ISD::isSignedIntSetCC(Code: CC) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
8481 }
8482 default:
8483 llvm_unreachable("unexpected opcode!");
8484 }
8485}
8486
8487SDValue SITargetLowering::promoteUniformOpToI32(SDValue Op,
8488 DAGCombinerInfo &DCI) const {
8489 const unsigned Opc = Op.getOpcode();
8490 assert(Opc == ISD::ADD || Opc == ISD::SUB || Opc == ISD::SHL ||
8491 Opc == ISD::SRL || Opc == ISD::SRA || Opc == ISD::AND ||
8492 Opc == ISD::OR || Opc == ISD::XOR || Opc == ISD::MUL ||
8493 Opc == ISD::SETCC || Opc == ISD::SELECT || Opc == ISD::SMIN ||
8494 Opc == ISD::SMAX || Opc == ISD::UMIN || Opc == ISD::UMAX);
8495
8496 EVT OpTy = (Opc != ISD::SETCC) ? Op.getValueType()
8497 : Op->getOperand(Num: 0).getValueType();
8498 auto &DAG = DCI.DAG;
8499 auto ExtTy = OpTy.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
8500
8501 if (DCI.isBeforeLegalizeOps() ||
8502 isNarrowingProfitable(N: Op.getNode(), SrcVT: ExtTy, DestVT: OpTy))
8503 return SDValue();
8504
8505 SDLoc DL(Op);
8506 SDValue LHS;
8507 SDValue RHS;
8508 if (Opc == ISD::SELECT) {
8509 LHS = Op->getOperand(Num: 1);
8510 RHS = Op->getOperand(Num: 2);
8511 } else {
8512 LHS = Op->getOperand(Num: 0);
8513 RHS = Op->getOperand(Num: 1);
8514 }
8515
8516 const unsigned ExtOp = getExtOpcodeForPromotedOp(Op);
8517 LHS = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: {LHS});
8518
8519 // Special case: for shifts, the RHS always needs a zext.
8520 if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA)
8521 RHS = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtTy, Operand: {RHS});
8522 else
8523 RHS = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: {RHS});
8524
8525 // setcc always return i1/i1 vec so no need to truncate after.
8526 if (Opc == ISD::SETCC) {
8527 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
8528 return DAG.getSetCC(DL, VT: Op.getValueType(), LHS, RHS, Cond: CC);
8529 }
8530
8531 // For other ops, we extend the operation's return type as well so we need to
8532 // truncate back to the original type.
8533 SDValue NewVal;
8534 if (Opc == ISD::SELECT)
8535 NewVal = DAG.getNode(Opcode: ISD::SELECT, DL, VT: ExtTy, Ops: {Op->getOperand(Num: 0), LHS, RHS});
8536 else
8537 NewVal = DAG.getNode(Opcode: Opc, DL, VT: ExtTy, Ops: {LHS, RHS});
8538
8539 return DAG.getZExtOrTrunc(Op: NewVal, DL, VT: OpTy);
8540}
8541
8542SDValue SITargetLowering::lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
8543 SDValue Mag = Op.getOperand(i: 0);
8544 EVT MagVT = Mag.getValueType();
8545
8546 if (MagVT.getVectorNumElements() > 2)
8547 return splitBinaryVectorOp(Op, DAG);
8548
8549 SDValue Sign = Op.getOperand(i: 1);
8550 EVT SignVT = Sign.getValueType();
8551
8552 if (MagVT == SignVT)
8553 return Op;
8554
8555 // fcopysign v2f16:mag, v2f32:sign ->
8556 // fcopysign v2f16:mag, bitcast (trunc (bitcast sign to v2i32) to v2i16)
8557
8558 SDLoc SL(Op);
8559 SDValue SignAsInt32 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Sign);
8560 SDValue SignAsInt16 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::v2i16, Operand: SignAsInt32);
8561
8562 SDValue SignAsHalf16 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MagVT, Operand: SignAsInt16);
8563
8564 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT: MagVT, N1: Mag, N2: SignAsHalf16);
8565}
8566
8567// Custom lowering for vector multiplications and s_mul_u64.
8568SDValue SITargetLowering::lowerMUL(SDValue Op, SelectionDAG &DAG) const {
8569 EVT VT = Op.getValueType();
8570
8571 // Split vector operands.
8572 if (VT.isVector())
8573 return splitBinaryVectorOp(Op, DAG);
8574
8575 assert(VT == MVT::i64 && "The following code is a special for s_mul_u64");
8576
8577 // There are four ways to lower s_mul_u64:
8578 //
8579 // 1. If all the operands are uniform, then we lower it as it is.
8580 //
8581 // 2. If the operands are divergent, then we have to split s_mul_u64 in 32-bit
8582 // multiplications because there is not a vector equivalent of s_mul_u64.
8583 //
8584 // 3. If the cost model decides that it is more efficient to use vector
8585 // registers, then we have to split s_mul_u64 in 32-bit multiplications.
8586 // This happens in splitScalarSMULU64() in SIInstrInfo.cpp .
8587 //
8588 // 4. If the cost model decides to use vector registers and both of the
8589 // operands are zero-extended/sign-extended from 32-bits, then we split the
8590 // s_mul_u64 in two 32-bit multiplications. The problem is that it is not
8591 // possible to check if the operands are zero-extended or sign-extended in
8592 // SIInstrInfo.cpp. For this reason, here, we replace s_mul_u64 with
8593 // s_mul_u64_u32_pseudo if both operands are zero-extended and we replace
8594 // s_mul_u64 with s_mul_i64_i32_pseudo if both operands are sign-extended.
8595 // If the cost model decides that we have to use vector registers, then
8596 // splitScalarSMulPseudo() (in SIInstrInfo.cpp) split s_mul_u64_u32/
8597 // s_mul_i64_i32_pseudo in two vector multiplications. If the cost model
8598 // decides that we should use scalar registers, then s_mul_u64_u32_pseudo/
8599 // s_mul_i64_i32_pseudo is lowered as s_mul_u64 in expandPostRAPseudo() in
8600 // SIInstrInfo.cpp .
8601
8602 if (Op->isDivergent())
8603 return SDValue();
8604
8605 SDValue Op0 = Op.getOperand(i: 0);
8606 SDValue Op1 = Op.getOperand(i: 1);
8607 // If all the operands are zero-enteted to 32-bits, then we replace s_mul_u64
8608 // with s_mul_u64_u32_pseudo. If all the operands are sign-extended to
8609 // 32-bits, then we replace s_mul_u64 with s_mul_i64_i32_pseudo.
8610 KnownBits Op0KnownBits = DAG.computeKnownBits(Op: Op0);
8611 unsigned Op0LeadingZeros = Op0KnownBits.countMinLeadingZeros();
8612 KnownBits Op1KnownBits = DAG.computeKnownBits(Op: Op1);
8613 unsigned Op1LeadingZeros = Op1KnownBits.countMinLeadingZeros();
8614 SDLoc SL(Op);
8615 if (Op0LeadingZeros >= 32 && Op1LeadingZeros >= 32)
8616 return SDValue(
8617 DAG.getMachineNode(Opcode: AMDGPU::S_MUL_U64_U32_PSEUDO, dl: SL, VT, Op1: Op0, Op2: Op1), 0);
8618 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op: Op0);
8619 unsigned Op1SignBits = DAG.ComputeNumSignBits(Op: Op1);
8620 if (Op0SignBits >= 33 && Op1SignBits >= 33)
8621 return SDValue(
8622 DAG.getMachineNode(Opcode: AMDGPU::S_MUL_I64_I32_PSEUDO, dl: SL, VT, Op1: Op0, Op2: Op1), 0);
8623 // If all the operands are uniform, then we lower s_mul_u64 as it is.
8624 return Op;
8625}
8626
8627SDValue SITargetLowering::lowerXMULO(SDValue Op, SelectionDAG &DAG) const {
8628 EVT VT = Op.getValueType();
8629 SDLoc SL(Op);
8630 SDValue LHS = Op.getOperand(i: 0);
8631 SDValue RHS = Op.getOperand(i: 1);
8632 bool isSigned = Op.getOpcode() == ISD::SMULO;
8633
8634 if (ConstantSDNode *RHSC = isConstOrConstSplat(N: RHS)) {
8635 const APInt &C = RHSC->getAPIntValue();
8636 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
8637 if (C.isPowerOf2()) {
8638 // smulo(x, signed_min) is same as umulo(x, signed_min).
8639 bool UseArithShift = isSigned && !C.isMinSignedValue();
8640 SDValue ShiftAmt = DAG.getConstant(Val: C.logBase2(), DL: SL, VT: MVT::i32);
8641 SDValue Result = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT, N1: LHS, N2: ShiftAmt);
8642 SDValue Overflow =
8643 DAG.getSetCC(DL: SL, VT: MVT::i1,
8644 LHS: DAG.getNode(Opcode: UseArithShift ? ISD::SRA : ISD::SRL, DL: SL, VT,
8645 N1: Result, N2: ShiftAmt),
8646 RHS: LHS, Cond: ISD::SETNE);
8647 return DAG.getMergeValues(Ops: {Result, Overflow}, dl: SL);
8648 }
8649 }
8650
8651 SDValue Result = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT, N1: LHS, N2: RHS);
8652 SDValue Top =
8653 DAG.getNode(Opcode: isSigned ? ISD::MULHS : ISD::MULHU, DL: SL, VT, N1: LHS, N2: RHS);
8654
8655 SDValue Sign = isSigned
8656 ? DAG.getNode(Opcode: ISD::SRA, DL: SL, VT, N1: Result,
8657 N2: DAG.getConstant(Val: VT.getScalarSizeInBits() - 1,
8658 DL: SL, VT: MVT::i32))
8659 : DAG.getConstant(Val: 0, DL: SL, VT);
8660 SDValue Overflow = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Top, RHS: Sign, Cond: ISD::SETNE);
8661
8662 return DAG.getMergeValues(Ops: {Result, Overflow}, dl: SL);
8663}
8664
8665SDValue SITargetLowering::lowerXMUL_LOHI(SDValue Op, SelectionDAG &DAG) const {
8666 if (Op->isDivergent()) {
8667 // Select to V_MAD_[IU]64_[IU]32.
8668 return Op;
8669 }
8670 if (Subtarget->hasSMulHi()) {
8671 // Expand to S_MUL_I32 + S_MUL_HI_[IU]32.
8672 return SDValue();
8673 }
8674 // The multiply is uniform but we would have to use V_MUL_HI_[IU]32 to
8675 // calculate the high part, so we might as well do the whole thing with
8676 // V_MAD_[IU]64_[IU]32.
8677 return Op;
8678}
8679
8680SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const {
8681 if (!Subtarget->hasTrapHandler() ||
8682 Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
8683 return lowerTrapEndpgm(Op, DAG);
8684
8685 return Subtarget->supportsGetDoorbellID() ? lowerTrapHsa(Op, DAG)
8686 : lowerTrapHsaQueuePtr(Op, DAG);
8687}
8688
8689SDValue SITargetLowering::lowerTrapEndpgm(SDValue Op, SelectionDAG &DAG) const {
8690 SDLoc SL(Op);
8691 SDValue Chain = Op.getOperand(i: 0);
8692 return DAG.getNode(Opcode: AMDGPUISD::ENDPGM_TRAP, DL: SL, VT: MVT::Other, Operand: Chain);
8693}
8694
8695SDValue
8696SITargetLowering::loadImplicitKernelArgument(SelectionDAG &DAG, MVT VT,
8697 const SDLoc &DL, Align Alignment,
8698 ImplicitParameter Param) const {
8699 MachineFunction &MF = DAG.getMachineFunction();
8700 uint64_t Offset = getImplicitParameterOffset(MF, Param);
8701 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL: DL, Chain: DAG.getEntryNode(), Offset);
8702 MachinePointerInfo PtrInfo =
8703 getKernargSegmentPtrInfo(MF&: DAG.getMachineFunction());
8704 return DAG.getLoad(
8705 VT, dl: DL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: PtrInfo.getWithOffset(O: Offset), Alignment,
8706 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
8707}
8708
8709SDValue SITargetLowering::lowerTrapHsaQueuePtr(SDValue Op,
8710 SelectionDAG &DAG) const {
8711 SDLoc SL(Op);
8712 SDValue Chain = Op.getOperand(i: 0);
8713
8714 SDValue QueuePtr;
8715 // For code object version 5, QueuePtr is passed through implicit kernarg.
8716 const Module *M = DAG.getMachineFunction().getFunction().getParent();
8717 if (AMDGPU::getAMDHSACodeObjectVersion(M: *M) >= AMDGPU::AMDHSA_COV5) {
8718 QueuePtr =
8719 loadImplicitKernelArgument(DAG, VT: MVT::i64, DL: SL, Alignment: Align(8), Param: QUEUE_PTR);
8720 } else {
8721 MachineFunction &MF = DAG.getMachineFunction();
8722 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8723 Register UserSGPR = Info->getQueuePtrUserSGPR();
8724
8725 if (UserSGPR == AMDGPU::NoRegister) {
8726 // We probably are in a function incorrectly marked with
8727 // amdgpu-no-queue-ptr. This is undefined. We don't want to delete the
8728 // trap, so just use a null pointer.
8729 QueuePtr = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i64);
8730 } else {
8731 QueuePtr = CreateLiveInRegister(DAG, RC: &AMDGPU::SReg_64RegClass, Reg: UserSGPR,
8732 VT: MVT::i64);
8733 }
8734 }
8735
8736 SDValue SGPR01 = DAG.getRegister(Reg: AMDGPU::SGPR0_SGPR1, VT: MVT::i64);
8737 SDValue ToReg = DAG.getCopyToReg(Chain, dl: SL, Reg: SGPR01, N: QueuePtr, Glue: SDValue());
8738
8739 uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap);
8740 SDValue Ops[] = {ToReg, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16), SGPR01,
8741 ToReg.getValue(R: 1)};
8742 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
8743}
8744
8745SDValue SITargetLowering::lowerTrapHsa(SDValue Op, SelectionDAG &DAG) const {
8746 SDLoc SL(Op);
8747 SDValue Chain = Op.getOperand(i: 0);
8748
8749 // We need to simulate the 's_trap 2' instruction on targets that run in
8750 // PRIV=1 (where it is treated as a nop).
8751 if (Subtarget->hasPrivEnabledTrap2NopBug())
8752 return DAG.getNode(Opcode: AMDGPUISD::SIMULATED_TRAP, DL: SL, VT: MVT::Other, Operand: Chain);
8753
8754 uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap);
8755 SDValue Ops[] = {Chain, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16)};
8756 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
8757}
8758
8759SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const {
8760 SDLoc SL(Op);
8761 SDValue Chain = Op.getOperand(i: 0);
8762 MachineFunction &MF = DAG.getMachineFunction();
8763
8764 if (!Subtarget->hasTrapHandler() ||
8765 Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) {
8766 LLVMContext &Ctx = MF.getFunction().getContext();
8767 Ctx.diagnose(DI: DiagnosticInfoUnsupported(MF.getFunction(),
8768 "debugtrap handler not supported",
8769 Op.getDebugLoc(), DS_Warning));
8770 return Chain;
8771 }
8772
8773 uint64_t TrapID =
8774 static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap);
8775 SDValue Ops[] = {Chain, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16)};
8776 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
8777}
8778
8779SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL,
8780 SelectionDAG &DAG) const {
8781 if (Subtarget->hasApertureRegs()) {
8782 const unsigned ApertureRegNo = (AS == AMDGPUAS::LOCAL_ADDRESS)
8783 ? AMDGPU::SRC_SHARED_BASE
8784 : AMDGPU::SRC_PRIVATE_BASE;
8785 assert((ApertureRegNo != AMDGPU::SRC_PRIVATE_BASE ||
8786 !Subtarget->hasGloballyAddressableScratch()) &&
8787 "Cannot use src_private_base with globally addressable scratch!");
8788 // Note: this feature (register) is broken. When used as a 32-bit operand,
8789 // it returns a wrong value (all zeroes?). The real value is in the upper 32
8790 // bits.
8791 //
8792 // To work around the issue, emit a 64 bit copy from this register
8793 // then extract the high bits. Note that this shouldn't even result in a
8794 // shift being emitted and simply become a pair of registers (e.g.):
8795 // s_mov_b64 s[6:7], src_shared_base
8796 // v_mov_b32_e32 v1, s7
8797 SDValue Copy =
8798 DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: ApertureRegNo, VT: MVT::v2i32);
8799 return DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: Copy, Idx: 1);
8800 }
8801
8802 // For code object version 5, private_base and shared_base are passed through
8803 // implicit kernargs.
8804 const Module *M = DAG.getMachineFunction().getFunction().getParent();
8805 if (AMDGPU::getAMDHSACodeObjectVersion(M: *M) >= AMDGPU::AMDHSA_COV5) {
8806 ImplicitParameter Param =
8807 (AS == AMDGPUAS::LOCAL_ADDRESS) ? SHARED_BASE : PRIVATE_BASE;
8808 return loadImplicitKernelArgument(DAG, VT: MVT::i32, DL, Alignment: Align(4), Param);
8809 }
8810
8811 MachineFunction &MF = DAG.getMachineFunction();
8812 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8813 Register UserSGPR = Info->getQueuePtrUserSGPR();
8814 if (UserSGPR == AMDGPU::NoRegister) {
8815 // We probably are in a function incorrectly marked with
8816 // amdgpu-no-queue-ptr. This is undefined.
8817 return DAG.getPOISON(VT: MVT::i32);
8818 }
8819
8820 SDValue QueuePtr =
8821 CreateLiveInRegister(DAG, RC: &AMDGPU::SReg_64RegClass, Reg: UserSGPR, VT: MVT::i64);
8822
8823 // Offset into amd_queue_t for group_segment_aperture_base_hi /
8824 // private_segment_aperture_base_hi.
8825 uint32_t StructOffset = (AS == AMDGPUAS::LOCAL_ADDRESS) ? 0x40 : 0x44;
8826
8827 SDValue Ptr =
8828 DAG.getObjectPtrOffset(SL: DL, Ptr: QueuePtr, Offset: TypeSize::getFixed(ExactSize: StructOffset));
8829
8830 // TODO: Use custom target PseudoSourceValue.
8831 // TODO: We should use the value from the IR intrinsic call, but it might not
8832 // be available and how do we get it?
8833 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
8834 return DAG.getLoad(VT: MVT::i32, dl: DL, Chain: QueuePtr.getValue(R: 1), Ptr, PtrInfo,
8835 Alignment: commonAlignment(A: Align(64), Offset: StructOffset),
8836 MMOFlags: MachineMemOperand::MODereferenceable |
8837 MachineMemOperand::MOInvariant);
8838}
8839
8840/// Return true if the value is a known valid address, such that a null check is
8841/// not necessary.
8842static bool isKnownNonNull(SDValue Val, SelectionDAG &DAG,
8843 const AMDGPUTargetMachine &TM, unsigned AddrSpace) {
8844 if (isa<FrameIndexSDNode, GlobalAddressSDNode, BasicBlockSDNode>(Val))
8845 return true;
8846
8847 if (auto *ConstVal = dyn_cast<ConstantSDNode>(Val))
8848 return ConstVal->getSExtValue() != AMDGPU::getNullPointerValue(AS: AddrSpace);
8849
8850 // TODO: Search through arithmetic, handle arguments and loads
8851 // marked nonnull.
8852 return false;
8853}
8854
8855SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op,
8856 SelectionDAG &DAG) const {
8857 SDLoc SL(Op);
8858
8859 const AMDGPUTargetMachine &TM =
8860 static_cast<const AMDGPUTargetMachine &>(getTargetMachine());
8861
8862 unsigned DestAS, SrcAS;
8863 SDValue Src;
8864 bool IsNonNull = false;
8865 if (const auto *ASC = dyn_cast<AddrSpaceCastSDNode>(Val&: Op)) {
8866 SrcAS = ASC->getSrcAddressSpace();
8867 Src = ASC->getOperand(Num: 0);
8868 DestAS = ASC->getDestAddressSpace();
8869 } else {
8870 assert(Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
8871 Op.getConstantOperandVal(0) ==
8872 Intrinsic::amdgcn_addrspacecast_nonnull);
8873 Src = Op->getOperand(Num: 1);
8874 SrcAS = Op->getConstantOperandVal(Num: 2);
8875 DestAS = Op->getConstantOperandVal(Num: 3);
8876 IsNonNull = true;
8877 }
8878
8879 SDValue FlatNullPtr = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i64);
8880
8881 // flat -> local/private
8882 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
8883 if (DestAS == AMDGPUAS::LOCAL_ADDRESS ||
8884 DestAS == AMDGPUAS::PRIVATE_ADDRESS) {
8885 SDValue Ptr = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Src);
8886
8887 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
8888 Subtarget->hasGloballyAddressableScratch()) {
8889 // flat -> private with globally addressable scratch: subtract
8890 // src_flat_scratch_base_lo.
8891 SDValue FlatScratchBaseLo(
8892 DAG.getMachineNode(
8893 Opcode: AMDGPU::S_MOV_B32, dl: SL, VT: MVT::i32,
8894 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO, VT: MVT::i32)),
8895 0);
8896 Ptr = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: Ptr, N2: FlatScratchBaseLo);
8897 }
8898
8899 if (IsNonNull || isKnownNonNull(Val: Op, DAG, TM, AddrSpace: SrcAS))
8900 return Ptr;
8901
8902 unsigned NullVal = AMDGPU::getNullPointerValue(AS: DestAS);
8903 SDValue SegmentNullPtr = DAG.getConstant(Val: NullVal, DL: SL, VT: MVT::i32);
8904 SDValue NonNull = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Src, RHS: FlatNullPtr, Cond: ISD::SETNE);
8905
8906 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i32, N1: NonNull, N2: Ptr,
8907 N3: SegmentNullPtr);
8908 }
8909 }
8910
8911 // local/private -> flat
8912 if (DestAS == AMDGPUAS::FLAT_ADDRESS) {
8913 if (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
8914 SrcAS == AMDGPUAS::PRIVATE_ADDRESS) {
8915 SDValue CvtPtr;
8916 if (SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
8917 Subtarget->hasGloballyAddressableScratch()) {
8918 // For wave32: Addr = (TID[4:0] << 52) + FLAT_SCRATCH_BASE + privateAddr
8919 // For wave64: Addr = (TID[5:0] << 51) + FLAT_SCRATCH_BASE + privateAddr
8920 SDValue AllOnes = DAG.getSignedTargetConstant(Val: -1, DL: SL, VT: MVT::i32);
8921 SDValue ThreadID = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
8922 ThreadID = DAG.getNode(
8923 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
8924 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_mbcnt_lo, DL: SL, VT: MVT::i32),
8925 N2: AllOnes, N3: ThreadID);
8926 if (Subtarget->isWave64())
8927 ThreadID = DAG.getNode(
8928 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
8929 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_mbcnt_hi, DL: SL, VT: MVT::i32),
8930 N2: AllOnes, N3: ThreadID);
8931 SDValue ShAmt = DAG.getShiftAmountConstant(
8932 Val: 57 - 32 - Subtarget->getWavefrontSizeLog2(), VT: MVT::i32, DL: SL);
8933 SDValue SrcHi = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: ThreadID, N2: ShAmt);
8934 CvtPtr = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: SrcHi);
8935 CvtPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
8936 // Accessing src_flat_scratch_base_lo as a 64-bit operand gives the full
8937 // 64-bit hi:lo value.
8938 SDValue FlatScratchBase = {
8939 DAG.getMachineNode(
8940 Opcode: AMDGPU::S_MOV_B64, dl: SL, VT: MVT::i64,
8941 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE, VT: MVT::i64)),
8942 0};
8943 CvtPtr = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i64, N1: CvtPtr, N2: FlatScratchBase);
8944 } else {
8945 SDValue Aperture = getSegmentAperture(AS: SrcAS, DL: SL, DAG);
8946 CvtPtr = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: Aperture);
8947 CvtPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
8948 }
8949
8950 if (IsNonNull || isKnownNonNull(Val: Op, DAG, TM, AddrSpace: SrcAS))
8951 return CvtPtr;
8952
8953 unsigned NullVal = AMDGPU::getNullPointerValue(AS: SrcAS);
8954 SDValue SegmentNullPtr = DAG.getConstant(Val: NullVal, DL: SL, VT: MVT::i32);
8955
8956 SDValue NonNull =
8957 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Src, RHS: SegmentNullPtr, Cond: ISD::SETNE);
8958
8959 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: NonNull, N2: CvtPtr,
8960 N3: FlatNullPtr);
8961 }
8962 }
8963
8964 if (SrcAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
8965 Op.getValueType() == MVT::i64) {
8966 const SIMachineFunctionInfo *Info =
8967 DAG.getMachineFunction().getInfo<SIMachineFunctionInfo>();
8968 if (Info->get32BitAddressHighBits() == 0)
8969 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i64, Operand: Src);
8970
8971 SDValue Hi = DAG.getConstant(Val: Info->get32BitAddressHighBits(), DL: SL, VT: MVT::i32);
8972 SDValue Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: Hi);
8973 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
8974 }
8975
8976 if (DestAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
8977 Src.getValueType() == MVT::i64)
8978 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Src);
8979
8980 // global <-> flat are no-ops and never emitted.
8981
8982 // Invalid casts are poison.
8983 return DAG.getPOISON(VT: Op->getValueType(ResNo: 0));
8984}
8985
8986// This lowers an INSERT_SUBVECTOR by extracting the individual elements from
8987// the small vector and inserting them into the big vector. That is better than
8988// the default expansion of doing it via a stack slot. Even though the use of
8989// the stack slot would be optimized away afterwards, the stack slot itself
8990// remains.
8991SDValue SITargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
8992 SelectionDAG &DAG) const {
8993 SDValue Vec = Op.getOperand(i: 0);
8994 SDValue Ins = Op.getOperand(i: 1);
8995 SDValue Idx = Op.getOperand(i: 2);
8996 EVT VecVT = Vec.getValueType();
8997 EVT InsVT = Ins.getValueType();
8998 EVT EltVT = VecVT.getVectorElementType();
8999 unsigned InsNumElts = InsVT.getVectorNumElements();
9000 unsigned IdxVal = Idx->getAsZExtVal();
9001 SDLoc SL(Op);
9002
9003 if (EltVT.getScalarSizeInBits() == 16 && IdxVal % 2 == 0) {
9004 // Insert 32-bit registers at a time.
9005 assert(InsNumElts % 2 == 0 && "expect legal vector types");
9006
9007 unsigned VecNumElts = VecVT.getVectorNumElements();
9008 EVT NewVecVT =
9009 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: VecNumElts / 2);
9010 EVT NewInsVT = InsNumElts == 2 ? MVT::i32
9011 : EVT::getVectorVT(Context&: *DAG.getContext(),
9012 VT: MVT::i32, NumElements: InsNumElts / 2);
9013
9014 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVecVT, Operand: Vec);
9015 Ins = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewInsVT, Operand: Ins);
9016
9017 for (unsigned I = 0; I != InsNumElts / 2; ++I) {
9018 SDValue Elt;
9019 if (InsNumElts == 2) {
9020 Elt = Ins;
9021 } else {
9022 Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Ins,
9023 N2: DAG.getConstant(Val: I, DL: SL, VT: MVT::i32));
9024 }
9025 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NewVecVT, N1: Vec, N2: Elt,
9026 N3: DAG.getConstant(Val: IdxVal / 2 + I, DL: SL, VT: MVT::i32));
9027 }
9028
9029 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: Vec);
9030 }
9031
9032 for (unsigned I = 0; I != InsNumElts; ++I) {
9033 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Ins,
9034 N2: DAG.getConstant(Val: I, DL: SL, VT: MVT::i32));
9035 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: VecVT, N1: Vec, N2: Elt,
9036 N3: DAG.getConstant(Val: IdxVal + I, DL: SL, VT: MVT::i32));
9037 }
9038 return Vec;
9039}
9040
9041SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
9042 SelectionDAG &DAG) const {
9043 SDValue Vec = Op.getOperand(i: 0);
9044 SDValue InsVal = Op.getOperand(i: 1);
9045 SDValue Idx = Op.getOperand(i: 2);
9046 EVT VecVT = Vec.getValueType();
9047 EVT EltVT = VecVT.getVectorElementType();
9048 unsigned VecSize = VecVT.getSizeInBits();
9049 unsigned EltSize = EltVT.getSizeInBits();
9050 SDLoc SL(Op);
9051
9052 // Specially handle the case of v4i16 with static indexing.
9053 unsigned NumElts = VecVT.getVectorNumElements();
9054 auto *KIdx = dyn_cast<ConstantSDNode>(Val&: Idx);
9055 if (NumElts == 4 && EltSize == 16 && KIdx) {
9056 SDValue BCVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Vec);
9057
9058 SDValue LoHalf = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: BCVec,
9059 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
9060 SDValue HiHalf = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: BCVec,
9061 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
9062
9063 SDValue LoVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: LoHalf);
9064 SDValue HiVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: HiHalf);
9065
9066 unsigned Idx = KIdx->getZExtValue();
9067 bool InsertLo = Idx < 2;
9068 SDValue InsHalf = DAG.getNode(
9069 Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: MVT::v2i16, N1: InsertLo ? LoVec : HiVec,
9070 N2: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: InsVal),
9071 N3: DAG.getConstant(Val: InsertLo ? Idx : (Idx - 2), DL: SL, VT: MVT::i32));
9072
9073 InsHalf = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: InsHalf);
9074
9075 SDValue Concat =
9076 InsertLo ? DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {InsHalf, HiHalf})
9077 : DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {LoHalf, InsHalf});
9078
9079 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: Concat);
9080 }
9081
9082 // Static indexing does not lower to stack access, and hence there is no need
9083 // for special custom lowering to avoid stack access.
9084 if (isa<ConstantSDNode>(Val: Idx))
9085 return SDValue();
9086
9087 // Avoid stack access for dynamic indexing by custom lowering to
9088 // v_bfi_b32 (v_bfm_b32 16, (shl idx, 16)), val, vec
9089
9090 assert(VecSize <= 64 && "Expected target vector size to be <= 64 bits");
9091
9092 MVT IntVT = MVT::getIntegerVT(BitWidth: VecSize);
9093
9094 // Convert vector index to bit-index and get the required bit mask.
9095 assert(isPowerOf2_32(EltSize));
9096 const auto EltMask = maskTrailingOnes<uint64_t>(N: EltSize);
9097 SDValue ScaleFactor = DAG.getConstant(Val: Log2_32(Value: EltSize), DL: SL, VT: MVT::i32);
9098 SDValue ScaledIdx = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Idx, N2: ScaleFactor);
9099 SDValue BFM = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: IntVT,
9100 N1: DAG.getConstant(Val: EltMask, DL: SL, VT: IntVT), N2: ScaledIdx);
9101
9102 // 1. Create a congruent vector with the target value in each element.
9103 SDValue ExtVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT,
9104 Operand: DAG.getSplatBuildVector(VT: VecVT, DL: SL, Op: InsVal));
9105
9106 // 2. Mask off all other indices except the required index within (1).
9107 SDValue LHS = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IntVT, N1: BFM, N2: ExtVal);
9108
9109 // 3. Mask off the required index within the target vector.
9110 SDValue BCVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT, Operand: Vec);
9111 SDValue RHS =
9112 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IntVT, N1: DAG.getNOT(DL: SL, Val: BFM, VT: IntVT), N2: BCVec);
9113
9114 // 4. Get (2) and (3) ORed into the target vector.
9115 SDValue BFI =
9116 DAG.getNode(Opcode: ISD::OR, DL: SL, VT: IntVT, N1: LHS, N2: RHS, Flags: SDNodeFlags::Disjoint);
9117
9118 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: BFI);
9119}
9120
9121SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
9122 SelectionDAG &DAG) const {
9123 SDLoc SL(Op);
9124
9125 EVT ResultVT = Op.getValueType();
9126 SDValue Vec = Op.getOperand(i: 0);
9127 SDValue Idx = Op.getOperand(i: 1);
9128 EVT VecVT = Vec.getValueType();
9129 unsigned VecSize = VecVT.getSizeInBits();
9130 EVT EltVT = VecVT.getVectorElementType();
9131
9132 DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr);
9133
9134 // Make sure we do any optimizations that will make it easier to fold
9135 // source modifiers before obscuring it with bit operations.
9136
9137 // XXX - Why doesn't this get called when vector_shuffle is expanded?
9138 if (SDValue Combined = performExtractVectorEltCombine(N: Op.getNode(), DCI))
9139 return Combined;
9140
9141 if (VecSize == 128 || VecSize == 256 || VecSize == 512) {
9142 SDValue Lo, Hi;
9143 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
9144
9145 if (VecSize == 128) {
9146 SDValue V2 = DAG.getBitcast(VT: MVT::v2i64, V: Vec);
9147 Lo = DAG.getBitcast(VT: LoVT,
9148 V: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9149 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32)));
9150 Hi = DAG.getBitcast(VT: HiVT,
9151 V: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9152 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32)));
9153 } else if (VecSize == 256) {
9154 SDValue V2 = DAG.getBitcast(VT: MVT::v4i64, V: Vec);
9155 SDValue Parts[4];
9156 for (unsigned P = 0; P < 4; ++P) {
9157 Parts[P] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9158 N2: DAG.getConstant(Val: P, DL: SL, VT: MVT::i32));
9159 }
9160
9161 Lo = DAG.getBitcast(VT: LoVT, V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i64,
9162 N1: Parts[0], N2: Parts[1]));
9163 Hi = DAG.getBitcast(VT: HiVT, V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i64,
9164 N1: Parts[2], N2: Parts[3]));
9165 } else {
9166 assert(VecSize == 512);
9167
9168 SDValue V2 = DAG.getBitcast(VT: MVT::v8i64, V: Vec);
9169 SDValue Parts[8];
9170 for (unsigned P = 0; P < 8; ++P) {
9171 Parts[P] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9172 N2: DAG.getConstant(Val: P, DL: SL, VT: MVT::i32));
9173 }
9174
9175 Lo = DAG.getBitcast(VT: LoVT,
9176 V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v4i64,
9177 N1: Parts[0], N2: Parts[1], N3: Parts[2], N4: Parts[3]));
9178 Hi = DAG.getBitcast(VT: HiVT,
9179 V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v4i64,
9180 N1: Parts[4], N2: Parts[5], N3: Parts[6], N4: Parts[7]));
9181 }
9182
9183 EVT IdxVT = Idx.getValueType();
9184 unsigned NElem = VecVT.getVectorNumElements();
9185 assert(isPowerOf2_32(NElem));
9186 SDValue IdxMask = DAG.getConstant(Val: NElem / 2 - 1, DL: SL, VT: IdxVT);
9187 SDValue NewIdx = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IdxVT, N1: Idx, N2: IdxMask);
9188 SDValue Half = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IdxMask, True: Hi, False: Lo, Cond: ISD::SETUGT);
9189 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Half, N2: NewIdx);
9190 }
9191
9192 assert(VecSize <= 64);
9193
9194 MVT IntVT = MVT::getIntegerVT(BitWidth: VecSize);
9195
9196 // If Vec is just a SCALAR_TO_VECTOR, then use the scalar integer directly.
9197 SDValue VecBC = peekThroughBitcasts(V: Vec);
9198 if (VecBC.getOpcode() == ISD::SCALAR_TO_VECTOR) {
9199 SDValue Src = VecBC.getOperand(i: 0);
9200 Src = DAG.getBitcast(VT: Src.getValueType().changeTypeToInteger(), V: Src);
9201 Vec = DAG.getAnyExtOrTrunc(Op: Src, DL: SL, VT: IntVT);
9202 }
9203
9204 unsigned EltSize = EltVT.getSizeInBits();
9205 assert(isPowerOf2_32(EltSize));
9206
9207 SDValue ScaleFactor = DAG.getConstant(Val: Log2_32(Value: EltSize), DL: SL, VT: MVT::i32);
9208
9209 // Convert vector index to bit-index (* EltSize)
9210 SDValue ScaledIdx = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Idx, N2: ScaleFactor);
9211
9212 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT, Operand: Vec);
9213 SDValue Elt = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: IntVT, N1: BC, N2: ScaledIdx);
9214
9215 if (ResultVT == MVT::f16 || ResultVT == MVT::bf16) {
9216 SDValue Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i16, Operand: Elt);
9217 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ResultVT, Operand: Result);
9218 }
9219
9220 return DAG.getAnyExtOrTrunc(Op: Elt, DL: SL, VT: ResultVT);
9221}
9222
9223static bool elementPairIsContiguous(ArrayRef<int> Mask, int Elt) {
9224 assert(Elt % 2 == 0);
9225 return Mask[Elt + 1] == Mask[Elt] + 1 && (Mask[Elt] % 2 == 0);
9226}
9227
9228static bool elementPairIsOddToEven(ArrayRef<int> Mask, int Elt) {
9229 assert(Elt % 2 == 0);
9230 return Mask[Elt] >= 0 && Mask[Elt + 1] >= 0 && (Mask[Elt] & 1) &&
9231 !(Mask[Elt + 1] & 1);
9232}
9233
9234SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
9235 SelectionDAG &DAG) const {
9236 SDLoc SL(Op);
9237 EVT ResultVT = Op.getValueType();
9238 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
9239 MVT EltVT = ResultVT.getVectorElementType().getSimpleVT();
9240 const int NewSrcNumElts = 2;
9241 MVT PackVT = MVT::getVectorVT(VT: EltVT, NumElements: NewSrcNumElts);
9242 int SrcNumElts = Op.getOperand(i: 0).getValueType().getVectorNumElements();
9243
9244 // Break up the shuffle into registers sized pieces.
9245 //
9246 // We're trying to form sub-shuffles that the register allocation pipeline
9247 // won't be able to figure out, like how to use v_pk_mov_b32 to do a register
9248 // blend or 16-bit op_sel. It should be able to figure out how to reassemble a
9249 // pair of copies into a consecutive register copy, so use the ordinary
9250 // extract_vector_elt lowering unless we can use the shuffle.
9251 //
9252 // TODO: This is a bit of hack, and we should probably always use
9253 // extract_subvector for the largest possible subvector we can (or at least
9254 // use it for PackVT aligned pieces). However we have worse support for
9255 // combines on them don't directly treat extract_subvector / insert_subvector
9256 // as legal. The DAG scheduler also ends up doing a worse job with the
9257 // extract_subvectors.
9258 const bool ShouldUseConsecutiveExtract = EltVT.getSizeInBits() == 16;
9259
9260 // vector_shuffle <0,1,6,7> lhs, rhs
9261 // -> concat_vectors (extract_subvector lhs, 0), (extract_subvector rhs, 2)
9262 //
9263 // vector_shuffle <6,7,2,3> lhs, rhs
9264 // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 2)
9265 //
9266 // vector_shuffle <6,7,0,1> lhs, rhs
9267 // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 0)
9268
9269 // Avoid scalarizing when both halves are reading from consecutive elements.
9270
9271 // If we're treating 2 element shuffles as legal, also create odd-to-even
9272 // shuffles of neighboring pairs.
9273 //
9274 // vector_shuffle <3,2,7,6> lhs, rhs
9275 // -> concat_vectors vector_shuffle <1, 0> (extract_subvector lhs, 0)
9276 // vector_shuffle <1, 0> (extract_subvector rhs, 2)
9277
9278 SmallVector<SDValue, 16> Pieces;
9279 for (int I = 0, N = ResultVT.getVectorNumElements(); I != N; I += 2) {
9280 if (ShouldUseConsecutiveExtract &&
9281 elementPairIsContiguous(Mask: SVN->getMask(), Elt: I)) {
9282 const int Idx = SVN->getMaskElt(Idx: I);
9283 int VecIdx = Idx < SrcNumElts ? 0 : 1;
9284 int EltIdx = Idx < SrcNumElts ? Idx : Idx - SrcNumElts;
9285 SDValue SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT,
9286 N1: SVN->getOperand(Num: VecIdx),
9287 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
9288 Pieces.push_back(Elt: SubVec);
9289 } else if (elementPairIsOddToEven(Mask: SVN->getMask(), Elt: I) &&
9290 isOperationLegal(Op: ISD::VECTOR_SHUFFLE, VT: PackVT)) {
9291 int Idx0 = SVN->getMaskElt(Idx: I);
9292 int Idx1 = SVN->getMaskElt(Idx: I + 1);
9293
9294 SDValue SrcOp0 = SVN->getOperand(Num: 0);
9295 SDValue SrcOp1 = SrcOp0;
9296 if (Idx0 >= SrcNumElts) {
9297 SrcOp0 = SVN->getOperand(Num: 1);
9298 Idx0 -= SrcNumElts;
9299 }
9300
9301 if (Idx1 >= SrcNumElts) {
9302 SrcOp1 = SVN->getOperand(Num: 1);
9303 Idx1 -= SrcNumElts;
9304 }
9305
9306 int AlignedIdx0 = Idx0 & ~(NewSrcNumElts - 1);
9307 int AlignedIdx1 = Idx1 & ~(NewSrcNumElts - 1);
9308
9309 // Extract nearest even aligned piece.
9310 SDValue SubVec0 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT, N1: SrcOp0,
9311 N2: DAG.getConstant(Val: AlignedIdx0, DL: SL, VT: MVT::i32));
9312 SDValue SubVec1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT, N1: SrcOp1,
9313 N2: DAG.getConstant(Val: AlignedIdx1, DL: SL, VT: MVT::i32));
9314
9315 int NewMaskIdx0 = Idx0 - AlignedIdx0;
9316 int NewMaskIdx1 = Idx1 - AlignedIdx1;
9317
9318 SDValue Result0 = SubVec0;
9319 SDValue Result1 = SubVec0;
9320
9321 if (SubVec0 != SubVec1) {
9322 NewMaskIdx1 += NewSrcNumElts;
9323 Result1 = SubVec1;
9324 } else {
9325 Result1 = DAG.getPOISON(VT: PackVT);
9326 }
9327
9328 SDValue Shuf = DAG.getVectorShuffle(VT: PackVT, dl: SL, N1: Result0, N2: Result1,
9329 Mask: {NewMaskIdx0, NewMaskIdx1});
9330 Pieces.push_back(Elt: Shuf);
9331 } else {
9332 const int Idx0 = SVN->getMaskElt(Idx: I);
9333 const int Idx1 = SVN->getMaskElt(Idx: I + 1);
9334 int VecIdx0 = Idx0 < SrcNumElts ? 0 : 1;
9335 int VecIdx1 = Idx1 < SrcNumElts ? 0 : 1;
9336 int EltIdx0 = Idx0 < SrcNumElts ? Idx0 : Idx0 - SrcNumElts;
9337 int EltIdx1 = Idx1 < SrcNumElts ? Idx1 : Idx1 - SrcNumElts;
9338
9339 SDValue Vec0 = SVN->getOperand(Num: VecIdx0);
9340 SDValue Elt0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec0,
9341 N2: DAG.getSignedConstant(Val: EltIdx0, DL: SL, VT: MVT::i32));
9342
9343 SDValue Vec1 = SVN->getOperand(Num: VecIdx1);
9344 SDValue Elt1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec1,
9345 N2: DAG.getSignedConstant(Val: EltIdx1, DL: SL, VT: MVT::i32));
9346 Pieces.push_back(Elt: DAG.getBuildVector(VT: PackVT, DL: SL, Ops: {Elt0, Elt1}));
9347 }
9348 }
9349
9350 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: ResultVT, Ops: Pieces);
9351}
9352
9353SDValue SITargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
9354 SelectionDAG &DAG) const {
9355 SDValue SVal = Op.getOperand(i: 0);
9356 EVT ResultVT = Op.getValueType();
9357 EVT SValVT = SVal.getValueType();
9358 SDValue UndefVal = DAG.getPOISON(VT: SValVT);
9359 SDLoc SL(Op);
9360
9361 SmallVector<SDValue, 8> VElts;
9362 VElts.push_back(Elt: SVal);
9363 for (int I = 1, E = ResultVT.getVectorNumElements(); I < E; ++I)
9364 VElts.push_back(Elt: UndefVal);
9365
9366 return DAG.getBuildVector(VT: ResultVT, DL: SL, Ops: VElts);
9367}
9368
9369SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op,
9370 SelectionDAG &DAG) const {
9371 SDLoc SL(Op);
9372 EVT VT = Op.getValueType();
9373
9374 if (VT == MVT::v2f16 || VT == MVT::v2i16 || VT == MVT::v2bf16) {
9375 assert(!Subtarget->hasVOP3PInsts() && "this should be legal");
9376
9377 SDValue Lo = Op.getOperand(i: 0);
9378 SDValue Hi = Op.getOperand(i: 1);
9379
9380 // Avoid adding defined bits with the zero_extend.
9381 if (Hi.isUndef()) {
9382 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Lo);
9383 SDValue ExtLo = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Lo);
9384 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: ExtLo);
9385 }
9386
9387 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Hi);
9388 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Hi);
9389
9390 SDValue ShlHi = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Hi,
9391 N2: DAG.getConstant(Val: 16, DL: SL, VT: MVT::i32));
9392 if (Lo.isUndef())
9393 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: ShlHi);
9394
9395 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Lo);
9396 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Lo);
9397
9398 SDValue Or =
9399 DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Lo, N2: ShlHi, Flags: SDNodeFlags::Disjoint);
9400 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Or);
9401 }
9402
9403 // Split into 2-element chunks.
9404 const unsigned NumParts = VT.getVectorNumElements() / 2;
9405 EVT PartVT = MVT::getVectorVT(VT: VT.getVectorElementType().getSimpleVT(), NumElements: 2);
9406 MVT PartIntVT = MVT::getIntegerVT(BitWidth: PartVT.getSizeInBits());
9407
9408 SmallVector<SDValue> Casts;
9409 for (unsigned P = 0; P < NumParts; ++P) {
9410 SDValue Vec = DAG.getBuildVector(
9411 VT: PartVT, DL: SL, Ops: {Op.getOperand(i: P * 2), Op.getOperand(i: P * 2 + 1)});
9412 Casts.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: PartIntVT, Operand: Vec));
9413 }
9414
9415 SDValue Blend =
9416 DAG.getBuildVector(VT: MVT::getVectorVT(VT: PartIntVT, NumElements: NumParts), DL: SL, Ops: Casts);
9417 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Blend);
9418}
9419
9420bool SITargetLowering::isOffsetFoldingLegal(
9421 const GlobalAddressSDNode *GA) const {
9422 // OSes that use ELF REL relocations (instead of RELA) can only store a
9423 // 32-bit addend in the instruction, so it is not safe to allow offset folding
9424 // which can create arbitrary 64-bit addends. (This is only a problem for
9425 // R_AMDGPU_*32_HI relocations since other relocation types are unaffected by
9426 // the high 32 bits of the addend.)
9427 //
9428 // This should be kept in sync with how HasRelocationAddend is initialized in
9429 // the constructor of ELFAMDGPUAsmBackend.
9430 if (!Subtarget->isAmdHsaOS())
9431 return false;
9432
9433 // We can fold offsets for anything that doesn't require a GOT relocation.
9434 return (GA->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS ||
9435 GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
9436 GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
9437 !shouldEmitGOTReloc(GV: GA->getGlobal());
9438}
9439
9440static SDValue
9441buildPCRelGlobalAddress(SelectionDAG &DAG, const GlobalValue *GV,
9442 const SDLoc &DL, int64_t Offset, EVT PtrVT,
9443 unsigned GAFlags = SIInstrInfo::MO_NONE) {
9444 assert(isInt<32>(Offset + 4) && "32-bit offset is expected!");
9445 // In order to support pc-relative addressing, the PC_ADD_REL_OFFSET SDNode is
9446 // lowered to the following code sequence:
9447 //
9448 // For constant address space:
9449 // s_getpc_b64 s[0:1]
9450 // s_add_u32 s0, s0, $symbol
9451 // s_addc_u32 s1, s1, 0
9452 //
9453 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
9454 // a fixup or relocation is emitted to replace $symbol with a literal
9455 // constant, which is a pc-relative offset from the encoding of the $symbol
9456 // operand to the global variable.
9457 //
9458 // For global address space:
9459 // s_getpc_b64 s[0:1]
9460 // s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
9461 // s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
9462 //
9463 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
9464 // fixups or relocations are emitted to replace $symbol@*@lo and
9465 // $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
9466 // which is a 64-bit pc-relative offset from the encoding of the $symbol
9467 // operand to the global variable.
9468 if (((const GCNSubtarget &)DAG.getSubtarget()).has64BitLiterals()) {
9469 assert(GAFlags != SIInstrInfo::MO_NONE);
9470
9471 SDValue Ptr =
9472 DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i64, offset: Offset, TargetFlags: GAFlags + 2);
9473 return DAG.getNode(Opcode: AMDGPUISD::PC_ADD_REL_OFFSET64, DL, VT: PtrVT, Operand: Ptr);
9474 }
9475
9476 SDValue PtrLo = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: Offset, TargetFlags: GAFlags);
9477 SDValue PtrHi;
9478 if (GAFlags == SIInstrInfo::MO_NONE)
9479 PtrHi = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32);
9480 else
9481 PtrHi = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: Offset, TargetFlags: GAFlags + 1);
9482 return DAG.getNode(Opcode: AMDGPUISD::PC_ADD_REL_OFFSET, DL, VT: PtrVT, N1: PtrLo, N2: PtrHi);
9483}
9484
9485SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
9486 SDValue Op,
9487 SelectionDAG &DAG) const {
9488 GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Val&: Op);
9489 SDLoc DL(GSD);
9490 EVT PtrVT = Op.getValueType();
9491
9492 const GlobalValue *GV = GSD->getGlobal();
9493 if ((GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS &&
9494 shouldUseLDSConstAddress(GV)) ||
9495 GSD->getAddressSpace() == AMDGPUAS::REGION_ADDRESS ||
9496 GSD->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) {
9497 if (GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS &&
9498 GV->hasExternalLinkage()) {
9499 const GlobalVariable &GVar = *cast<GlobalVariable>(Val: GV);
9500 // HIP uses an unsized array `extern __shared__ T s[]` or similar
9501 // zero-sized type in other languages to declare the dynamic shared
9502 // memory which size is not known at the compile time. They will be
9503 // allocated by the runtime and placed directly after the static
9504 // allocated ones. They all share the same offset.
9505 if (GVar.getGlobalSize(DL: GVar.getDataLayout()) == 0) {
9506 assert(PtrVT == MVT::i32 && "32-bit pointer is expected.");
9507 // Adjust alignment for that dynamic shared memory array.
9508 Function &F = DAG.getMachineFunction().getFunction();
9509 MFI->setDynLDSAlign(F, GV: GVar);
9510 MFI->setUsesDynamicLDS(true);
9511 return SDValue(
9512 DAG.getMachineNode(Opcode: AMDGPU::GET_GROUPSTATICSIZE, dl: DL, VT: PtrVT), 0);
9513 }
9514 }
9515 return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG);
9516 }
9517
9518 if (GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
9519 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: GSD->getOffset(),
9520 TargetFlags: SIInstrInfo::MO_ABS32_LO);
9521 return DAG.getNode(Opcode: AMDGPUISD::LDS, DL, VT: MVT::i32, Operand: GA);
9522 }
9523
9524 if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS()) {
9525 if (Subtarget->has64BitLiterals()) {
9526 SDValue Addr = DAG.getTargetGlobalAddress(
9527 GV, DL, VT: MVT::i64, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS64);
9528 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B64, dl: DL, VT: MVT::i64, Op1: Addr),
9529 0);
9530 }
9531
9532 SDValue AddrLo = DAG.getTargetGlobalAddress(
9533 GV, DL, VT: MVT::i32, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS32_LO);
9534 AddrLo = {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: AddrLo), 0};
9535
9536 SDValue AddrHi = DAG.getTargetGlobalAddress(
9537 GV, DL, VT: MVT::i32, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS32_HI);
9538 AddrHi = {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: AddrHi), 0};
9539
9540 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: AddrLo, N2: AddrHi);
9541 }
9542
9543 if (shouldEmitFixup(GV))
9544 return buildPCRelGlobalAddress(DAG, GV, DL, Offset: GSD->getOffset(), PtrVT);
9545
9546 if (shouldEmitPCReloc(GV))
9547 return buildPCRelGlobalAddress(DAG, GV, DL, Offset: GSD->getOffset(), PtrVT,
9548 GAFlags: SIInstrInfo::MO_REL32);
9549
9550 SDValue GOTAddr = buildPCRelGlobalAddress(DAG, GV, DL, Offset: 0, PtrVT,
9551 GAFlags: SIInstrInfo::MO_GOTPCREL32);
9552 PointerType *PtrTy =
9553 PointerType::get(C&: *DAG.getContext(), AddressSpace: AMDGPUAS::CONSTANT_ADDRESS);
9554 const DataLayout &DataLayout = DAG.getDataLayout();
9555 Align Alignment = DataLayout.getABITypeAlign(Ty: PtrTy);
9556 MachinePointerInfo PtrInfo =
9557 MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction());
9558
9559 return DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: GOTAddr, PtrInfo, Alignment,
9560 MMOFlags: MachineMemOperand::MODereferenceable |
9561 MachineMemOperand::MOInvariant);
9562}
9563
9564SDValue SITargetLowering::LowerExternalSymbol(SDValue Op,
9565 SelectionDAG &DAG) const {
9566 // TODO: Handle this. It should be mostly the same as LowerGlobalAddress.
9567 const Function &Fn = DAG.getMachineFunction().getFunction();
9568 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
9569 Fn, "unsupported external symbol", Op.getDebugLoc()));
9570 return DAG.getPOISON(VT: Op.getValueType());
9571}
9572
9573SDValue SITargetLowering::copyToM0(SelectionDAG &DAG, SDValue Chain,
9574 const SDLoc &DL, SDValue V) const {
9575 // We can't use S_MOV_B32 directly, because there is no way to specify m0 as
9576 // the destination register.
9577 //
9578 // We can't use CopyToReg, because MachineCSE won't combine COPY instructions,
9579 // so we will end up with redundant moves to m0.
9580 //
9581 // We use a pseudo to ensure we emit s_mov_b32 with m0 as the direct result.
9582
9583 // A Null SDValue creates a glue result.
9584 SDNode *M0 = DAG.getMachineNode(Opcode: AMDGPU::SI_INIT_M0, dl: DL, VT1: MVT::Other, VT2: MVT::Glue,
9585 Op1: V, Op2: Chain);
9586 return SDValue(M0, 0);
9587}
9588
9589SDValue SITargetLowering::lowerImplicitZextParam(SelectionDAG &DAG, SDValue Op,
9590 MVT VT,
9591 unsigned Offset) const {
9592 SDLoc SL(Op);
9593 SDValue Param = lowerKernargMemParameter(
9594 DAG, VT: MVT::i32, MemVT: MVT::i32, SL, Chain: DAG.getEntryNode(), Offset, Alignment: Align(4), Signed: false);
9595 // The local size values will have the hi 16-bits as zero.
9596 return DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: MVT::i32, N1: Param,
9597 N2: DAG.getValueType(VT));
9598}
9599
9600static SDValue emitNonHSAIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
9601 EVT VT) {
9602 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
9603 DAG.getMachineFunction().getFunction(),
9604 "non-hsa intrinsic with hsa target", DL.getDebugLoc()));
9605 return DAG.getPOISON(VT);
9606}
9607
9608static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
9609 EVT VT) {
9610 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
9611 DAG.getMachineFunction().getFunction(),
9612 "intrinsic not supported on subtarget", DL.getDebugLoc()));
9613 return DAG.getPOISON(VT);
9614}
9615
9616static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL,
9617 ArrayRef<SDValue> Elts) {
9618 assert(!Elts.empty());
9619 MVT Type;
9620 unsigned NumElts = Elts.size();
9621
9622 if (NumElts <= 12) {
9623 Type = MVT::getVectorVT(VT: MVT::f32, NumElements: NumElts);
9624 } else {
9625 assert(Elts.size() <= 16);
9626 Type = MVT::v16f32;
9627 NumElts = 16;
9628 }
9629
9630 SmallVector<SDValue, 16> VecElts(NumElts);
9631 for (unsigned i = 0; i < Elts.size(); ++i) {
9632 SDValue Elt = Elts[i];
9633 if (Elt.getValueType() != MVT::f32)
9634 Elt = DAG.getBitcast(VT: MVT::f32, V: Elt);
9635 VecElts[i] = Elt;
9636 }
9637 for (unsigned i = Elts.size(); i < NumElts; ++i)
9638 VecElts[i] = DAG.getPOISON(VT: MVT::f32);
9639
9640 if (NumElts == 1)
9641 return VecElts[0];
9642 return DAG.getBuildVector(VT: Type, DL, Ops: VecElts);
9643}
9644
9645static SDValue padEltsToUndef(SelectionDAG &DAG, const SDLoc &DL, EVT CastVT,
9646 SDValue Src, int ExtraElts) {
9647 EVT SrcVT = Src.getValueType();
9648
9649 SmallVector<SDValue, 8> Elts;
9650
9651 if (SrcVT.isVector())
9652 DAG.ExtractVectorElements(Op: Src, Args&: Elts);
9653 else
9654 Elts.push_back(Elt: Src);
9655
9656 SDValue Undef = DAG.getPOISON(VT: SrcVT.getScalarType());
9657 while (ExtraElts--)
9658 Elts.push_back(Elt: Undef);
9659
9660 return DAG.getBuildVector(VT: CastVT, DL, Ops: Elts);
9661}
9662
9663// Re-construct the required return value for a image load intrinsic.
9664// This is more complicated due to the optional use TexFailCtrl which means the
9665// required return type is an aggregate
9666static SDValue constructRetValue(SelectionDAG &DAG, MachineSDNode *Result,
9667 ArrayRef<EVT> ResultTypes, bool IsTexFail,
9668 bool Unpacked, bool IsD16, int DMaskPop,
9669 int NumVDataDwords, bool IsAtomicPacked16Bit,
9670 const SDLoc &DL) {
9671 // Determine the required return type. This is the same regardless of
9672 // IsTexFail flag
9673 EVT ReqRetVT = ResultTypes[0];
9674 int ReqRetNumElts = ReqRetVT.isVector() ? ReqRetVT.getVectorNumElements() : 1;
9675 int NumDataDwords = ((IsD16 && !Unpacked) || IsAtomicPacked16Bit)
9676 ? (ReqRetNumElts + 1) / 2
9677 : ReqRetNumElts;
9678
9679 int MaskPopDwords = (!IsD16 || Unpacked) ? DMaskPop : (DMaskPop + 1) / 2;
9680
9681 MVT DataDwordVT =
9682 NumDataDwords == 1 ? MVT::i32 : MVT::getVectorVT(VT: MVT::i32, NumElements: NumDataDwords);
9683
9684 MVT MaskPopVT =
9685 MaskPopDwords == 1 ? MVT::i32 : MVT::getVectorVT(VT: MVT::i32, NumElements: MaskPopDwords);
9686
9687 SDValue Data(Result, 0);
9688 SDValue TexFail;
9689
9690 if (DMaskPop > 0 && Data.getValueType() != MaskPopVT) {
9691 SDValue ZeroIdx = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
9692 if (MaskPopVT.isVector()) {
9693 Data = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: MaskPopVT,
9694 N1: SDValue(Result, 0), N2: ZeroIdx);
9695 } else {
9696 Data = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MaskPopVT,
9697 N1: SDValue(Result, 0), N2: ZeroIdx);
9698 }
9699 }
9700
9701 if (DataDwordVT.isVector() && !IsAtomicPacked16Bit)
9702 Data = padEltsToUndef(DAG, DL, CastVT: DataDwordVT, Src: Data,
9703 ExtraElts: NumDataDwords - MaskPopDwords);
9704
9705 if (IsD16)
9706 Data = adjustLoadValueTypeImpl(Result: Data, LoadVT: ReqRetVT, DL, DAG, Unpacked);
9707
9708 EVT LegalReqRetVT = ReqRetVT;
9709 if (!ReqRetVT.isVector()) {
9710 if (!Data.getValueType().isInteger())
9711 Data = DAG.getNode(Opcode: ISD::BITCAST, DL,
9712 VT: Data.getValueType().changeTypeToInteger(), Operand: Data);
9713 Data = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ReqRetVT.changeTypeToInteger(), Operand: Data);
9714 } else {
9715 // We need to widen the return vector to a legal type
9716 if ((ReqRetVT.getVectorNumElements() % 2) == 1 &&
9717 ReqRetVT.getVectorElementType().getSizeInBits() == 16) {
9718 LegalReqRetVT =
9719 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ReqRetVT.getVectorElementType(),
9720 NumElements: ReqRetVT.getVectorNumElements() + 1);
9721 }
9722 }
9723 Data = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LegalReqRetVT, Operand: Data);
9724
9725 if (IsTexFail) {
9726 TexFail =
9727 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: SDValue(Result, 0),
9728 N2: DAG.getConstant(Val: MaskPopDwords, DL, VT: MVT::i32));
9729
9730 return DAG.getMergeValues(Ops: {Data, TexFail, SDValue(Result, 1)}, dl: DL);
9731 }
9732
9733 if (Result->getNumValues() == 1)
9734 return Data;
9735
9736 return DAG.getMergeValues(Ops: {Data, SDValue(Result, 1)}, dl: DL);
9737}
9738
9739static bool parseTexFail(SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE,
9740 SDValue *LWE, bool &IsTexFail) {
9741 auto *TexFailCtrlConst = cast<ConstantSDNode>(Val: TexFailCtrl.getNode());
9742
9743 uint64_t Value = TexFailCtrlConst->getZExtValue();
9744 if (Value) {
9745 IsTexFail = true;
9746 }
9747
9748 SDLoc DL(TexFailCtrlConst);
9749 *TFE = DAG.getTargetConstant(Val: (Value & 0x1) ? 1 : 0, DL, VT: MVT::i32);
9750 Value &= ~(uint64_t)0x1;
9751 *LWE = DAG.getTargetConstant(Val: (Value & 0x2) ? 1 : 0, DL, VT: MVT::i32);
9752 Value &= ~(uint64_t)0x2;
9753
9754 return Value == 0;
9755}
9756
9757static void packImage16bitOpsToDwords(SelectionDAG &DAG, SDValue Op,
9758 MVT PackVectorVT,
9759 SmallVectorImpl<SDValue> &PackedAddrs,
9760 unsigned DimIdx, unsigned EndIdx,
9761 unsigned NumGradients) {
9762 SDLoc DL(Op);
9763 for (unsigned I = DimIdx; I < EndIdx; I++) {
9764 SDValue Addr = Op.getOperand(i: I);
9765
9766 // Gradients are packed with undef for each coordinate.
9767 // In <hi 16 bit>,<lo 16 bit> notation, the registers look like this:
9768 // 1D: undef,dx/dh; undef,dx/dv
9769 // 2D: dy/dh,dx/dh; dy/dv,dx/dv
9770 // 3D: dy/dh,dx/dh; undef,dz/dh; dy/dv,dx/dv; undef,dz/dv
9771 if (((I + 1) >= EndIdx) ||
9772 ((NumGradients / 2) % 2 == 1 && (I == DimIdx + (NumGradients / 2) - 1 ||
9773 I == DimIdx + NumGradients - 1))) {
9774 if (Addr.getValueType() != MVT::i16)
9775 Addr = DAG.getBitcast(VT: MVT::i16, V: Addr);
9776 Addr = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Addr);
9777 } else {
9778 Addr = DAG.getBuildVector(VT: PackVectorVT, DL, Ops: {Addr, Op.getOperand(i: I + 1)});
9779 I++;
9780 }
9781 Addr = DAG.getBitcast(VT: MVT::f32, V: Addr);
9782 PackedAddrs.push_back(Elt: Addr);
9783 }
9784}
9785
9786SDValue SITargetLowering::lowerImage(SDValue Op,
9787 const AMDGPU::ImageDimIntrinsicInfo *Intr,
9788 SelectionDAG &DAG, bool WithChain) const {
9789 SDLoc DL(Op);
9790 MachineFunction &MF = DAG.getMachineFunction();
9791 const GCNSubtarget *ST = &MF.getSubtarget<GCNSubtarget>();
9792 unsigned IntrOpcode = Intr->BaseOpcode;
9793 // For image atomic: use no-return opcode if result is unused.
9794 if (Intr->AtomicNoRetBaseOpcode != Intr->BaseOpcode &&
9795 !Op.getNode()->hasAnyUseOfValue(Value: 0))
9796 IntrOpcode = Intr->AtomicNoRetBaseOpcode;
9797 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
9798 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: IntrOpcode);
9799 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(DimEnum: Intr->Dim);
9800 bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI: *Subtarget);
9801 bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI: *Subtarget);
9802 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
9803
9804 SmallVector<EVT, 3> ResultTypes(Op->values());
9805 SmallVector<EVT, 3> OrigResultTypes(Op->values());
9806 if (BaseOpcode->NoReturn && BaseOpcode->Atomic)
9807 ResultTypes.erase(CI: &ResultTypes[0]);
9808
9809 bool IsD16 = false;
9810 bool IsG16 = false;
9811 bool IsA16 = false;
9812 SDValue VData;
9813 int NumVDataDwords = 0;
9814 bool AdjustRetType = false;
9815 bool IsAtomicPacked16Bit = false;
9816
9817 // Offset of intrinsic arguments
9818 const unsigned ArgOffset = WithChain ? 2 : 1;
9819
9820 unsigned DMask;
9821 unsigned DMaskLanes = 0;
9822
9823 if (BaseOpcode->Atomic) {
9824 VData = Op.getOperand(i: 2);
9825
9826 IsAtomicPacked16Bit =
9827 (IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
9828 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16_NORTN ||
9829 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16 ||
9830 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16_NORTN);
9831
9832 bool Is64Bit = VData.getValueSizeInBits() == 64;
9833 if (BaseOpcode->AtomicX2) {
9834 SDValue VData2 = Op.getOperand(i: 3);
9835 VData = DAG.getBuildVector(VT: Is64Bit ? MVT::v2i64 : MVT::v2i32, DL,
9836 Ops: {VData, VData2});
9837 if (Is64Bit)
9838 VData = DAG.getBitcast(VT: MVT::v4i32, V: VData);
9839
9840 if (!BaseOpcode->NoReturn)
9841 ResultTypes[0] = Is64Bit ? MVT::v2i64 : MVT::v2i32;
9842
9843 DMask = Is64Bit ? 0xf : 0x3;
9844 NumVDataDwords = Is64Bit ? 4 : 2;
9845 } else {
9846 DMask = Is64Bit ? 0x3 : 0x1;
9847 NumVDataDwords = Is64Bit ? 2 : 1;
9848 }
9849 } else {
9850 DMask = Op->getConstantOperandVal(Num: ArgOffset + Intr->DMaskIndex);
9851 DMaskLanes = BaseOpcode->Gather4 ? 4 : llvm::popcount(Value: DMask);
9852
9853 if (BaseOpcode->Store) {
9854 VData = Op.getOperand(i: 2);
9855
9856 MVT StoreVT = VData.getSimpleValueType();
9857 if (StoreVT.getScalarType() == MVT::f16) {
9858 if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
9859 return Op; // D16 is unsupported for this instruction
9860
9861 IsD16 = true;
9862 VData = handleD16VData(VData, DAG, ImageStore: true);
9863 }
9864
9865 NumVDataDwords = (VData.getValueType().getSizeInBits() + 31) / 32;
9866 } else if (!BaseOpcode->NoReturn) {
9867 // Work out the num dwords based on the dmask popcount and underlying type
9868 // and whether packing is supported.
9869 MVT LoadVT = ResultTypes[0].getSimpleVT();
9870 if (LoadVT.getScalarType() == MVT::f16) {
9871 if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
9872 return Op; // D16 is unsupported for this instruction
9873
9874 IsD16 = true;
9875 }
9876
9877 // Confirm that the return type is large enough for the dmask specified
9878 if ((LoadVT.isVector() && LoadVT.getVectorNumElements() < DMaskLanes) ||
9879 (!LoadVT.isVector() && DMaskLanes > 1))
9880 return Op;
9881
9882 // The sq block of gfx8 and gfx9 do not estimate register use correctly
9883 // for d16 image_gather4, image_gather4_l, and image_gather4_lz
9884 // instructions.
9885 if (IsD16 && !Subtarget->hasUnpackedD16VMem() &&
9886 !(BaseOpcode->Gather4 && Subtarget->hasImageGather4D16Bug()))
9887 NumVDataDwords = (DMaskLanes + 1) / 2;
9888 else
9889 NumVDataDwords = DMaskLanes;
9890
9891 AdjustRetType = true;
9892 }
9893 }
9894
9895 unsigned VAddrEnd = ArgOffset + Intr->VAddrEnd;
9896 SmallVector<SDValue, 4> VAddrs;
9897
9898 // Check for 16 bit addresses or derivatives and pack if true.
9899 MVT VAddrVT =
9900 Op.getOperand(i: ArgOffset + Intr->GradientStart).getSimpleValueType();
9901 MVT VAddrScalarVT = VAddrVT.getScalarType();
9902 MVT GradPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
9903 IsG16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
9904
9905 VAddrVT = Op.getOperand(i: ArgOffset + Intr->CoordStart).getSimpleValueType();
9906 VAddrScalarVT = VAddrVT.getScalarType();
9907 MVT AddrPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
9908 IsA16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
9909
9910 // Push back extra arguments.
9911 for (unsigned I = Intr->VAddrStart; I < Intr->GradientStart; I++) {
9912 if (IsA16 && (Op.getOperand(i: ArgOffset + I).getValueType() == MVT::f16)) {
9913 assert(I == Intr->BiasIndex && "Got unexpected 16-bit extra argument");
9914 // Special handling of bias when A16 is on. Bias is of type half but
9915 // occupies full 32-bit.
9916 SDValue Bias = DAG.getBuildVector(
9917 VT: MVT::v2f16, DL,
9918 Ops: {Op.getOperand(i: ArgOffset + I), DAG.getPOISON(VT: MVT::f16)});
9919 VAddrs.push_back(Elt: Bias);
9920 } else {
9921 assert((!IsA16 || Intr->NumBiasArgs == 0 || I != Intr->BiasIndex) &&
9922 "Bias needs to be converted to 16 bit in A16 mode");
9923 VAddrs.push_back(Elt: Op.getOperand(i: ArgOffset + I));
9924 }
9925 }
9926
9927 if (BaseOpcode->Gradients && !ST->hasG16() && (IsA16 != IsG16)) {
9928 // 16 bit gradients are supported, but are tied to the A16 control
9929 // so both gradients and addresses must be 16 bit
9930 LLVM_DEBUG(
9931 dbgs() << "Failed to lower image intrinsic: 16 bit addresses "
9932 "require 16 bit args for both gradients and addresses");
9933 return Op;
9934 }
9935
9936 if (IsA16) {
9937 if (!ST->hasA16()) {
9938 LLVM_DEBUG(dbgs() << "Failed to lower image intrinsic: Target does not "
9939 "support 16 bit addresses\n");
9940 return Op;
9941 }
9942 }
9943
9944 // We've dealt with incorrect input so we know that if IsA16, IsG16
9945 // are set then we have to compress/pack operands (either address,
9946 // gradient or both)
9947 // In the case where a16 and gradients are tied (no G16 support) then we
9948 // have already verified that both IsA16 and IsG16 are true
9949 if (BaseOpcode->Gradients && IsG16 && ST->hasG16()) {
9950 // Activate g16
9951 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
9952 AMDGPU::getMIMGG16MappingInfo(G: Intr->BaseOpcode);
9953 IntrOpcode = G16MappingInfo->G16; // set new opcode to variant with _g16
9954 }
9955
9956 // Add gradients (packed or unpacked)
9957 if (IsG16) {
9958 // Pack the gradients
9959 // const int PackEndIdx = IsA16 ? VAddrEnd : (ArgOffset + Intr->CoordStart);
9960 packImage16bitOpsToDwords(DAG, Op, PackVectorVT: GradPackVectorVT, PackedAddrs&: VAddrs,
9961 DimIdx: ArgOffset + Intr->GradientStart,
9962 EndIdx: ArgOffset + Intr->CoordStart, NumGradients: Intr->NumGradients);
9963 } else {
9964 for (unsigned I = ArgOffset + Intr->GradientStart;
9965 I < ArgOffset + Intr->CoordStart; I++)
9966 VAddrs.push_back(Elt: Op.getOperand(i: I));
9967 }
9968
9969 // Add addresses (packed or unpacked)
9970 if (IsA16) {
9971 packImage16bitOpsToDwords(DAG, Op, PackVectorVT: AddrPackVectorVT, PackedAddrs&: VAddrs,
9972 DimIdx: ArgOffset + Intr->CoordStart, EndIdx: VAddrEnd,
9973 NumGradients: 0 /* No gradients */);
9974 } else {
9975 // Add uncompressed address
9976 for (unsigned I = ArgOffset + Intr->CoordStart; I < VAddrEnd; I++)
9977 VAddrs.push_back(Elt: Op.getOperand(i: I));
9978 }
9979
9980 // If the register allocator cannot place the address registers contiguously
9981 // without introducing moves, then using the non-sequential address encoding
9982 // is always preferable, since it saves VALU instructions and is usually a
9983 // wash in terms of code size or even better.
9984 //
9985 // However, we currently have no way of hinting to the register allocator that
9986 // MIMG addresses should be placed contiguously when it is possible to do so,
9987 // so force non-NSA for the common 2-address case as a heuristic.
9988 //
9989 // SIShrinkInstructions will convert NSA encodings to non-NSA after register
9990 // allocation when possible.
9991 //
9992 // Partial NSA is allowed on GFX11+ where the final register is a contiguous
9993 // set of the remaining addresses.
9994 const unsigned NSAMaxSize = ST->getNSAMaxSize(HasSampler: BaseOpcode->Sampler);
9995 const bool HasPartialNSAEncoding = ST->hasPartialNSAEncoding();
9996 const bool UseNSA = ST->hasNSAEncoding() &&
9997 VAddrs.size() >= ST->getNSAThreshold(MF) &&
9998 (VAddrs.size() <= NSAMaxSize || HasPartialNSAEncoding);
9999 const bool UsePartialNSA =
10000 UseNSA && HasPartialNSAEncoding && VAddrs.size() > NSAMaxSize;
10001
10002 SDValue VAddr;
10003 if (UsePartialNSA) {
10004 VAddr = getBuildDwordsVector(DAG, DL,
10005 Elts: ArrayRef(VAddrs).drop_front(N: NSAMaxSize - 1));
10006 } else if (!UseNSA) {
10007 VAddr = getBuildDwordsVector(DAG, DL, Elts: VAddrs);
10008 }
10009
10010 SDValue True = DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1);
10011 SDValue False = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1);
10012 SDValue Unorm;
10013 if (!BaseOpcode->Sampler) {
10014 Unorm = True;
10015 } else {
10016 uint64_t UnormConst =
10017 Op.getConstantOperandVal(i: ArgOffset + Intr->UnormIndex);
10018
10019 Unorm = UnormConst ? True : False;
10020 }
10021
10022 SDValue TFE;
10023 SDValue LWE;
10024 SDValue TexFail = Op.getOperand(i: ArgOffset + Intr->TexFailCtrlIndex);
10025 bool IsTexFail = false;
10026 if (!parseTexFail(TexFailCtrl: TexFail, DAG, TFE: &TFE, LWE: &LWE, IsTexFail))
10027 return Op;
10028
10029 if (IsTexFail) {
10030 if (!DMaskLanes) {
10031 // Expecting to get an error flag since TFC is on - and dmask is 0
10032 // Force dmask to be at least 1 otherwise the instruction will fail
10033 DMask = 0x1;
10034 DMaskLanes = 1;
10035 NumVDataDwords = 1;
10036 }
10037 NumVDataDwords += 1;
10038 AdjustRetType = true;
10039 }
10040
10041 // Has something earlier tagged that the return type needs adjusting
10042 // This happens if the instruction is a load or has set TexFailCtrl flags
10043 if (AdjustRetType) {
10044 // NumVDataDwords reflects the true number of dwords required in the return
10045 // type
10046 if (DMaskLanes == 0 && !BaseOpcode->Store) {
10047 // This is a no-op load. This can be eliminated
10048 SDValue Undef = DAG.getPOISON(VT: Op.getValueType());
10049 if (isa<MemSDNode>(Val: Op))
10050 return DAG.getMergeValues(Ops: {Undef, Op.getOperand(i: 0)}, dl: DL);
10051 return Undef;
10052 }
10053
10054 EVT NewVT = NumVDataDwords > 1 ? EVT::getVectorVT(Context&: *DAG.getContext(),
10055 VT: MVT::i32, NumElements: NumVDataDwords)
10056 : MVT::i32;
10057
10058 ResultTypes[0] = NewVT;
10059 if (ResultTypes.size() == 3) {
10060 // Original result was aggregate type used for TexFailCtrl results
10061 // The actual instruction returns as a vector type which has now been
10062 // created. Remove the aggregate result.
10063 ResultTypes.erase(CI: &ResultTypes[1]);
10064 }
10065 }
10066
10067 unsigned CPol = Op.getConstantOperandVal(i: ArgOffset + Intr->CachePolicyIndex);
10068 // Keep GLC only when the atomic's result is actually used.
10069 if (BaseOpcode->Atomic && !BaseOpcode->NoReturn)
10070 CPol |= AMDGPU::CPol::GLC;
10071 if (CPol & ~((IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12) |
10072 AMDGPU::CPol::VOLATILE))
10073 return Op;
10074
10075 SmallVector<SDValue, 26> Ops;
10076 if (BaseOpcode->Store || BaseOpcode->Atomic)
10077 Ops.push_back(Elt: VData); // vdata
10078 if (UsePartialNSA) {
10079 append_range(C&: Ops, R: ArrayRef(VAddrs).take_front(N: NSAMaxSize - 1));
10080 Ops.push_back(Elt: VAddr);
10081 } else if (UseNSA)
10082 append_range(C&: Ops, R&: VAddrs);
10083 else
10084 Ops.push_back(Elt: VAddr);
10085 SDValue Rsrc = Op.getOperand(i: ArgOffset + Intr->RsrcIndex);
10086 EVT RsrcVT = Rsrc.getValueType();
10087 if (RsrcVT != MVT::v4i32 && RsrcVT != MVT::v8i32)
10088 return Op;
10089 Ops.push_back(Elt: Rsrc);
10090 if (BaseOpcode->Sampler) {
10091 SDValue Samp = Op.getOperand(i: ArgOffset + Intr->SampIndex);
10092 if (Samp.getValueType() != MVT::v4i32)
10093 return Op;
10094 Ops.push_back(Elt: Samp);
10095 }
10096 Ops.push_back(Elt: DAG.getTargetConstant(Val: DMask, DL, VT: MVT::i32));
10097 if (IsGFX10Plus)
10098 Ops.push_back(Elt: DAG.getTargetConstant(Val: DimInfo->Encoding, DL, VT: MVT::i32));
10099 if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
10100 Ops.push_back(Elt: Unorm);
10101 Ops.push_back(Elt: DAG.getTargetConstant(Val: CPol, DL, VT: MVT::i32));
10102 Ops.push_back(Elt: IsA16 && // r128, a16 for gfx9
10103 ST->hasFeature(Feature: AMDGPU::FeatureR128A16)
10104 ? True
10105 : False);
10106 if (IsGFX10Plus)
10107 Ops.push_back(Elt: IsA16 ? True : False);
10108
10109 if (!Subtarget->hasGFX90AInsts())
10110 Ops.push_back(Elt: TFE); // tfe
10111 else if (TFE->getAsZExtVal()) {
10112 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10113 DAG.getMachineFunction().getFunction(),
10114 "TFE is not supported on this GPU", DL.getDebugLoc()));
10115 }
10116
10117 if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
10118 Ops.push_back(Elt: LWE); // lwe
10119 if (!IsGFX10Plus)
10120 Ops.push_back(Elt: DimInfo->DA ? True : False);
10121 if (BaseOpcode->HasD16)
10122 Ops.push_back(Elt: IsD16 ? True : False);
10123 if (isa<MemSDNode>(Val: Op))
10124 Ops.push_back(Elt: Op.getOperand(i: 0)); // chain
10125
10126 int NumVAddrDwords =
10127 UseNSA ? VAddrs.size() : VAddr.getValueType().getSizeInBits() / 32;
10128 int Opcode = -1;
10129
10130 if (IsGFX12Plus) {
10131 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx12,
10132 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10133 } else if (IsGFX11Plus) {
10134 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
10135 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx11NSA
10136 : AMDGPU::MIMGEncGfx11Default,
10137 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10138 } else if (IsGFX10Plus) {
10139 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
10140 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx10NSA
10141 : AMDGPU::MIMGEncGfx10Default,
10142 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10143 } else {
10144 if (Subtarget->hasGFX90AInsts()) {
10145 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx90a,
10146 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10147 if (Opcode == -1) {
10148 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10149 DAG.getMachineFunction().getFunction(),
10150 "requested image instruction is not supported on this GPU",
10151 DL.getDebugLoc()));
10152
10153 unsigned Idx = 0;
10154 SmallVector<SDValue, 3> RetValues(OrigResultTypes.size());
10155 for (EVT VT : OrigResultTypes) {
10156 if (VT == MVT::Other)
10157 RetValues[Idx++] = Op.getOperand(i: 0); // Chain
10158 else
10159 RetValues[Idx++] = DAG.getPOISON(VT);
10160 }
10161
10162 return DAG.getMergeValues(Ops: RetValues, dl: DL);
10163 }
10164 }
10165 if (Opcode == -1 &&
10166 Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
10167 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx8,
10168 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10169 if (Opcode == -1)
10170 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx6,
10171 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10172 }
10173 if (Opcode == -1)
10174 return Op;
10175
10176 MachineSDNode *NewNode = DAG.getMachineNode(Opcode, dl: DL, ResultTys: ResultTypes, Ops);
10177 if (auto *MemOp = dyn_cast<MemSDNode>(Val&: Op)) {
10178 MachineMemOperand *MemRef = MemOp->getMemOperand();
10179 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
10180 }
10181
10182 if (BaseOpcode->NoReturn) {
10183 if (BaseOpcode->Atomic)
10184 return DAG.getMergeValues(
10185 Ops: {DAG.getPOISON(VT: OrigResultTypes[0]), SDValue(NewNode, 0)}, dl: DL);
10186
10187 return SDValue(NewNode, 0);
10188 }
10189
10190 if (BaseOpcode->AtomicX2) {
10191 SmallVector<SDValue, 1> Elt;
10192 DAG.ExtractVectorElements(Op: SDValue(NewNode, 0), Args&: Elt, Start: 0, Count: 1);
10193 return DAG.getMergeValues(Ops: {Elt[0], SDValue(NewNode, 1)}, dl: DL);
10194 }
10195
10196 return constructRetValue(DAG, Result: NewNode, ResultTypes: OrigResultTypes, IsTexFail,
10197 Unpacked: Subtarget->hasUnpackedD16VMem(), IsD16, DMaskPop: DMaskLanes,
10198 NumVDataDwords, IsAtomicPacked16Bit, DL);
10199}
10200
10201SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
10202 SDValue Offset, SDValue CachePolicy,
10203 SelectionDAG &DAG) const {
10204 MachineFunction &MF = DAG.getMachineFunction();
10205
10206 const DataLayout &DataLayout = DAG.getDataLayout();
10207 Align Alignment =
10208 DataLayout.getABITypeAlign(Ty: VT.getTypeForEVT(Context&: *DAG.getContext()));
10209
10210 MachineMemOperand *MMO = MF.getMachineMemOperand(
10211 PtrInfo: MachinePointerInfo(),
10212 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
10213 MachineMemOperand::MOInvariant,
10214 Size: VT.getStoreSize(), BaseAlignment: Alignment);
10215
10216 if (!Offset->isDivergent()) {
10217 SDValue Ops[] = {Rsrc, Offset, CachePolicy};
10218
10219 // Lower llvm.amdgcn.s.buffer.load.{i16, u16} intrinsics. Initially, the
10220 // s_buffer_load_u16 instruction is emitted for both signed and unsigned
10221 // loads. Later, DAG combiner tries to combine s_buffer_load_u16 with sext
10222 // and generates s_buffer_load_i16 (performSignExtendInRegCombine).
10223 if (VT == MVT::i16 && Subtarget->hasScalarSubwordLoads()) {
10224 SDValue BufferLoad =
10225 DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::SBUFFER_LOAD_USHORT, dl: DL,
10226 VTList: DAG.getVTList(VT: MVT::i32), Ops, MemVT: VT, MMO);
10227 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: BufferLoad);
10228 }
10229
10230 // Widen vec3 load to vec4.
10231 if (VT.isVector() && VT.getVectorNumElements() == 3 &&
10232 !Subtarget->hasScalarDwordx3Loads()) {
10233 EVT WidenedVT =
10234 EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(), NumElements: 4);
10235 auto WidenedOp = DAG.getMemIntrinsicNode(
10236 Opcode: AMDGPUISD::SBUFFER_LOAD, dl: DL, VTList: DAG.getVTList(VT: WidenedVT), Ops, MemVT: WidenedVT,
10237 MMO: MF.getMachineMemOperand(MMO, Offset: 0, Size: WidenedVT.getStoreSize()));
10238 auto Subvector = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: WidenedOp,
10239 N2: DAG.getVectorIdxConstant(Val: 0, DL));
10240 return Subvector;
10241 }
10242
10243 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::SBUFFER_LOAD, dl: DL,
10244 VTList: DAG.getVTList(VT), Ops, MemVT: VT, MMO);
10245 }
10246
10247 // We have a divergent offset. Emit a MUBUF buffer load instead. We can
10248 // assume that the buffer is unswizzled.
10249 SDValue Ops[] = {
10250 DAG.getEntryNode(), // Chain
10251 Rsrc, // rsrc
10252 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
10253 {}, // voffset
10254 {}, // soffset
10255 {}, // offset
10256 CachePolicy, // cachepolicy
10257 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
10258 };
10259 if (VT == MVT::i16 && Subtarget->hasScalarSubwordLoads()) {
10260 setBufferOffsets(CombinedOffset: Offset, DAG, Offsets: &Ops[3], Alignment: Align(4));
10261 return handleByteShortBufferLoads(DAG, LoadVT: VT, DL, Ops, MMO);
10262 }
10263
10264 SmallVector<SDValue, 4> Loads;
10265 unsigned NumLoads = 1;
10266 MVT LoadVT = VT.getSimpleVT();
10267 unsigned NumElts = LoadVT.isVector() ? LoadVT.getVectorNumElements() : 1;
10268 assert((LoadVT.getScalarType() == MVT::i32 ||
10269 LoadVT.getScalarType() == MVT::f32));
10270
10271 if (NumElts == 8 || NumElts == 16) {
10272 NumLoads = NumElts / 4;
10273 LoadVT = MVT::getVectorVT(VT: LoadVT.getScalarType(), NumElements: 4);
10274 }
10275
10276 SDVTList VTList = DAG.getVTList(VTs: {LoadVT, MVT::Other});
10277
10278 // Use the alignment to ensure that the required offsets will fit into the
10279 // immediate offsets.
10280 setBufferOffsets(CombinedOffset: Offset, DAG, Offsets: &Ops[3],
10281 Alignment: NumLoads > 1 ? Align(16 * NumLoads) : Align(4));
10282
10283 uint64_t InstOffset = Ops[5]->getAsZExtVal();
10284 for (unsigned i = 0; i < NumLoads; ++i) {
10285 Ops[5] = DAG.getTargetConstant(Val: InstOffset + 16 * i, DL, VT: MVT::i32);
10286 Loads.push_back(Elt: getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_LOAD, DL, VTList, Ops,
10287 MemVT: LoadVT, MMO, DAG));
10288 }
10289
10290 if (NumElts == 8 || NumElts == 16)
10291 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: Loads);
10292
10293 return Loads[0];
10294}
10295
10296SDValue SITargetLowering::lowerWaveID(SelectionDAG &DAG, SDValue Op) const {
10297 // With architected SGPRs, waveIDinGroup is in TTMP8[29:25].
10298 if (!Subtarget->hasArchitectedSGPRs())
10299 return {};
10300 SDLoc SL(Op);
10301 MVT VT = MVT::i32;
10302 SDValue TTMP8 = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: AMDGPU::TTMP8, VT);
10303 return DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT, N1: TTMP8,
10304 N2: DAG.getConstant(Val: 25, DL: SL, VT), N3: DAG.getConstant(Val: 5, DL: SL, VT));
10305}
10306
10307SDValue SITargetLowering::lowerConstHwRegRead(SelectionDAG &DAG, SDValue Op,
10308 AMDGPU::Hwreg::Id HwReg,
10309 unsigned LowBit,
10310 unsigned Width) const {
10311 SDLoc SL(Op);
10312 using namespace AMDGPU::Hwreg;
10313 return {DAG.getMachineNode(
10314 Opcode: AMDGPU::S_GETREG_B32_const, dl: SL, VT: MVT::i32,
10315 Op1: DAG.getTargetConstant(Val: HwregEncoding::encode(Values: HwReg, Values: LowBit, Values: Width),
10316 DL: SL, VT: MVT::i32)),
10317 0};
10318}
10319
10320SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
10321 unsigned Dim,
10322 const ArgDescriptor &Arg) const {
10323 SDLoc SL(Op);
10324 MachineFunction &MF = DAG.getMachineFunction();
10325 unsigned MaxID = Subtarget->getMaxWorkitemID(Kernel: MF.getFunction(), Dimension: Dim);
10326 if (MaxID == 0)
10327 return DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
10328
10329 // It's undefined behavior if a function marked with the amdgpu-no-*
10330 // attributes uses the corresponding intrinsic.
10331 if (!Arg)
10332 return DAG.getPOISON(VT: Op->getValueType(ResNo: 0));
10333
10334 SDValue Val = loadInputValue(DAG, RC: &AMDGPU::VGPR_32RegClass, VT: MVT::i32,
10335 SL: SDLoc(DAG.getEntryNode()), Arg);
10336
10337 // Don't bother inserting AssertZext for packed IDs since we're emitting the
10338 // masking operations anyway.
10339 //
10340 // TODO: We could assert the top bit is 0 for the source copy.
10341 if (Arg.isMasked())
10342 return Val;
10343
10344 // Preserve the known bits after expansion to a copy.
10345 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: llvm::bit_width(Value: MaxID));
10346 return DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: MVT::i32, N1: Val,
10347 N2: DAG.getValueType(SmallVT));
10348}
10349
10350SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
10351 SelectionDAG &DAG) const {
10352 MachineFunction &MF = DAG.getMachineFunction();
10353 auto *MFI = MF.getInfo<SIMachineFunctionInfo>();
10354
10355 EVT VT = Op.getValueType();
10356 SDLoc DL(Op);
10357 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
10358
10359 // TODO: Should this propagate fast-math-flags?
10360
10361 switch (IntrinsicID) {
10362 case Intrinsic::amdgcn_implicit_buffer_ptr: {
10363 if (getSubtarget()->isAmdHsaOrMesa(F: MF.getFunction()))
10364 return emitNonHSAIntrinsicError(DAG, DL, VT);
10365 return getPreloadedValue(DAG, MFI: *MFI, VT,
10366 PVID: AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR);
10367 }
10368 case Intrinsic::amdgcn_dispatch_ptr:
10369 case Intrinsic::amdgcn_queue_ptr: {
10370 if (!Subtarget->isAmdHsaOrMesa(F: MF.getFunction())) {
10371 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10372 MF.getFunction(), "unsupported hsa intrinsic without hsa target",
10373 DL.getDebugLoc()));
10374 return DAG.getPOISON(VT);
10375 }
10376
10377 auto RegID = IntrinsicID == Intrinsic::amdgcn_dispatch_ptr
10378 ? AMDGPUFunctionArgInfo::DISPATCH_PTR
10379 : AMDGPUFunctionArgInfo::QUEUE_PTR;
10380 return getPreloadedValue(DAG, MFI: *MFI, VT, PVID: RegID);
10381 }
10382 case Intrinsic::amdgcn_implicitarg_ptr: {
10383 if (MFI->isEntryFunction())
10384 return getImplicitArgPtr(DAG, SL: DL);
10385 return getPreloadedValue(DAG, MFI: *MFI, VT,
10386 PVID: AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
10387 }
10388 case Intrinsic::amdgcn_kernarg_segment_ptr: {
10389 if (!AMDGPU::isKernel(F: MF.getFunction())) {
10390 // This only makes sense to call in a kernel, so just lower to null.
10391 return DAG.getConstant(Val: 0, DL, VT);
10392 }
10393
10394 return getPreloadedValue(DAG, MFI: *MFI, VT,
10395 PVID: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
10396 }
10397 case Intrinsic::amdgcn_dispatch_id: {
10398 return getPreloadedValue(DAG, MFI: *MFI, VT, PVID: AMDGPUFunctionArgInfo::DISPATCH_ID);
10399 }
10400 case Intrinsic::amdgcn_rcp:
10401 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT, Operand: Op.getOperand(i: 1));
10402 case Intrinsic::amdgcn_rsq:
10403 return DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: Op.getOperand(i: 1));
10404 case Intrinsic::amdgcn_rsq_legacy:
10405 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
10406 return emitRemovedIntrinsicError(DAG, DL, VT);
10407 return SDValue();
10408 case Intrinsic::amdgcn_rcp_legacy:
10409 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
10410 return emitRemovedIntrinsicError(DAG, DL, VT);
10411 return DAG.getNode(Opcode: AMDGPUISD::RCP_LEGACY, DL, VT, Operand: Op.getOperand(i: 1));
10412 case Intrinsic::amdgcn_rsq_clamp: {
10413 if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
10414 return DAG.getNode(Opcode: AMDGPUISD::RSQ_CLAMP, DL, VT, Operand: Op.getOperand(i: 1));
10415
10416 Type *Type = VT.getTypeForEVT(Context&: *DAG.getContext());
10417 APFloat Max = APFloat::getLargest(Sem: Type->getFltSemantics());
10418 APFloat Min = APFloat::getLargest(Sem: Type->getFltSemantics(), Negative: true);
10419
10420 SDValue Rsq = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: Op.getOperand(i: 1));
10421 SDValue Tmp =
10422 DAG.getNode(Opcode: ISD::FMINNUM, DL, VT, N1: Rsq, N2: DAG.getConstantFP(Val: Max, DL, VT));
10423 return DAG.getNode(Opcode: ISD::FMAXNUM, DL, VT, N1: Tmp,
10424 N2: DAG.getConstantFP(Val: Min, DL, VT));
10425 }
10426 case Intrinsic::r600_read_ngroups_x:
10427 if (Subtarget->isAmdHsaOS())
10428 return emitNonHSAIntrinsicError(DAG, DL, VT);
10429
10430 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
10431 Offset: SI::KernelInputOffsets::NGROUPS_X, Alignment: Align(4),
10432 Signed: false);
10433 case Intrinsic::r600_read_ngroups_y:
10434 if (Subtarget->isAmdHsaOS())
10435 return emitNonHSAIntrinsicError(DAG, DL, VT);
10436
10437 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
10438 Offset: SI::KernelInputOffsets::NGROUPS_Y, Alignment: Align(4),
10439 Signed: false);
10440 case Intrinsic::r600_read_ngroups_z:
10441 if (Subtarget->isAmdHsaOS())
10442 return emitNonHSAIntrinsicError(DAG, DL, VT);
10443
10444 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
10445 Offset: SI::KernelInputOffsets::NGROUPS_Z, Alignment: Align(4),
10446 Signed: false);
10447 case Intrinsic::r600_read_local_size_x:
10448 if (Subtarget->isAmdHsaOS())
10449 return emitNonHSAIntrinsicError(DAG, DL, VT);
10450
10451 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
10452 Offset: SI::KernelInputOffsets::LOCAL_SIZE_X);
10453 case Intrinsic::r600_read_local_size_y:
10454 if (Subtarget->isAmdHsaOS())
10455 return emitNonHSAIntrinsicError(DAG, DL, VT);
10456
10457 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
10458 Offset: SI::KernelInputOffsets::LOCAL_SIZE_Y);
10459 case Intrinsic::r600_read_local_size_z:
10460 if (Subtarget->isAmdHsaOS())
10461 return emitNonHSAIntrinsicError(DAG, DL, VT);
10462
10463 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
10464 Offset: SI::KernelInputOffsets::LOCAL_SIZE_Z);
10465 case Intrinsic::amdgcn_workgroup_id_x:
10466 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
10467 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_X,
10468 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X,
10469 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X);
10470 case Intrinsic::amdgcn_workgroup_id_y:
10471 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
10472 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,
10473 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y,
10474 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y);
10475 case Intrinsic::amdgcn_workgroup_id_z:
10476 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
10477 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,
10478 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z,
10479 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z);
10480 case Intrinsic::amdgcn_cluster_id_x:
10481 return Subtarget->hasClusters()
10482 ? getPreloadedValue(DAG, MFI: *MFI, VT,
10483 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_X)
10484 : DAG.getPOISON(VT);
10485 case Intrinsic::amdgcn_cluster_id_y:
10486 return Subtarget->hasClusters()
10487 ? getPreloadedValue(DAG, MFI: *MFI, VT,
10488 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_Y)
10489 : DAG.getPOISON(VT);
10490 case Intrinsic::amdgcn_cluster_id_z:
10491 return Subtarget->hasClusters()
10492 ? getPreloadedValue(DAG, MFI: *MFI, VT,
10493 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_Z)
10494 : DAG.getPOISON(VT);
10495 case Intrinsic::amdgcn_cluster_workgroup_id_x:
10496 return Subtarget->hasClusters()
10497 ? getPreloadedValue(
10498 DAG, MFI: *MFI, VT,
10499 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X)
10500 : DAG.getPOISON(VT);
10501 case Intrinsic::amdgcn_cluster_workgroup_id_y:
10502 return Subtarget->hasClusters()
10503 ? getPreloadedValue(
10504 DAG, MFI: *MFI, VT,
10505 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y)
10506 : DAG.getPOISON(VT);
10507 case Intrinsic::amdgcn_cluster_workgroup_id_z:
10508 return Subtarget->hasClusters()
10509 ? getPreloadedValue(
10510 DAG, MFI: *MFI, VT,
10511 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z)
10512 : DAG.getPOISON(VT);
10513 case Intrinsic::amdgcn_cluster_workgroup_flat_id:
10514 return Subtarget->hasClusters()
10515 ? lowerConstHwRegRead(DAG, Op, HwReg: AMDGPU::Hwreg::ID_IB_STS2, LowBit: 21, Width: 4)
10516 : SDValue();
10517 case Intrinsic::amdgcn_cluster_workgroup_max_id_x:
10518 return Subtarget->hasClusters()
10519 ? getPreloadedValue(
10520 DAG, MFI: *MFI, VT,
10521 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X)
10522 : DAG.getPOISON(VT);
10523 case Intrinsic::amdgcn_cluster_workgroup_max_id_y:
10524 return Subtarget->hasClusters()
10525 ? getPreloadedValue(
10526 DAG, MFI: *MFI, VT,
10527 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y)
10528 : DAG.getPOISON(VT);
10529 case Intrinsic::amdgcn_cluster_workgroup_max_id_z:
10530 return Subtarget->hasClusters()
10531 ? getPreloadedValue(
10532 DAG, MFI: *MFI, VT,
10533 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z)
10534 : DAG.getPOISON(VT);
10535 case Intrinsic::amdgcn_cluster_workgroup_max_flat_id:
10536 return Subtarget->hasClusters()
10537 ? getPreloadedValue(
10538 DAG, MFI: *MFI, VT,
10539 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID)
10540 : DAG.getPOISON(VT);
10541 case Intrinsic::amdgcn_wave_id:
10542 return lowerWaveID(DAG, Op);
10543 case Intrinsic::amdgcn_lds_kernel_id: {
10544 if (MFI->isEntryFunction())
10545 return getLDSKernelId(DAG, SL: DL);
10546 return getPreloadedValue(DAG, MFI: *MFI, VT,
10547 PVID: AMDGPUFunctionArgInfo::LDS_KERNEL_ID);
10548 }
10549 case Intrinsic::amdgcn_workitem_id_x:
10550 return lowerWorkitemID(DAG, Op, Dim: 0, Arg: MFI->getArgInfo().WorkItemIDX);
10551 case Intrinsic::amdgcn_workitem_id_y:
10552 return lowerWorkitemID(DAG, Op, Dim: 1, Arg: MFI->getArgInfo().WorkItemIDY);
10553 case Intrinsic::amdgcn_workitem_id_z:
10554 return lowerWorkitemID(DAG, Op, Dim: 2, Arg: MFI->getArgInfo().WorkItemIDZ);
10555 case Intrinsic::amdgcn_wavefrontsize:
10556 return DAG.getConstant(Val: MF.getSubtarget<GCNSubtarget>().getWavefrontSize(),
10557 DL: SDLoc(Op), VT: MVT::i32);
10558 case Intrinsic::amdgcn_s_buffer_load: {
10559 unsigned CPol = Op.getConstantOperandVal(i: 3);
10560 // s_buffer_load, because of how it's optimized, can't be volatile
10561 // so reject ones with the volatile bit set.
10562 if (CPol & ~((Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12)
10563 ? AMDGPU::CPol::ALL
10564 : AMDGPU::CPol::ALL_pregfx12))
10565 return Op;
10566 return lowerSBuffer(VT, DL, Rsrc: Op.getOperand(i: 1), Offset: Op.getOperand(i: 2),
10567 CachePolicy: Op.getOperand(i: 3), DAG);
10568 }
10569 case Intrinsic::amdgcn_fdiv_fast:
10570 return lowerFDIV_FAST(Op, DAG);
10571 case Intrinsic::amdgcn_sin:
10572 return DAG.getNode(Opcode: AMDGPUISD::SIN_HW, DL, VT, Operand: Op.getOperand(i: 1));
10573
10574 case Intrinsic::amdgcn_cos:
10575 return DAG.getNode(Opcode: AMDGPUISD::COS_HW, DL, VT, Operand: Op.getOperand(i: 1));
10576
10577 case Intrinsic::amdgcn_mul_u24:
10578 return DAG.getNode(Opcode: AMDGPUISD::MUL_U24, DL, VT, N1: Op.getOperand(i: 1),
10579 N2: Op.getOperand(i: 2));
10580 case Intrinsic::amdgcn_mul_i24:
10581 return DAG.getNode(Opcode: AMDGPUISD::MUL_I24, DL, VT, N1: Op.getOperand(i: 1),
10582 N2: Op.getOperand(i: 2));
10583
10584 case Intrinsic::amdgcn_log_clamp: {
10585 if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
10586 return SDValue();
10587
10588 return emitRemovedIntrinsicError(DAG, DL, VT);
10589 }
10590 case Intrinsic::amdgcn_fract:
10591 return DAG.getNode(Opcode: AMDGPUISD::FRACT, DL, VT, Operand: Op.getOperand(i: 1));
10592
10593 case Intrinsic::amdgcn_class:
10594 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT, N1: Op.getOperand(i: 1),
10595 N2: Op.getOperand(i: 2));
10596 case Intrinsic::amdgcn_div_fmas:
10597 return DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL, VT, N1: Op.getOperand(i: 1),
10598 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), N4: Op.getOperand(i: 4));
10599
10600 case Intrinsic::amdgcn_div_fixup:
10601 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL, VT, N1: Op.getOperand(i: 1),
10602 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
10603
10604 case Intrinsic::amdgcn_div_scale: {
10605 const ConstantSDNode *Param = cast<ConstantSDNode>(Val: Op.getOperand(i: 3));
10606
10607 // Translate to the operands expected by the machine instruction. The
10608 // first parameter must be the same as the first instruction.
10609 SDValue Numerator = Op.getOperand(i: 1);
10610 SDValue Denominator = Op.getOperand(i: 2);
10611
10612 // Note this order is opposite of the machine instruction's operations,
10613 // which is s0.f = Quotient, s1.f = Denominator, s2.f = Numerator. The
10614 // intrinsic has the numerator as the first operand to match a normal
10615 // division operation.
10616
10617 SDValue Src0 = Param->isAllOnes() ? Numerator : Denominator;
10618
10619 return DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL, VTList: Op->getVTList(), N1: Src0,
10620 N2: Denominator, N3: Numerator);
10621 }
10622 case Intrinsic::amdgcn_icmp: {
10623 // There is a Pat that handles this variant, so return it as-is.
10624 if (Op.getOperand(i: 1).getValueType() == MVT::i1 &&
10625 Op.getConstantOperandVal(i: 2) == 0 &&
10626 Op.getConstantOperandVal(i: 3) == ICmpInst::Predicate::ICMP_NE)
10627 return Op;
10628 return lowerICMPIntrinsic(TLI: *this, N: Op.getNode(), DAG);
10629 }
10630 case Intrinsic::amdgcn_fcmp: {
10631 return lowerFCMPIntrinsic(TLI: *this, N: Op.getNode(), DAG);
10632 }
10633 case Intrinsic::amdgcn_ballot:
10634 return lowerBALLOTIntrinsic(TLI: *this, N: Op.getNode(), DAG);
10635 case Intrinsic::amdgcn_fmed3:
10636 return DAG.getNode(Opcode: AMDGPUISD::FMED3, DL, VT, N1: Op.getOperand(i: 1),
10637 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
10638 case Intrinsic::amdgcn_fdot2:
10639 return DAG.getNode(Opcode: AMDGPUISD::FDOT2, DL, VT, N1: Op.getOperand(i: 1),
10640 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), N4: Op.getOperand(i: 4));
10641 case Intrinsic::amdgcn_fmul_legacy:
10642 return DAG.getNode(Opcode: AMDGPUISD::FMUL_LEGACY, DL, VT, N1: Op.getOperand(i: 1),
10643 N2: Op.getOperand(i: 2));
10644 case Intrinsic::amdgcn_sbfe:
10645 return DAG.getNode(Opcode: AMDGPUISD::BFE_I32, DL, VT, N1: Op.getOperand(i: 1),
10646 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
10647 case Intrinsic::amdgcn_ubfe:
10648 return DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL, VT, N1: Op.getOperand(i: 1),
10649 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
10650 case Intrinsic::amdgcn_cvt_pkrtz:
10651 case Intrinsic::amdgcn_cvt_pknorm_i16:
10652 case Intrinsic::amdgcn_cvt_pknorm_u16:
10653 case Intrinsic::amdgcn_cvt_pk_i16:
10654 case Intrinsic::amdgcn_cvt_pk_u16: {
10655 // FIXME: Stop adding cast if v2f16/v2i16 are legal.
10656 EVT VT = Op.getValueType();
10657 unsigned Opcode;
10658
10659 if (IntrinsicID == Intrinsic::amdgcn_cvt_pkrtz)
10660 Opcode = AMDGPUISD::CVT_PKRTZ_F16_F32;
10661 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_i16)
10662 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
10663 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_u16)
10664 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
10665 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pk_i16)
10666 Opcode = AMDGPUISD::CVT_PK_I16_I32;
10667 else
10668 Opcode = AMDGPUISD::CVT_PK_U16_U32;
10669
10670 if (isTypeLegal(VT))
10671 return DAG.getNode(Opcode, DL, VT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
10672
10673 SDValue Node =
10674 DAG.getNode(Opcode, DL, VT: MVT::i32, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
10675 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Node);
10676 }
10677 case Intrinsic::amdgcn_fmad_ftz:
10678 return DAG.getNode(Opcode: AMDGPUISD::FMAD_FTZ, DL, VT, N1: Op.getOperand(i: 1),
10679 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
10680
10681 case Intrinsic::amdgcn_if_break:
10682 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::SI_IF_BREAK, dl: DL, VT,
10683 Op1: Op->getOperand(Num: 1), Op2: Op->getOperand(Num: 2)),
10684 0);
10685
10686 case Intrinsic::amdgcn_groupstaticsize: {
10687 Triple::OSType OS = getTargetMachine().getTargetTriple().getOS();
10688 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL)
10689 return Op;
10690
10691 const Module *M = MF.getFunction().getParent();
10692 const GlobalValue *GV =
10693 Intrinsic::getDeclarationIfExists(M, id: Intrinsic::amdgcn_groupstaticsize);
10694 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: 0,
10695 TargetFlags: SIInstrInfo::MO_ABS32_LO);
10696 return {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: GA), 0};
10697 }
10698 case Intrinsic::amdgcn_is_shared:
10699 case Intrinsic::amdgcn_is_private: {
10700 SDLoc SL(Op);
10701 SDValue SrcVec =
10702 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
10703 SDValue SrcHi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: SrcVec,
10704 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
10705
10706 unsigned AS = (IntrinsicID == Intrinsic::amdgcn_is_shared)
10707 ? AMDGPUAS::LOCAL_ADDRESS
10708 : AMDGPUAS::PRIVATE_ADDRESS;
10709 if (AS == AMDGPUAS::PRIVATE_ADDRESS &&
10710 Subtarget->hasGloballyAddressableScratch()) {
10711 SDValue FlatScratchBaseHi(
10712 DAG.getMachineNode(
10713 Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32,
10714 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, VT: MVT::i32)),
10715 0);
10716 // Test bits 63..58 against the aperture address.
10717 return DAG.getSetCC(
10718 DL: SL, VT: MVT::i1,
10719 LHS: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: SrcHi, N2: FlatScratchBaseHi),
10720 RHS: DAG.getConstant(Val: 1u << 26, DL: SL, VT: MVT::i32), Cond: ISD::SETULT);
10721 }
10722
10723 SDValue Aperture = getSegmentAperture(AS, DL: SL, DAG);
10724 return DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: SrcHi, RHS: Aperture, Cond: ISD::SETEQ);
10725 }
10726 case Intrinsic::amdgcn_perm:
10727 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: Op.getOperand(i: 1),
10728 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
10729 case Intrinsic::amdgcn_reloc_constant: {
10730 Module *M = MF.getFunction().getParent();
10731 const MDNode *Metadata = cast<MDNodeSDNode>(Val: Op.getOperand(i: 1))->getMD();
10732 auto SymbolName = cast<MDString>(Val: Metadata->getOperand(I: 0))->getString();
10733 auto *RelocSymbol = cast<GlobalVariable>(
10734 Val: M->getOrInsertGlobal(Name: SymbolName, Ty: Type::getInt32Ty(C&: M->getContext())));
10735 SDValue GA = DAG.getTargetGlobalAddress(GV: RelocSymbol, DL, VT: MVT::i32, offset: 0,
10736 TargetFlags: SIInstrInfo::MO_ABS32_LO);
10737 return {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: GA), 0};
10738 }
10739 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
10740 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
10741 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
10742 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
10743 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
10744 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
10745 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
10746 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8: {
10747 if (Op.getOperand(i: 4).getValueType() == MVT::i32)
10748 return SDValue();
10749
10750 SDLoc SL(Op);
10751 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 4), DL: SL, VT: MVT::i32);
10752 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
10753 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1), N3: Op.getOperand(i: 2),
10754 N4: Op.getOperand(i: 3), N5: IndexKeyi32);
10755 }
10756 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
10757 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
10758 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
10759 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
10760 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
10761 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
10762 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
10763 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8: {
10764 if (Op.getOperand(i: 4).getValueType() == MVT::i64)
10765 return SDValue();
10766
10767 SDLoc SL(Op);
10768 auto IndexKeyi64 =
10769 Op.getOperand(i: 4).getValueType() == MVT::v2i32
10770 ? DAG.getBitcast(VT: MVT::i64, V: Op.getOperand(i: 4))
10771 : DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 4), DL: SL, VT: MVT::i64);
10772 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
10773 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
10774 Op.getOperand(i: 3), IndexKeyi64, Op.getOperand(i: 5),
10775 Op.getOperand(i: 6)});
10776 }
10777 case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
10778 case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
10779 case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
10780 case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
10781 case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
10782 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8: {
10783 EVT IndexKeyTy = IntrinsicID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8
10784 ? MVT::i64
10785 : MVT::i32;
10786 if (Op.getOperand(i: 6).getValueType() == IndexKeyTy)
10787 return SDValue();
10788
10789 SDLoc SL(Op);
10790 auto IndexKey =
10791 Op.getOperand(i: 6).getValueType().isVector()
10792 ? DAG.getBitcast(VT: IndexKeyTy, V: Op.getOperand(i: 6))
10793 : DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 6), DL: SL, VT: IndexKeyTy);
10794 SmallVector<SDValue> Args{
10795 Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
10796 Op.getOperand(i: 3), Op.getOperand(i: 4), Op.getOperand(i: 5),
10797 IndexKey, Op.getOperand(i: 7), Op.getOperand(i: 8)};
10798 if (IntrinsicID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8)
10799 Args.push_back(Elt: Op.getOperand(i: 9));
10800 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(), Ops: Args);
10801 }
10802 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
10803 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
10804 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4: {
10805 if (Op.getOperand(i: 6).getValueType() == MVT::i32)
10806 return SDValue();
10807
10808 SDLoc SL(Op);
10809 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 6), DL: SL, VT: MVT::i32);
10810 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
10811 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
10812 Op.getOperand(i: 3), Op.getOperand(i: 4), Op.getOperand(i: 5),
10813 IndexKeyi32, Op.getOperand(i: 7)});
10814 }
10815 case Intrinsic::amdgcn_addrspacecast_nonnull:
10816 return lowerADDRSPACECAST(Op, DAG);
10817 case Intrinsic::amdgcn_readlane:
10818 case Intrinsic::amdgcn_readfirstlane:
10819 case Intrinsic::amdgcn_writelane:
10820 case Intrinsic::amdgcn_permlane16:
10821 case Intrinsic::amdgcn_permlanex16:
10822 case Intrinsic::amdgcn_permlane64:
10823 case Intrinsic::amdgcn_set_inactive:
10824 case Intrinsic::amdgcn_set_inactive_chain_arg:
10825 case Intrinsic::amdgcn_mov_dpp8:
10826 case Intrinsic::amdgcn_update_dpp:
10827 return lowerLaneOp(TLI: *this, N: Op.getNode(), DAG);
10828 case Intrinsic::amdgcn_dead: {
10829 SmallVector<SDValue, 8> Poisons;
10830 for (const EVT ValTy : Op.getNode()->values())
10831 Poisons.push_back(Elt: DAG.getPOISON(VT: ValTy));
10832 return DAG.getMergeValues(Ops: Poisons, dl: SDLoc(Op));
10833 }
10834 case Intrinsic::amdgcn_wave_shuffle:
10835 return lowerWaveShuffle(TLI: *this, N: Op.getNode(), DAG);
10836 default:
10837 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
10838 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrinsicID))
10839 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: false);
10840
10841 return Op;
10842 }
10843}
10844
10845// On targets not supporting constant in soffset field, turn zero to
10846// SGPR_NULL to avoid generating an extra s_mov with zero.
10847static SDValue selectSOffset(SDValue SOffset, SelectionDAG &DAG,
10848 const GCNSubtarget *Subtarget) {
10849 if (Subtarget->hasRestrictedSOffset() && isNullConstant(V: SOffset))
10850 return DAG.getRegister(Reg: AMDGPU::SGPR_NULL, VT: MVT::i32);
10851 return SOffset;
10852}
10853
10854SDValue SITargetLowering::lowerRawBufferAtomicIntrin(SDValue Op,
10855 SelectionDAG &DAG,
10856 unsigned NewOpcode) const {
10857 SDLoc DL(Op);
10858
10859 SDValue VData = Op.getOperand(i: 2);
10860 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
10861 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
10862 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
10863 SDValue Ops[] = {
10864 Op.getOperand(i: 0), // Chain
10865 VData, // vdata
10866 Rsrc, // rsrc
10867 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
10868 VOffset, // voffset
10869 SOffset, // soffset
10870 Offset, // offset
10871 Op.getOperand(i: 6), // cachepolicy
10872 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
10873 };
10874
10875 auto *M = cast<MemSDNode>(Val&: Op);
10876
10877 EVT MemVT = VData.getValueType();
10878 return DAG.getMemIntrinsicNode(Opcode: NewOpcode, dl: DL, VTList: Op->getVTList(), Ops, MemVT,
10879 MMO: M->getMemOperand());
10880}
10881
10882SDValue
10883SITargetLowering::lowerStructBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG,
10884 unsigned NewOpcode) const {
10885 SDLoc DL(Op);
10886
10887 SDValue VData = Op.getOperand(i: 2);
10888 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
10889 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
10890 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
10891 SDValue Ops[] = {
10892 Op.getOperand(i: 0), // Chain
10893 VData, // vdata
10894 Rsrc, // rsrc
10895 Op.getOperand(i: 4), // vindex
10896 VOffset, // voffset
10897 SOffset, // soffset
10898 Offset, // offset
10899 Op.getOperand(i: 7), // cachepolicy
10900 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
10901 };
10902
10903 auto *M = cast<MemSDNode>(Val&: Op);
10904
10905 EVT MemVT = VData.getValueType();
10906 return DAG.getMemIntrinsicNode(Opcode: NewOpcode, dl: DL, VTList: Op->getVTList(), Ops, MemVT,
10907 MMO: M->getMemOperand());
10908}
10909
10910SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
10911 SelectionDAG &DAG) const {
10912 unsigned IntrID = Op.getConstantOperandVal(i: 1);
10913 SDLoc DL(Op);
10914
10915 switch (IntrID) {
10916 case Intrinsic::amdgcn_ds_ordered_add:
10917 case Intrinsic::amdgcn_ds_ordered_swap: {
10918 MemSDNode *M = cast<MemSDNode>(Val&: Op);
10919 SDValue Chain = M->getOperand(Num: 0);
10920 SDValue M0 = M->getOperand(Num: 2);
10921 SDValue Value = M->getOperand(Num: 3);
10922 unsigned IndexOperand = M->getConstantOperandVal(Num: 7);
10923 unsigned WaveRelease = M->getConstantOperandVal(Num: 8);
10924 unsigned WaveDone = M->getConstantOperandVal(Num: 9);
10925
10926 unsigned OrderedCountIndex = IndexOperand & 0x3f;
10927 IndexOperand &= ~0x3f;
10928 unsigned CountDw = 0;
10929
10930 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10) {
10931 CountDw = (IndexOperand >> 24) & 0xf;
10932 IndexOperand &= ~(0xf << 24);
10933
10934 if (CountDw < 1 || CountDw > 4) {
10935 const Function &Fn = DAG.getMachineFunction().getFunction();
10936 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10937 Fn, "ds_ordered_count: dword count must be between 1 and 4",
10938 DL.getDebugLoc()));
10939 CountDw = 1;
10940 }
10941 }
10942
10943 if (IndexOperand) {
10944 const Function &Fn = DAG.getMachineFunction().getFunction();
10945 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10946 Fn, "ds_ordered_count: bad index operand", DL.getDebugLoc()));
10947 }
10948
10949 if (WaveDone && !WaveRelease) {
10950 // TODO: Move this to IR verifier
10951 const Function &Fn = DAG.getMachineFunction().getFunction();
10952 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10953 Fn, "ds_ordered_count: wave_done requires wave_release",
10954 DL.getDebugLoc()));
10955 }
10956
10957 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
10958 unsigned ShaderType =
10959 SIInstrInfo::getDSShaderTypeValue(MF: DAG.getMachineFunction());
10960 unsigned Offset0 = OrderedCountIndex << 2;
10961 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (Instruction << 4);
10962
10963 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10)
10964 Offset1 |= (CountDw - 1) << 6;
10965
10966 if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX11)
10967 Offset1 |= ShaderType << 2;
10968
10969 unsigned Offset = Offset0 | (Offset1 << 8);
10970
10971 SDValue Ops[] = {
10972 Chain, Value, DAG.getTargetConstant(Val: Offset, DL, VT: MVT::i16),
10973 copyToM0(DAG, Chain, DL, V: M0).getValue(R: 1), // Glue
10974 };
10975 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::DS_ORDERED_COUNT, dl: DL,
10976 VTList: M->getVTList(), Ops, MemVT: M->getMemoryVT(),
10977 MMO: M->getMemOperand());
10978 }
10979 case Intrinsic::amdgcn_raw_buffer_load:
10980 case Intrinsic::amdgcn_raw_ptr_buffer_load:
10981 case Intrinsic::amdgcn_raw_atomic_buffer_load:
10982 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
10983 case Intrinsic::amdgcn_raw_buffer_load_format:
10984 case Intrinsic::amdgcn_raw_ptr_buffer_load_format: {
10985 const bool IsFormat =
10986 IntrID == Intrinsic::amdgcn_raw_buffer_load_format ||
10987 IntrID == Intrinsic::amdgcn_raw_ptr_buffer_load_format;
10988
10989 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
10990 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 3), DAG);
10991 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 4), DAG, Subtarget);
10992 SDValue Ops[] = {
10993 Op.getOperand(i: 0), // Chain
10994 Rsrc, // rsrc
10995 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
10996 VOffset, // voffset
10997 SOffset, // soffset
10998 Offset, // offset
10999 Op.getOperand(i: 5), // cachepolicy, swizzled buffer
11000 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
11001 };
11002
11003 auto *M = cast<MemSDNode>(Val&: Op);
11004 return lowerIntrinsicLoad(M, IsFormat, DAG, Ops);
11005 }
11006 case Intrinsic::amdgcn_struct_buffer_load:
11007 case Intrinsic::amdgcn_struct_ptr_buffer_load:
11008 case Intrinsic::amdgcn_struct_buffer_load_format:
11009 case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
11010 case Intrinsic::amdgcn_struct_atomic_buffer_load:
11011 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
11012 const bool IsFormat =
11013 IntrID == Intrinsic::amdgcn_struct_buffer_load_format ||
11014 IntrID == Intrinsic::amdgcn_struct_ptr_buffer_load_format;
11015
11016 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
11017 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
11018 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
11019 SDValue Ops[] = {
11020 Op.getOperand(i: 0), // Chain
11021 Rsrc, // rsrc
11022 Op.getOperand(i: 3), // vindex
11023 VOffset, // voffset
11024 SOffset, // soffset
11025 Offset, // offset
11026 Op.getOperand(i: 6), // cachepolicy, swizzled buffer
11027 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
11028 };
11029
11030 return lowerIntrinsicLoad(M: cast<MemSDNode>(Val&: Op), IsFormat, DAG, Ops);
11031 }
11032 case Intrinsic::amdgcn_raw_tbuffer_load:
11033 case Intrinsic::amdgcn_raw_ptr_tbuffer_load: {
11034 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11035 EVT LoadVT = Op.getValueType();
11036 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
11037 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 3), DAG);
11038 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 4), DAG, Subtarget);
11039
11040 SDValue Ops[] = {
11041 Op.getOperand(i: 0), // Chain
11042 Rsrc, // rsrc
11043 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
11044 VOffset, // voffset
11045 SOffset, // soffset
11046 Offset, // offset
11047 Op.getOperand(i: 5), // format
11048 Op.getOperand(i: 6), // cachepolicy, swizzled buffer
11049 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
11050 };
11051
11052 if (LoadVT.getScalarType() == MVT::f16)
11053 return adjustLoadValueType(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
11054 Ops);
11055 return getMemIntrinsicNode(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
11056 VTList: Op->getVTList(), Ops, MemVT: LoadVT, MMO: M->getMemOperand(),
11057 DAG);
11058 }
11059 case Intrinsic::amdgcn_struct_tbuffer_load:
11060 case Intrinsic::amdgcn_struct_ptr_tbuffer_load: {
11061 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11062 EVT LoadVT = Op.getValueType();
11063 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
11064 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
11065 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
11066
11067 SDValue Ops[] = {
11068 Op.getOperand(i: 0), // Chain
11069 Rsrc, // rsrc
11070 Op.getOperand(i: 3), // vindex
11071 VOffset, // voffset
11072 SOffset, // soffset
11073 Offset, // offset
11074 Op.getOperand(i: 6), // format
11075 Op.getOperand(i: 7), // cachepolicy, swizzled buffer
11076 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
11077 };
11078
11079 if (LoadVT.getScalarType() == MVT::f16)
11080 return adjustLoadValueType(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
11081 Ops);
11082 return getMemIntrinsicNode(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
11083 VTList: Op->getVTList(), Ops, MemVT: LoadVT, MMO: M->getMemOperand(),
11084 DAG);
11085 }
11086 case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
11087 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
11088 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FADD);
11089 case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
11090 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
11091 return lowerStructBufferAtomicIntrin(Op, DAG,
11092 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FADD);
11093 case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
11094 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
11095 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMIN);
11096 case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
11097 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
11098 return lowerStructBufferAtomicIntrin(Op, DAG,
11099 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMIN);
11100 case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
11101 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
11102 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMAX);
11103 case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
11104 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
11105 return lowerStructBufferAtomicIntrin(Op, DAG,
11106 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMAX);
11107 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
11108 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
11109 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SWAP);
11110 case Intrinsic::amdgcn_raw_buffer_atomic_add:
11111 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
11112 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_ADD);
11113 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
11114 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
11115 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SUB);
11116 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
11117 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
11118 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMIN);
11119 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
11120 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
11121 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMIN);
11122 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
11123 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
11124 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMAX);
11125 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
11126 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
11127 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMAX);
11128 case Intrinsic::amdgcn_raw_buffer_atomic_and:
11129 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
11130 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_AND);
11131 case Intrinsic::amdgcn_raw_buffer_atomic_or:
11132 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
11133 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_OR);
11134 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
11135 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
11136 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_XOR);
11137 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
11138 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
11139 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_INC);
11140 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
11141 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
11142 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_DEC);
11143 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
11144 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
11145 return lowerStructBufferAtomicIntrin(Op, DAG,
11146 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SWAP);
11147 case Intrinsic::amdgcn_struct_buffer_atomic_add:
11148 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
11149 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_ADD);
11150 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
11151 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
11152 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SUB);
11153 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
11154 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
11155 return lowerStructBufferAtomicIntrin(Op, DAG,
11156 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMIN);
11157 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
11158 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
11159 return lowerStructBufferAtomicIntrin(Op, DAG,
11160 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMIN);
11161 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
11162 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
11163 return lowerStructBufferAtomicIntrin(Op, DAG,
11164 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMAX);
11165 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
11166 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
11167 return lowerStructBufferAtomicIntrin(Op, DAG,
11168 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMAX);
11169 case Intrinsic::amdgcn_struct_buffer_atomic_and:
11170 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
11171 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_AND);
11172 case Intrinsic::amdgcn_struct_buffer_atomic_or:
11173 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
11174 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_OR);
11175 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
11176 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
11177 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_XOR);
11178 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
11179 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
11180 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_INC);
11181 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
11182 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
11183 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_DEC);
11184 case Intrinsic::amdgcn_raw_buffer_atomic_sub_clamp_u32:
11185 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
11186 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_CSUB);
11187 case Intrinsic::amdgcn_struct_buffer_atomic_sub_clamp_u32:
11188 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
11189 return lowerStructBufferAtomicIntrin(Op, DAG,
11190 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_CSUB);
11191 case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
11192 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
11193 return lowerRawBufferAtomicIntrin(Op, DAG,
11194 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_COND_SUB_U32);
11195 case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
11196 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
11197 return lowerStructBufferAtomicIntrin(Op, DAG,
11198 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_COND_SUB_U32);
11199 case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
11200 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap: {
11201 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 4), DAG);
11202 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
11203 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
11204 SDValue Ops[] = {
11205 Op.getOperand(i: 0), // Chain
11206 Op.getOperand(i: 2), // src
11207 Op.getOperand(i: 3), // cmp
11208 Rsrc, // rsrc
11209 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
11210 VOffset, // voffset
11211 SOffset, // soffset
11212 Offset, // offset
11213 Op.getOperand(i: 7), // cachepolicy
11214 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
11215 };
11216 EVT VT = Op.getValueType();
11217 auto *M = cast<MemSDNode>(Val&: Op);
11218
11219 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, dl: DL,
11220 VTList: Op->getVTList(), Ops, MemVT: VT,
11221 MMO: M->getMemOperand());
11222 }
11223 case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
11224 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap: {
11225 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op->getOperand(Num: 4), DAG);
11226 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 6), DAG);
11227 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 7), DAG, Subtarget);
11228 SDValue Ops[] = {
11229 Op.getOperand(i: 0), // Chain
11230 Op.getOperand(i: 2), // src
11231 Op.getOperand(i: 3), // cmp
11232 Rsrc, // rsrc
11233 Op.getOperand(i: 5), // vindex
11234 VOffset, // voffset
11235 SOffset, // soffset
11236 Offset, // offset
11237 Op.getOperand(i: 8), // cachepolicy
11238 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
11239 };
11240 EVT VT = Op.getValueType();
11241 auto *M = cast<MemSDNode>(Val&: Op);
11242
11243 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, dl: DL,
11244 VTList: Op->getVTList(), Ops, MemVT: VT,
11245 MMO: M->getMemOperand());
11246 }
11247 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
11248 case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
11249 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11250 SDValue NodePtr = M->getOperand(Num: 2);
11251 SDValue RayExtent = M->getOperand(Num: 3);
11252 SDValue InstanceMask = M->getOperand(Num: 4);
11253 SDValue RayOrigin = M->getOperand(Num: 5);
11254 SDValue RayDir = M->getOperand(Num: 6);
11255 SDValue Offsets = M->getOperand(Num: 7);
11256 SDValue TDescr = M->getOperand(Num: 8);
11257
11258 assert(NodePtr.getValueType() == MVT::i64);
11259 assert(RayDir.getValueType() == MVT::v3f32);
11260
11261 if (!Subtarget->hasBVHDualAndBVH8Insts()) {
11262 emitRemovedIntrinsicError(DAG, DL, VT: Op.getValueType());
11263 return SDValue();
11264 }
11265
11266 bool IsBVH8 = IntrID == Intrinsic::amdgcn_image_bvh8_intersect_ray;
11267 const unsigned NumVDataDwords = 10;
11268 const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12;
11269 int Opcode = AMDGPU::getMIMGOpcode(
11270 BaseOpcode: IsBVH8 ? AMDGPU::IMAGE_BVH8_INTERSECT_RAY
11271 : AMDGPU::IMAGE_BVH_DUAL_INTERSECT_RAY,
11272 MIMGEncoding: AMDGPU::MIMGEncGfx12, VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
11273 assert(Opcode != -1);
11274
11275 SmallVector<SDValue, 7> Ops;
11276 Ops.push_back(Elt: NodePtr);
11277 Ops.push_back(Elt: DAG.getBuildVector(
11278 VT: MVT::v2i32, DL,
11279 Ops: {DAG.getBitcast(VT: MVT::i32, V: RayExtent),
11280 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: InstanceMask)}));
11281 Ops.push_back(Elt: RayOrigin);
11282 Ops.push_back(Elt: RayDir);
11283 Ops.push_back(Elt: Offsets);
11284 Ops.push_back(Elt: TDescr);
11285 Ops.push_back(Elt: M->getChain());
11286
11287 auto *NewNode = DAG.getMachineNode(Opcode, dl: DL, VTs: M->getVTList(), Ops);
11288 MachineMemOperand *MemRef = M->getMemOperand();
11289 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
11290 return SDValue(NewNode, 0);
11291 }
11292 case Intrinsic::amdgcn_image_bvh_intersect_ray: {
11293 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11294 SDValue NodePtr = M->getOperand(Num: 2);
11295 SDValue RayExtent = M->getOperand(Num: 3);
11296 SDValue RayOrigin = M->getOperand(Num: 4);
11297 SDValue RayDir = M->getOperand(Num: 5);
11298 SDValue RayInvDir = M->getOperand(Num: 6);
11299 SDValue TDescr = M->getOperand(Num: 7);
11300
11301 assert(NodePtr.getValueType() == MVT::i32 ||
11302 NodePtr.getValueType() == MVT::i64);
11303 assert(RayDir.getValueType() == MVT::v3f16 ||
11304 RayDir.getValueType() == MVT::v3f32);
11305
11306 if (!Subtarget->hasGFX10_AEncoding()) {
11307 emitRemovedIntrinsicError(DAG, DL, VT: Op.getValueType());
11308 return SDValue();
11309 }
11310
11311 const bool IsGFX11 = AMDGPU::isGFX11(STI: *Subtarget);
11312 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI: *Subtarget);
11313 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
11314 const bool IsA16 = RayDir.getValueType().getVectorElementType() == MVT::f16;
11315 const bool Is64 = NodePtr.getValueType() == MVT::i64;
11316 const unsigned NumVDataDwords = 4;
11317 const unsigned NumVAddrDwords = IsA16 ? (Is64 ? 9 : 8) : (Is64 ? 12 : 11);
11318 const unsigned NumVAddrs = IsGFX11Plus ? (IsA16 ? 4 : 5) : NumVAddrDwords;
11319 const bool UseNSA = (Subtarget->hasNSAEncoding() &&
11320 NumVAddrs <= Subtarget->getNSAMaxSize()) ||
11321 IsGFX12Plus;
11322 const unsigned BaseOpcodes[2][2] = {
11323 {AMDGPU::IMAGE_BVH_INTERSECT_RAY, AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16},
11324 {AMDGPU::IMAGE_BVH64_INTERSECT_RAY,
11325 AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16}};
11326 int Opcode;
11327 if (UseNSA) {
11328 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: BaseOpcodes[Is64][IsA16],
11329 MIMGEncoding: IsGFX12Plus ? AMDGPU::MIMGEncGfx12
11330 : IsGFX11 ? AMDGPU::MIMGEncGfx11NSA
11331 : AMDGPU::MIMGEncGfx10NSA,
11332 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
11333 } else {
11334 assert(!IsGFX12Plus);
11335 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: BaseOpcodes[Is64][IsA16],
11336 MIMGEncoding: IsGFX11 ? AMDGPU::MIMGEncGfx11Default
11337 : AMDGPU::MIMGEncGfx10Default,
11338 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
11339 }
11340 assert(Opcode != -1);
11341
11342 SmallVector<SDValue, 16> Ops;
11343
11344 auto packLanes = [&DAG, &Ops, &DL](SDValue Op, bool IsAligned) {
11345 SmallVector<SDValue, 3> Lanes;
11346 DAG.ExtractVectorElements(Op, Args&: Lanes, Start: 0, Count: 3);
11347 if (Lanes[0].getValueSizeInBits() == 32) {
11348 for (unsigned I = 0; I < 3; ++I)
11349 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: Lanes[I]));
11350 } else {
11351 if (IsAligned) {
11352 Ops.push_back(Elt: DAG.getBitcast(
11353 VT: MVT::i32,
11354 V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Lanes[0], Lanes[1]})));
11355 Ops.push_back(Elt: Lanes[2]);
11356 } else {
11357 SDValue Elt0 = Ops.pop_back_val();
11358 Ops.push_back(Elt: DAG.getBitcast(
11359 VT: MVT::i32, V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Elt0, Lanes[0]})));
11360 Ops.push_back(Elt: DAG.getBitcast(
11361 VT: MVT::i32,
11362 V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Lanes[1], Lanes[2]})));
11363 }
11364 }
11365 };
11366
11367 if (UseNSA && IsGFX11Plus) {
11368 Ops.push_back(Elt: NodePtr);
11369 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: RayExtent));
11370 Ops.push_back(Elt: RayOrigin);
11371 if (IsA16) {
11372 SmallVector<SDValue, 3> DirLanes, InvDirLanes, MergedLanes;
11373 DAG.ExtractVectorElements(Op: RayDir, Args&: DirLanes, Start: 0, Count: 3);
11374 DAG.ExtractVectorElements(Op: RayInvDir, Args&: InvDirLanes, Start: 0, Count: 3);
11375 for (unsigned I = 0; I < 3; ++I) {
11376 MergedLanes.push_back(Elt: DAG.getBitcast(
11377 VT: MVT::i32, V: DAG.getBuildVector(VT: MVT::v2f16, DL,
11378 Ops: {DirLanes[I], InvDirLanes[I]})));
11379 }
11380 Ops.push_back(Elt: DAG.getBuildVector(VT: MVT::v3i32, DL, Ops: MergedLanes));
11381 } else {
11382 Ops.push_back(Elt: RayDir);
11383 Ops.push_back(Elt: RayInvDir);
11384 }
11385 } else {
11386 if (Is64)
11387 DAG.ExtractVectorElements(Op: DAG.getBitcast(VT: MVT::v2i32, V: NodePtr), Args&: Ops, Start: 0,
11388 Count: 2);
11389 else
11390 Ops.push_back(Elt: NodePtr);
11391
11392 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: RayExtent));
11393 packLanes(RayOrigin, true);
11394 packLanes(RayDir, true);
11395 packLanes(RayInvDir, false);
11396 }
11397
11398 if (!UseNSA) {
11399 // Build a single vector containing all the operands so far prepared.
11400 if (NumVAddrDwords > 12) {
11401 SDValue Undef = DAG.getPOISON(VT: MVT::i32);
11402 Ops.append(NumInputs: 16 - Ops.size(), Elt: Undef);
11403 }
11404 assert(Ops.size() >= 8 && Ops.size() <= 12);
11405 SDValue MergedOps =
11406 DAG.getBuildVector(VT: MVT::getVectorVT(VT: MVT::i32, NumElements: Ops.size()), DL, Ops);
11407 Ops.clear();
11408 Ops.push_back(Elt: MergedOps);
11409 }
11410
11411 Ops.push_back(Elt: TDescr);
11412 Ops.push_back(Elt: DAG.getTargetConstant(Val: IsA16, DL, VT: MVT::i1));
11413 Ops.push_back(Elt: M->getChain());
11414
11415 auto *NewNode = DAG.getMachineNode(Opcode, dl: DL, VTs: M->getVTList(), Ops);
11416 MachineMemOperand *MemRef = M->getMemOperand();
11417 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
11418 return SDValue(NewNode, 0);
11419 }
11420 case Intrinsic::amdgcn_global_atomic_fmin_num:
11421 case Intrinsic::amdgcn_global_atomic_fmax_num:
11422 case Intrinsic::amdgcn_flat_atomic_fmin_num:
11423 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
11424 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11425 SDValue Ops[] = {
11426 M->getOperand(Num: 0), // Chain
11427 M->getOperand(Num: 2), // Ptr
11428 M->getOperand(Num: 3) // Value
11429 };
11430 unsigned Opcode = 0;
11431 switch (IntrID) {
11432 case Intrinsic::amdgcn_global_atomic_fmin_num:
11433 case Intrinsic::amdgcn_flat_atomic_fmin_num: {
11434 Opcode = ISD::ATOMIC_LOAD_FMIN;
11435 break;
11436 }
11437 case Intrinsic::amdgcn_global_atomic_fmax_num:
11438 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
11439 Opcode = ISD::ATOMIC_LOAD_FMAX;
11440 break;
11441 }
11442 default:
11443 llvm_unreachable("unhandled atomic opcode");
11444 }
11445 return DAG.getAtomic(Opcode, dl: SDLoc(Op), MemVT: M->getMemoryVT(), VTList: M->getVTList(),
11446 Ops, MMO: M->getMemOperand());
11447 }
11448 case Intrinsic::amdgcn_s_alloc_vgpr: {
11449 SDValue NumVGPRs = Op.getOperand(i: 2);
11450 if (!NumVGPRs->isDivergent())
11451 return Op;
11452
11453 SDValue ReadFirstLaneID =
11454 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
11455 NumVGPRs = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
11456 N1: ReadFirstLaneID, N2: NumVGPRs);
11457
11458 return DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL, VTList: Op->getVTList(),
11459 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1), N3: NumVGPRs);
11460 }
11461 case Intrinsic::amdgcn_s_get_barrier_state:
11462 case Intrinsic::amdgcn_s_get_named_barrier_state: {
11463 SDValue Chain = Op->getOperand(Num: 0);
11464 SmallVector<SDValue, 2> Ops;
11465 unsigned Opc;
11466
11467 if (isa<ConstantSDNode>(Val: Op->getOperand(Num: 2))) {
11468 uint64_t BarID = cast<ConstantSDNode>(Val: Op->getOperand(Num: 2))->getZExtValue();
11469 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state)
11470 BarID = (BarID >> 4) & 0x3F;
11471 Opc = AMDGPU::S_GET_BARRIER_STATE_IMM;
11472 SDValue K = DAG.getTargetConstant(Val: BarID, DL, VT: MVT::i32);
11473 Ops.push_back(Elt: K);
11474 Ops.push_back(Elt: Chain);
11475 } else {
11476 Opc = AMDGPU::S_GET_BARRIER_STATE_M0;
11477 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
11478 SDValue M0Val;
11479 M0Val = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Op->getOperand(Num: 2),
11480 N2: DAG.getShiftAmountConstant(Val: 4, VT: MVT::i32, DL));
11481 M0Val = SDValue(
11482 DAG.getMachineNode(Opcode: AMDGPU::S_AND_B32, dl: DL, VT: MVT::i32, Op1: M0Val,
11483 Op2: DAG.getTargetConstant(Val: 0x3F, DL, VT: MVT::i32)),
11484 0);
11485 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
11486 } else
11487 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: Op->getOperand(Num: 2)).getValue(R: 0));
11488 }
11489
11490 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
11491 return SDValue(NewMI, 0);
11492 }
11493 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
11494 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
11495 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
11496 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
11497 SDValue Chain = Op->getOperand(Num: 0);
11498 SDValue Ptr = Op->getOperand(Num: 2);
11499 EVT VT = Op->getValueType(ResNo: 0);
11500 return DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl: DL, MemVT: MII->getMemoryVT(), VT,
11501 Chain, Ptr, MMO: MII->getMemOperand());
11502 }
11503 case Intrinsic::amdgcn_flat_load_monitor_b32:
11504 case Intrinsic::amdgcn_flat_load_monitor_b64:
11505 case Intrinsic::amdgcn_flat_load_monitor_b128: {
11506 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
11507 SDValue Chain = Op->getOperand(Num: 0);
11508 SDValue Ptr = Op->getOperand(Num: 2);
11509 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::FLAT_LOAD_MONITOR, dl: DL,
11510 VTList: Op->getVTList(), Ops: {Chain, Ptr},
11511 MemVT: MII->getMemoryVT(), MMO: MII->getMemOperand());
11512 }
11513 case Intrinsic::amdgcn_global_load_monitor_b32:
11514 case Intrinsic::amdgcn_global_load_monitor_b64:
11515 case Intrinsic::amdgcn_global_load_monitor_b128: {
11516 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
11517 SDValue Chain = Op->getOperand(Num: 0);
11518 SDValue Ptr = Op->getOperand(Num: 2);
11519 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::GLOBAL_LOAD_MONITOR, dl: DL,
11520 VTList: Op->getVTList(), Ops: {Chain, Ptr},
11521 MemVT: MII->getMemoryVT(), MMO: MII->getMemOperand());
11522 }
11523 default:
11524
11525 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
11526 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrID))
11527 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: true);
11528
11529 return SDValue();
11530 }
11531}
11532
11533// Call DAG.getMemIntrinsicNode for a load, but first widen a dwordx3 type to
11534// dwordx4 if on SI and handle TFE loads.
11535SDValue SITargetLowering::getMemIntrinsicNode(unsigned Opcode, const SDLoc &DL,
11536 SDVTList VTList,
11537 ArrayRef<SDValue> Ops, EVT MemVT,
11538 MachineMemOperand *MMO,
11539 SelectionDAG &DAG) const {
11540 LLVMContext &C = *DAG.getContext();
11541 MachineFunction &MF = DAG.getMachineFunction();
11542 EVT VT = VTList.VTs[0];
11543
11544 assert(VTList.NumVTs == 2 || VTList.NumVTs == 3);
11545 bool IsTFE = VTList.NumVTs == 3;
11546 if (IsTFE) {
11547 unsigned NumValueDWords = divideCeil(Numerator: VT.getSizeInBits(), Denominator: 32);
11548 unsigned NumOpDWords = NumValueDWords + 1;
11549 EVT OpDWordsVT = EVT::getVectorVT(Context&: C, VT: MVT::i32, NumElements: NumOpDWords);
11550 SDVTList OpDWordsVTList = DAG.getVTList(VT1: OpDWordsVT, VT2: VTList.VTs[2]);
11551 MachineMemOperand *OpDWordsMMO =
11552 MF.getMachineMemOperand(MMO, Offset: 0, Size: NumOpDWords * 4);
11553 SDValue Op = getMemIntrinsicNode(Opcode, DL, VTList: OpDWordsVTList, Ops,
11554 MemVT: OpDWordsVT, MMO: OpDWordsMMO, DAG);
11555 SDValue Status = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
11556 N2: DAG.getVectorIdxConstant(Val: NumValueDWords, DL));
11557 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL);
11558 SDValue ValueDWords =
11559 NumValueDWords == 1
11560 ? DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op, N2: ZeroIdx)
11561 : DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL,
11562 VT: EVT::getVectorVT(Context&: C, VT: MVT::i32, NumElements: NumValueDWords), N1: Op,
11563 N2: ZeroIdx);
11564 SDValue Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: ValueDWords);
11565 return DAG.getMergeValues(Ops: {Value, Status, SDValue(Op.getNode(), 1)}, dl: DL);
11566 }
11567
11568 if (!Subtarget->hasDwordx3LoadStores() &&
11569 (VT == MVT::v3i32 || VT == MVT::v3f32)) {
11570 EVT WidenedVT = EVT::getVectorVT(Context&: C, VT: VT.getVectorElementType(), NumElements: 4);
11571 EVT WidenedMemVT = EVT::getVectorVT(Context&: C, VT: MemVT.getVectorElementType(), NumElements: 4);
11572 MachineMemOperand *WidenedMMO = MF.getMachineMemOperand(MMO, Offset: 0, Size: 16);
11573 SDVTList WidenedVTList = DAG.getVTList(VT1: WidenedVT, VT2: VTList.VTs[1]);
11574 SDValue Op = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: WidenedVTList, Ops,
11575 MemVT: WidenedMemVT, MMO: WidenedMMO);
11576 SDValue Value = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Op,
11577 N2: DAG.getVectorIdxConstant(Val: 0, DL));
11578 return DAG.getMergeValues(Ops: {Value, SDValue(Op.getNode(), 1)}, dl: DL);
11579 }
11580
11581 return DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList, Ops, MemVT, MMO);
11582}
11583
11584SDValue SITargetLowering::handleD16VData(SDValue VData, SelectionDAG &DAG,
11585 bool ImageStore) const {
11586 EVT StoreVT = VData.getValueType();
11587
11588 // No change for f16 and legal vector D16 types.
11589 if (!StoreVT.isVector())
11590 return VData;
11591
11592 SDLoc DL(VData);
11593 unsigned NumElements = StoreVT.getVectorNumElements();
11594
11595 if (Subtarget->hasUnpackedD16VMem()) {
11596 // We need to unpack the packed data to store.
11597 EVT IntStoreVT = StoreVT.changeTypeToInteger();
11598 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
11599
11600 EVT EquivStoreVT =
11601 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements);
11602 SDValue ZExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EquivStoreVT, Operand: IntVData);
11603 return DAG.UnrollVectorOp(N: ZExt.getNode());
11604 }
11605
11606 // The sq block of gfx8.1 does not estimate register use correctly for d16
11607 // image store instructions. The data operand is computed as if it were not a
11608 // d16 image instruction.
11609 if (ImageStore && Subtarget->hasImageStoreD16Bug()) {
11610 // Bitcast to i16
11611 EVT IntStoreVT = StoreVT.changeTypeToInteger();
11612 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
11613
11614 // Decompose into scalars
11615 SmallVector<SDValue, 4> Elts;
11616 DAG.ExtractVectorElements(Op: IntVData, Args&: Elts);
11617
11618 // Group pairs of i16 into v2i16 and bitcast to i32
11619 SmallVector<SDValue, 4> PackedElts;
11620 for (unsigned I = 0; I < Elts.size() / 2; I += 1) {
11621 SDValue Pair =
11622 DAG.getBuildVector(VT: MVT::v2i16, DL, Ops: {Elts[I * 2], Elts[I * 2 + 1]});
11623 SDValue IntPair = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Pair);
11624 PackedElts.push_back(Elt: IntPair);
11625 }
11626 if ((NumElements % 2) == 1) {
11627 // Handle v3i16
11628 unsigned I = Elts.size() / 2;
11629 SDValue Pair = DAG.getBuildVector(VT: MVT::v2i16, DL,
11630 Ops: {Elts[I * 2], DAG.getPOISON(VT: MVT::i16)});
11631 SDValue IntPair = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Pair);
11632 PackedElts.push_back(Elt: IntPair);
11633 }
11634
11635 // Pad using UNDEF
11636 PackedElts.resize(N: Elts.size(), NV: DAG.getPOISON(VT: MVT::i32));
11637
11638 // Build final vector
11639 EVT VecVT =
11640 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: PackedElts.size());
11641 return DAG.getBuildVector(VT: VecVT, DL, Ops: PackedElts);
11642 }
11643
11644 if (NumElements == 3) {
11645 EVT IntStoreVT =
11646 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StoreVT.getStoreSizeInBits());
11647 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
11648
11649 EVT WidenedStoreVT = EVT::getVectorVT(
11650 Context&: *DAG.getContext(), VT: StoreVT.getVectorElementType(), NumElements: NumElements + 1);
11651 EVT WidenedIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
11652 BitWidth: WidenedStoreVT.getStoreSizeInBits());
11653 SDValue ZExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenedIntVT, Operand: IntVData);
11654 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WidenedStoreVT, Operand: ZExt);
11655 }
11656
11657 assert(isTypeLegal(StoreVT));
11658 return VData;
11659}
11660
11661static bool isAsyncLDSDMA(Intrinsic::ID Intr) {
11662 switch (Intr) {
11663 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
11664 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
11665 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
11666 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds:
11667 case Intrinsic::amdgcn_load_async_to_lds:
11668 case Intrinsic::amdgcn_global_load_async_lds:
11669 return true;
11670 }
11671 return false;
11672}
11673
11674SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
11675 SelectionDAG &DAG) const {
11676 SDLoc DL(Op);
11677 SDValue Chain = Op.getOperand(i: 0);
11678 unsigned IntrinsicID = Op.getConstantOperandVal(i: 1);
11679
11680 switch (IntrinsicID) {
11681 case Intrinsic::amdgcn_exp_compr: {
11682 if (!Subtarget->hasCompressedExport()) {
11683 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11684 DAG.getMachineFunction().getFunction(),
11685 "intrinsic not supported on subtarget", DL.getDebugLoc()));
11686 }
11687 SDValue Src0 = Op.getOperand(i: 4);
11688 SDValue Src1 = Op.getOperand(i: 5);
11689 // Hack around illegal type on SI by directly selecting it.
11690 if (isTypeLegal(VT: Src0.getValueType()))
11691 return SDValue();
11692
11693 const ConstantSDNode *Done = cast<ConstantSDNode>(Val: Op.getOperand(i: 6));
11694 SDValue Undef = DAG.getPOISON(VT: MVT::f32);
11695 const SDValue Ops[] = {
11696 Op.getOperand(i: 2), // tgt
11697 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: Src0), // src0
11698 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: Src1), // src1
11699 Undef, // src2
11700 Undef, // src3
11701 Op.getOperand(i: 7), // vm
11702 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // compr
11703 Op.getOperand(i: 3), // en
11704 Op.getOperand(i: 0) // Chain
11705 };
11706
11707 unsigned Opc = Done->isZero() ? AMDGPU::EXP : AMDGPU::EXP_DONE;
11708 return SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops), 0);
11709 }
11710
11711 case Intrinsic::amdgcn_struct_tbuffer_store:
11712 case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
11713 SDValue VData = Op.getOperand(i: 2);
11714 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16);
11715 if (IsD16)
11716 VData = handleD16VData(VData, DAG);
11717 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
11718 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
11719 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
11720 SDValue Ops[] = {
11721 Chain,
11722 VData, // vdata
11723 Rsrc, // rsrc
11724 Op.getOperand(i: 4), // vindex
11725 VOffset, // voffset
11726 SOffset, // soffset
11727 Offset, // offset
11728 Op.getOperand(i: 7), // format
11729 Op.getOperand(i: 8), // cachepolicy, swizzled buffer
11730 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
11731 };
11732 unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16
11733 : AMDGPUISD::TBUFFER_STORE_FORMAT;
11734 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11735 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
11736 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
11737 }
11738
11739 case Intrinsic::amdgcn_raw_tbuffer_store:
11740 case Intrinsic::amdgcn_raw_ptr_tbuffer_store: {
11741 SDValue VData = Op.getOperand(i: 2);
11742 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16);
11743 if (IsD16)
11744 VData = handleD16VData(VData, DAG);
11745 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
11746 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
11747 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
11748 SDValue Ops[] = {
11749 Chain,
11750 VData, // vdata
11751 Rsrc, // rsrc
11752 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
11753 VOffset, // voffset
11754 SOffset, // soffset
11755 Offset, // offset
11756 Op.getOperand(i: 6), // format
11757 Op.getOperand(i: 7), // cachepolicy, swizzled buffer
11758 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
11759 };
11760 unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16
11761 : AMDGPUISD::TBUFFER_STORE_FORMAT;
11762 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11763 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
11764 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
11765 }
11766
11767 case Intrinsic::amdgcn_raw_buffer_store:
11768 case Intrinsic::amdgcn_raw_ptr_buffer_store:
11769 case Intrinsic::amdgcn_raw_buffer_store_format:
11770 case Intrinsic::amdgcn_raw_ptr_buffer_store_format: {
11771 const bool IsFormat =
11772 IntrinsicID == Intrinsic::amdgcn_raw_buffer_store_format ||
11773 IntrinsicID == Intrinsic::amdgcn_raw_ptr_buffer_store_format;
11774
11775 SDValue VData = Op.getOperand(i: 2);
11776 EVT VDataVT = VData.getValueType();
11777 EVT EltType = VDataVT.getScalarType();
11778 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
11779 if (IsD16) {
11780 VData = handleD16VData(VData, DAG);
11781 VDataVT = VData.getValueType();
11782 }
11783
11784 if (!isTypeLegal(VT: VDataVT)) {
11785 VData =
11786 DAG.getNode(Opcode: ISD::BITCAST, DL,
11787 VT: getEquivalentMemType(Context&: *DAG.getContext(), VT: VDataVT), Operand: VData);
11788 }
11789
11790 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
11791 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
11792 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
11793 SDValue Ops[] = {
11794 Chain,
11795 VData,
11796 Rsrc,
11797 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
11798 VOffset, // voffset
11799 SOffset, // soffset
11800 Offset, // offset
11801 Op.getOperand(i: 6), // cachepolicy, swizzled buffer
11802 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
11803 };
11804 unsigned Opc =
11805 IsFormat ? AMDGPUISD::BUFFER_STORE_FORMAT : AMDGPUISD::BUFFER_STORE;
11806 Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 : Opc;
11807 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11808
11809 // Handle BUFFER_STORE_BYTE/SHORT overloaded intrinsics
11810 if (!IsD16 && !VDataVT.isVector() && EltType.getSizeInBits() < 32)
11811 return handleByteShortBufferStores(DAG, VDataType: VDataVT, DL, Ops, M);
11812
11813 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
11814 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
11815 }
11816
11817 case Intrinsic::amdgcn_struct_buffer_store:
11818 case Intrinsic::amdgcn_struct_ptr_buffer_store:
11819 case Intrinsic::amdgcn_struct_buffer_store_format:
11820 case Intrinsic::amdgcn_struct_ptr_buffer_store_format: {
11821 const bool IsFormat =
11822 IntrinsicID == Intrinsic::amdgcn_struct_buffer_store_format ||
11823 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_store_format;
11824
11825 SDValue VData = Op.getOperand(i: 2);
11826 EVT VDataVT = VData.getValueType();
11827 EVT EltType = VDataVT.getScalarType();
11828 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
11829
11830 if (IsD16) {
11831 VData = handleD16VData(VData, DAG);
11832 VDataVT = VData.getValueType();
11833 }
11834
11835 if (!isTypeLegal(VT: VDataVT)) {
11836 VData =
11837 DAG.getNode(Opcode: ISD::BITCAST, DL,
11838 VT: getEquivalentMemType(Context&: *DAG.getContext(), VT: VDataVT), Operand: VData);
11839 }
11840
11841 auto Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
11842 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
11843 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
11844 SDValue Ops[] = {
11845 Chain,
11846 VData,
11847 Rsrc,
11848 Op.getOperand(i: 4), // vindex
11849 VOffset, // voffset
11850 SOffset, // soffset
11851 Offset, // offset
11852 Op.getOperand(i: 7), // cachepolicy, swizzled buffer
11853 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
11854 };
11855 unsigned Opc =
11856 !IsFormat ? AMDGPUISD::BUFFER_STORE : AMDGPUISD::BUFFER_STORE_FORMAT;
11857 Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 : Opc;
11858 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11859
11860 // Handle BUFFER_STORE_BYTE/SHORT overloaded intrinsics
11861 EVT VDataType = VData.getValueType().getScalarType();
11862 if (!IsD16 && !VDataVT.isVector() && EltType.getSizeInBits() < 32)
11863 return handleByteShortBufferStores(DAG, VDataType, DL, Ops, M);
11864
11865 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
11866 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
11867 }
11868 case Intrinsic::amdgcn_raw_buffer_load_lds:
11869 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
11870 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
11871 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
11872 case Intrinsic::amdgcn_struct_buffer_load_lds:
11873 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
11874 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
11875 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
11876 if (!Subtarget->hasVMemToLDSLoad())
11877 return SDValue();
11878 unsigned Opc;
11879 bool HasVIndex =
11880 IntrinsicID == Intrinsic::amdgcn_struct_buffer_load_lds ||
11881 IntrinsicID == Intrinsic::amdgcn_struct_buffer_load_async_lds ||
11882 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_load_lds ||
11883 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds;
11884 unsigned OpOffset = HasVIndex ? 1 : 0;
11885 SDValue VOffset = Op.getOperand(i: 5 + OpOffset);
11886 bool HasVOffset = !isNullConstant(V: VOffset);
11887 unsigned Size = Op->getConstantOperandVal(Num: 4);
11888
11889 switch (Size) {
11890 default:
11891 return SDValue();
11892 case 1:
11893 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
11894 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
11895 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
11896 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
11897 break;
11898 case 2:
11899 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
11900 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
11901 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
11902 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
11903 break;
11904 case 4:
11905 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
11906 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
11907 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
11908 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
11909 break;
11910 case 12:
11911 if (!Subtarget->hasLDSLoadB96_B128())
11912 return SDValue();
11913 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
11914 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
11915 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
11916 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
11917 break;
11918 case 16:
11919 if (!Subtarget->hasLDSLoadB96_B128())
11920 return SDValue();
11921 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
11922 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
11923 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
11924 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
11925 break;
11926 }
11927
11928 SDValue M0Val = copyToM0(DAG, Chain, DL, V: Op.getOperand(i: 3));
11929
11930 SmallVector<SDValue, 8> Ops;
11931
11932 if (HasVIndex && HasVOffset)
11933 Ops.push_back(Elt: DAG.getBuildVector(VT: MVT::v2i32, DL,
11934 Ops: {Op.getOperand(i: 5), // VIndex
11935 VOffset}));
11936 else if (HasVIndex)
11937 Ops.push_back(Elt: Op.getOperand(i: 5));
11938 else if (HasVOffset)
11939 Ops.push_back(Elt: VOffset);
11940
11941 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
11942 Ops.push_back(Elt: Rsrc);
11943 Ops.push_back(Elt: Op.getOperand(i: 6 + OpOffset)); // soffset
11944 Ops.push_back(Elt: Op.getOperand(i: 7 + OpOffset)); // imm offset
11945 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
11946 unsigned Aux = Op.getConstantOperandVal(i: 8 + OpOffset);
11947 Ops.push_back(Elt: DAG.getTargetConstant(
11948 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12),
11949 DL, VT: MVT::i8)); // cpol
11950 Ops.push_back(Elt: DAG.getTargetConstant(
11951 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::SWZ : AMDGPU::CPol::SWZ_pregfx12)
11952 ? 1
11953 : 0,
11954 DL, VT: MVT::i8)); // swz
11955 Ops.push_back(
11956 Elt: DAG.getTargetConstant(Val: isAsyncLDSDMA(Intr: IntrinsicID), DL, VT: MVT::i8));
11957 Ops.push_back(Elt: M0Val.getValue(R: 0)); // Chain
11958 Ops.push_back(Elt: M0Val.getValue(R: 1)); // Glue
11959
11960 auto *M = cast<MemSDNode>(Val&: Op);
11961 auto *Load = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: M->getVTList(), Ops);
11962 DAG.setNodeMemRefs(N: Load, NewMemRefs: M->memoperands());
11963
11964 return SDValue(Load, 0);
11965 }
11966 // Buffers are handled by LowerBufferFatPointers, and we're going to go
11967 // for "trust me" that the remaining cases are global pointers until
11968 // such time as we can put two mem operands on an intrinsic.
11969 case Intrinsic::amdgcn_load_to_lds:
11970 case Intrinsic::amdgcn_load_async_to_lds:
11971 case Intrinsic::amdgcn_global_load_lds:
11972 case Intrinsic::amdgcn_global_load_async_lds: {
11973 if (!Subtarget->hasVMemToLDSLoad())
11974 return SDValue();
11975
11976 unsigned Opc;
11977 unsigned Size = Op->getConstantOperandVal(Num: 4);
11978 switch (Size) {
11979 default:
11980 return SDValue();
11981 case 1:
11982 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
11983 break;
11984 case 2:
11985 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
11986 break;
11987 case 4:
11988 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
11989 break;
11990 case 12:
11991 if (!Subtarget->hasLDSLoadB96_B128())
11992 return SDValue();
11993 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
11994 break;
11995 case 16:
11996 if (!Subtarget->hasLDSLoadB96_B128())
11997 return SDValue();
11998 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
11999 break;
12000 }
12001
12002 SDValue M0Val = copyToM0(DAG, Chain, DL, V: Op.getOperand(i: 3));
12003
12004 SmallVector<SDValue, 6> Ops;
12005
12006 SDValue Addr = Op.getOperand(i: 2); // Global ptr
12007 SDValue VOffset;
12008 // Try to split SAddr and VOffset. Global and LDS pointers share the same
12009 // immediate offset, so we cannot use a regular SelectGlobalSAddr().
12010 if (Addr->isDivergent() && Addr->isAnyAdd()) {
12011 SDValue LHS = Addr.getOperand(i: 0);
12012 SDValue RHS = Addr.getOperand(i: 1);
12013
12014 if (LHS->isDivergent())
12015 std::swap(a&: LHS, b&: RHS);
12016
12017 if (!LHS->isDivergent() && RHS.getOpcode() == ISD::ZERO_EXTEND &&
12018 RHS.getOperand(i: 0).getValueType() == MVT::i32) {
12019 // add (i64 sgpr), (zero_extend (i32 vgpr))
12020 Addr = LHS;
12021 VOffset = RHS.getOperand(i: 0);
12022 }
12023 }
12024
12025 Ops.push_back(Elt: Addr);
12026 if (!Addr->isDivergent()) {
12027 Opc = AMDGPU::getGlobalSaddrOp(Opcode: Opc);
12028 if (!VOffset)
12029 VOffset =
12030 SDValue(DAG.getMachineNode(Opcode: AMDGPU::V_MOV_B32_e32, dl: DL, VT: MVT::i32,
12031 Op1: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32)),
12032 0);
12033 Ops.push_back(Elt: VOffset);
12034 }
12035
12036 Ops.push_back(Elt: Op.getOperand(i: 5)); // Offset
12037
12038 unsigned Aux = Op.getConstantOperandVal(i: 6);
12039 Ops.push_back(Elt: DAG.getTargetConstant(Val: Aux & ~AMDGPU::CPol::VIRTUAL_BITS, DL,
12040 VT: MVT::i32)); // CPol
12041 Ops.push_back(
12042 Elt: DAG.getTargetConstant(Val: isAsyncLDSDMA(Intr: IntrinsicID), DL, VT: MVT::i8));
12043
12044 Ops.push_back(Elt: M0Val.getValue(R: 0)); // Chain
12045 Ops.push_back(Elt: M0Val.getValue(R: 1)); // Glue
12046
12047 auto *M = cast<MemSDNode>(Val&: Op);
12048 auto *Load = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
12049 DAG.setNodeMemRefs(N: Load, NewMemRefs: M->memoperands());
12050
12051 return SDValue(Load, 0);
12052 }
12053 case Intrinsic::amdgcn_end_cf:
12054 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::SI_END_CF, dl: DL, VT: MVT::Other,
12055 Op1: Op->getOperand(Num: 2), Op2: Chain),
12056 0);
12057 case Intrinsic::amdgcn_s_barrier_init:
12058 case Intrinsic::amdgcn_s_barrier_signal_var: {
12059 // these two intrinsics have two operands: barrier pointer and member count
12060 SDValue Chain = Op->getOperand(Num: 0);
12061 SmallVector<SDValue, 2> Ops;
12062 SDValue BarOp = Op->getOperand(Num: 2);
12063 SDValue CntOp = Op->getOperand(Num: 3);
12064 SDValue M0Val;
12065 unsigned Opc = IntrinsicID == Intrinsic::amdgcn_s_barrier_init
12066 ? AMDGPU::S_BARRIER_INIT_M0
12067 : AMDGPU::S_BARRIER_SIGNAL_M0;
12068 // extract the BarrierID from bits 4-9 of BarOp
12069 SDValue BarID;
12070 BarID = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: BarOp,
12071 N2: DAG.getShiftAmountConstant(Val: 4, VT: MVT::i32, DL));
12072 BarID =
12073 SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_AND_B32, dl: DL, VT: MVT::i32, Op1: BarID,
12074 Op2: DAG.getTargetConstant(Val: 0x3F, DL, VT: MVT::i32)),
12075 0);
12076 // Member count should be put into M0[ShAmt:+6]
12077 // Barrier ID should be put into M0[5:0]
12078 M0Val =
12079 SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_AND_B32, dl: DL, VT: MVT::i32, Op1: CntOp,
12080 Op2: DAG.getTargetConstant(Val: 0x3F, DL, VT: MVT::i32)),
12081 0);
12082 constexpr unsigned ShAmt = 16;
12083 M0Val = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: CntOp,
12084 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT: MVT::i32, DL));
12085
12086 M0Val = SDValue(
12087 DAG.getMachineNode(Opcode: AMDGPU::S_OR_B32, dl: DL, VT: MVT::i32, Op1: M0Val, Op2: BarID), 0);
12088
12089 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
12090
12091 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
12092 return SDValue(NewMI, 0);
12093 }
12094 case Intrinsic::amdgcn_s_wakeup_barrier: {
12095 if (!Subtarget->hasSWakeupBarrier())
12096 return SDValue();
12097 [[fallthrough]];
12098 }
12099 case Intrinsic::amdgcn_s_barrier_join: {
12100 // these three intrinsics have one operand: barrier pointer
12101 SDValue Chain = Op->getOperand(Num: 0);
12102 SmallVector<SDValue, 2> Ops;
12103 SDValue BarOp = Op->getOperand(Num: 2);
12104 unsigned Opc;
12105
12106 if (isa<ConstantSDNode>(Val: BarOp)) {
12107 uint64_t BarVal = cast<ConstantSDNode>(Val&: BarOp)->getZExtValue();
12108 switch (IntrinsicID) {
12109 default:
12110 return SDValue();
12111 case Intrinsic::amdgcn_s_barrier_join:
12112 Opc = AMDGPU::S_BARRIER_JOIN_IMM;
12113 break;
12114 case Intrinsic::amdgcn_s_wakeup_barrier:
12115 Opc = AMDGPU::S_WAKEUP_BARRIER_IMM;
12116 break;
12117 }
12118 // extract the BarrierID from bits 4-9 of the immediate
12119 unsigned BarID = (BarVal >> 4) & 0x3F;
12120 SDValue K = DAG.getTargetConstant(Val: BarID, DL, VT: MVT::i32);
12121 Ops.push_back(Elt: K);
12122 Ops.push_back(Elt: Chain);
12123 } else {
12124 switch (IntrinsicID) {
12125 default:
12126 return SDValue();
12127 case Intrinsic::amdgcn_s_barrier_join:
12128 Opc = AMDGPU::S_BARRIER_JOIN_M0;
12129 break;
12130 case Intrinsic::amdgcn_s_wakeup_barrier:
12131 Opc = AMDGPU::S_WAKEUP_BARRIER_M0;
12132 break;
12133 }
12134 // extract the BarrierID from bits 4-9 of BarOp, copy to M0[5:0]
12135 SDValue M0Val;
12136 M0Val = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: BarOp,
12137 N2: DAG.getShiftAmountConstant(Val: 4, VT: MVT::i32, DL));
12138 M0Val =
12139 SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_AND_B32, dl: DL, VT: MVT::i32, Op1: M0Val,
12140 Op2: DAG.getTargetConstant(Val: 0x3F, DL, VT: MVT::i32)),
12141 0);
12142 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
12143 }
12144
12145 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
12146 return SDValue(NewMI, 0);
12147 }
12148 case Intrinsic::amdgcn_s_prefetch_data: {
12149 // For non-global address space preserve the chain and remove the call.
12150 if (!AMDGPU::isFlatGlobalAddrSpace(AS: cast<MemSDNode>(Val&: Op)->getAddressSpace()))
12151 return Op.getOperand(i: 0);
12152 return Op;
12153 }
12154 case Intrinsic::amdgcn_s_buffer_prefetch_data: {
12155 SDValue Ops[] = {
12156 Chain, bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG),
12157 Op.getOperand(i: 3), // offset
12158 Op.getOperand(i: 4), // length
12159 };
12160
12161 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12162 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::SBUFFER_PREFETCH_DATA, dl: DL,
12163 VTList: Op->getVTList(), Ops, MemVT: M->getMemoryVT(),
12164 MMO: M->getMemOperand());
12165 }
12166 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
12167 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
12168 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
12169 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12170 SDValue Chain = Op->getOperand(Num: 0);
12171 SDValue Ptr = Op->getOperand(Num: 2);
12172 SDValue Val = Op->getOperand(Num: 3);
12173 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MII->getMemoryVT(), Chain, Ptr: Val,
12174 Val: Ptr, MMO: MII->getMemOperand());
12175 }
12176 default: {
12177 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
12178 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrinsicID))
12179 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: true);
12180
12181 return Op;
12182 }
12183 }
12184}
12185
12186// Return whether the operation has NoUnsignedWrap property.
12187static bool isNoUnsignedWrap(SDValue Addr) {
12188 return (Addr.getOpcode() == ISD::ADD &&
12189 Addr->getFlags().hasNoUnsignedWrap()) ||
12190 Addr->getOpcode() == ISD::OR;
12191}
12192
12193bool SITargetLowering::shouldPreservePtrArith(const Function &F,
12194 EVT PtrVT) const {
12195 return PtrVT == MVT::i64;
12196}
12197
12198bool SITargetLowering::canTransformPtrArithOutOfBounds(const Function &F,
12199 EVT PtrVT) const {
12200 return true;
12201}
12202
12203// The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
12204// offset (the offset that is included in bounds checking and swizzling, to be
12205// split between the instruction's voffset and immoffset fields) and soffset
12206// (the offset that is excluded from bounds checking and swizzling, to go in
12207// the instruction's soffset field). This function takes the first kind of
12208// offset and figures out how to split it between voffset and immoffset.
12209std::pair<SDValue, SDValue>
12210SITargetLowering::splitBufferOffsets(SDValue Offset, SelectionDAG &DAG) const {
12211 SDLoc DL(Offset);
12212 const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(ST: *Subtarget);
12213 SDValue N0 = Offset;
12214 ConstantSDNode *C1 = nullptr;
12215
12216 if ((C1 = dyn_cast<ConstantSDNode>(Val&: N0)))
12217 N0 = SDValue();
12218 else if (DAG.isBaseWithConstantOffset(Op: N0)) {
12219 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
12220 // being added, so we can only safely match a 32-bit addition with no
12221 // unsigned overflow.
12222 bool CheckNUW = Subtarget->hasGFX1250Insts();
12223 if (!CheckNUW || isNoUnsignedWrap(Addr: N0)) {
12224 C1 = cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
12225 N0 = N0.getOperand(i: 0);
12226 }
12227 }
12228
12229 if (C1) {
12230 unsigned ImmOffset = C1->getZExtValue();
12231 // If the immediate value is too big for the immoffset field, put only bits
12232 // that would normally fit in the immoffset field. The remaining value that
12233 // is copied/added for the voffset field is a large power of 2, and it
12234 // stands more chance of being CSEd with the copy/add for another similar
12235 // load/store.
12236 // However, do not do that rounding down if that is a negative
12237 // number, as it appears to be illegal to have a negative offset in the
12238 // vgpr, even if adding the immediate offset makes it positive.
12239 unsigned Overflow = ImmOffset & ~MaxImm;
12240 ImmOffset -= Overflow;
12241 if ((int32_t)Overflow < 0) {
12242 Overflow += ImmOffset;
12243 ImmOffset = 0;
12244 }
12245 C1 = cast<ConstantSDNode>(Val: DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32));
12246 if (Overflow) {
12247 auto OverflowVal = DAG.getConstant(Val: Overflow, DL, VT: MVT::i32);
12248 if (!N0)
12249 N0 = OverflowVal;
12250 else {
12251 SDValue Ops[] = {N0, OverflowVal};
12252 N0 = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, Ops);
12253 }
12254 }
12255 }
12256 if (!N0)
12257 N0 = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
12258 if (!C1)
12259 C1 = cast<ConstantSDNode>(Val: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
12260 return {N0, SDValue(C1, 0)};
12261}
12262
12263// Analyze a combined offset from an amdgcn_s_buffer_load intrinsic and store
12264// the three offsets (voffset, soffset and instoffset) into the SDValue[3] array
12265// pointed to by Offsets.
12266void SITargetLowering::setBufferOffsets(SDValue CombinedOffset,
12267 SelectionDAG &DAG, SDValue *Offsets,
12268 Align Alignment) const {
12269 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
12270 SDLoc DL(CombinedOffset);
12271 if (auto *C = dyn_cast<ConstantSDNode>(Val&: CombinedOffset)) {
12272 uint32_t Imm = C->getZExtValue();
12273 uint32_t SOffset, ImmOffset;
12274 if (TII->splitMUBUFOffset(Imm, SOffset, ImmOffset, Alignment)) {
12275 Offsets[0] = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
12276 Offsets[1] = DAG.getConstant(Val: SOffset, DL, VT: MVT::i32);
12277 Offsets[2] = DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32);
12278 return;
12279 }
12280 }
12281 if (DAG.isBaseWithConstantOffset(Op: CombinedOffset)) {
12282 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
12283 // being added, so we can only safely match a 32-bit addition with no
12284 // unsigned overflow.
12285 bool CheckNUW = Subtarget->hasGFX1250Insts();
12286 SDValue N0 = CombinedOffset.getOperand(i: 0);
12287 SDValue N1 = CombinedOffset.getOperand(i: 1);
12288 uint32_t SOffset, ImmOffset;
12289 int Offset = cast<ConstantSDNode>(Val&: N1)->getSExtValue();
12290 if (Offset >= 0 && (!CheckNUW || isNoUnsignedWrap(Addr: CombinedOffset)) &&
12291 TII->splitMUBUFOffset(Imm: Offset, SOffset, ImmOffset, Alignment)) {
12292 Offsets[0] = N0;
12293 Offsets[1] = DAG.getConstant(Val: SOffset, DL, VT: MVT::i32);
12294 Offsets[2] = DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32);
12295 return;
12296 }
12297 }
12298
12299 SDValue SOffsetZero = Subtarget->hasRestrictedSOffset()
12300 ? DAG.getRegister(Reg: AMDGPU::SGPR_NULL, VT: MVT::i32)
12301 : DAG.getConstant(Val: 0, DL, VT: MVT::i32);
12302
12303 Offsets[0] = CombinedOffset;
12304 Offsets[1] = SOffsetZero;
12305 Offsets[2] = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32);
12306}
12307
12308SDValue SITargetLowering::bufferRsrcPtrToVector(SDValue MaybePointer,
12309 SelectionDAG &DAG) const {
12310 if (!MaybePointer.getValueType().isScalarInteger())
12311 return MaybePointer;
12312
12313 SDValue Rsrc = DAG.getBitcast(VT: MVT::v4i32, V: MaybePointer);
12314 return Rsrc;
12315}
12316
12317// Wrap a global or flat pointer into a buffer intrinsic using the flags
12318// specified in the intrinsic.
12319SDValue SITargetLowering::lowerPointerAsRsrcIntrin(SDNode *Op,
12320 SelectionDAG &DAG) const {
12321 SDLoc Loc(Op);
12322
12323 SDValue Pointer = Op->getOperand(Num: 1);
12324 SDValue Stride = Op->getOperand(Num: 2);
12325 SDValue NumRecords = Op->getOperand(Num: 3);
12326 SDValue Flags = Op->getOperand(Num: 4);
12327
12328 SDValue ExtStride = DAG.getAnyExtOrTrunc(Op: Stride, DL: Loc, VT: MVT::i32);
12329 SDValue Rsrc;
12330
12331 if (Subtarget->has45BitNumRecordsBufferResource()) {
12332 SDValue Zero = DAG.getConstant(Val: 0, DL: Loc, VT: MVT::i32);
12333 // Build the lower 64-bit value, which has a 57-bit base and the lower 7-bit
12334 // num_records.
12335 SDValue ExtPointer = DAG.getAnyExtOrTrunc(Op: Pointer, DL: Loc, VT: MVT::i64);
12336 SDValue NumRecordsLHS =
12337 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i64, N1: NumRecords,
12338 N2: DAG.getShiftAmountConstant(Val: 57, VT: MVT::i32, DL: Loc));
12339 SDValue LowHalf =
12340 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: ExtPointer, N2: NumRecordsLHS);
12341
12342 // Build the higher 64-bit value, which has the higher 38-bit num_records,
12343 // 6-bit zero (omit), 16-bit stride and scale and 4-bit flag.
12344 SDValue NumRecordsRHS =
12345 DAG.getNode(Opcode: ISD::SRL, DL: Loc, VT: MVT::i64, N1: NumRecords,
12346 N2: DAG.getShiftAmountConstant(Val: 7, VT: MVT::i32, DL: Loc));
12347 SDValue ShiftedStride =
12348 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: ExtStride,
12349 N2: DAG.getShiftAmountConstant(Val: 12, VT: MVT::i32, DL: Loc));
12350 SDValue ExtShiftedStrideVec =
12351 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i32, N1: Zero, N2: ShiftedStride);
12352 SDValue ExtShiftedStride =
12353 DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i64, Operand: ExtShiftedStrideVec);
12354 SDValue ShiftedFlags =
12355 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: Flags,
12356 N2: DAG.getShiftAmountConstant(Val: 28, VT: MVT::i32, DL: Loc));
12357 SDValue ExtShiftedFlagsVec =
12358 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i32, N1: Zero, N2: ShiftedFlags);
12359 SDValue ExtShiftedFlags =
12360 DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i64, Operand: ExtShiftedFlagsVec);
12361 SDValue CombinedFields =
12362 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: NumRecordsRHS, N2: ExtShiftedStride);
12363 SDValue HighHalf =
12364 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: CombinedFields, N2: ExtShiftedFlags);
12365
12366 Rsrc = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i64, N1: LowHalf, N2: HighHalf);
12367 } else {
12368 NumRecords = DAG.getAnyExtOrTrunc(Op: NumRecords, DL: Loc, VT: MVT::i32);
12369 auto [LowHalf, HighHalf] =
12370 DAG.SplitScalar(N: Pointer, DL: Loc, LoVT: MVT::i32, HiVT: MVT::i32);
12371 SDValue Mask = DAG.getConstant(Val: 0x0000ffff, DL: Loc, VT: MVT::i32);
12372 SDValue Masked = DAG.getNode(Opcode: ISD::AND, DL: Loc, VT: MVT::i32, N1: HighHalf, N2: Mask);
12373 SDValue ShiftedStride =
12374 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: ExtStride,
12375 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: Loc));
12376 SDValue NewHighHalf =
12377 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i32, N1: Masked, N2: ShiftedStride);
12378
12379 Rsrc = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v4i32, N1: LowHalf, N2: NewHighHalf,
12380 N3: NumRecords, N4: Flags);
12381 }
12382
12383 SDValue RsrcPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i128, Operand: Rsrc);
12384 return RsrcPtr;
12385}
12386
12387// Handle 8 bit and 16 bit buffer loads
12388SDValue SITargetLowering::handleByteShortBufferLoads(SelectionDAG &DAG,
12389 EVT LoadVT, SDLoc DL,
12390 ArrayRef<SDValue> Ops,
12391 MachineMemOperand *MMO,
12392 bool IsTFE) const {
12393 EVT IntVT = LoadVT.changeTypeToInteger();
12394
12395 if (IsTFE) {
12396 unsigned Opc = (LoadVT.getScalarType() == MVT::i8)
12397 ? AMDGPUISD::BUFFER_LOAD_UBYTE_TFE
12398 : AMDGPUISD::BUFFER_LOAD_USHORT_TFE;
12399 MachineFunction &MF = DAG.getMachineFunction();
12400 MachineMemOperand *OpMMO = MF.getMachineMemOperand(MMO, Offset: 0, Size: 8);
12401 SDVTList VTs = DAG.getVTList(VT1: MVT::v2i32, VT2: MVT::Other);
12402 SDValue Op = getMemIntrinsicNode(Opcode: Opc, DL, VTList: VTs, Ops, MemVT: MVT::v2i32, MMO: OpMMO, DAG);
12403 SDValue Status = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
12404 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
12405 SDValue Data = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
12406 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
12407 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: Data);
12408 SDValue Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: Trunc);
12409 return DAG.getMergeValues(Ops: {Value, Status, SDValue(Op.getNode(), 1)}, dl: DL);
12410 }
12411
12412 unsigned Opc = LoadVT.getScalarType() == MVT::i8
12413 ? AMDGPUISD::BUFFER_LOAD_UBYTE
12414 : AMDGPUISD::BUFFER_LOAD_USHORT;
12415
12416 SDVTList ResList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
12417 SDValue BufferLoad =
12418 DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: ResList, Ops, MemVT: IntVT, MMO);
12419 SDValue LoadVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: BufferLoad);
12420 LoadVal = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: LoadVal);
12421
12422 return DAG.getMergeValues(Ops: {LoadVal, BufferLoad.getValue(R: 1)}, dl: DL);
12423}
12424
12425// Handle 8 bit and 16 bit buffer stores
12426SDValue SITargetLowering::handleByteShortBufferStores(SelectionDAG &DAG,
12427 EVT VDataType, SDLoc DL,
12428 SDValue Ops[],
12429 MemSDNode *M) const {
12430 if (VDataType == MVT::f16 || VDataType == MVT::bf16)
12431 Ops[1] = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Ops[1]);
12432
12433 SDValue BufferStoreExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Ops[1]);
12434 Ops[1] = BufferStoreExt;
12435 unsigned Opc = (VDataType == MVT::i8) ? AMDGPUISD::BUFFER_STORE_BYTE
12436 : AMDGPUISD::BUFFER_STORE_SHORT;
12437 ArrayRef<SDValue> OpsRef = ArrayRef(&Ops[0], 9);
12438 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: M->getVTList(), Ops: OpsRef, MemVT: VDataType,
12439 MMO: M->getMemOperand());
12440}
12441
12442static SDValue getLoadExtOrTrunc(SelectionDAG &DAG, ISD::LoadExtType ExtType,
12443 SDValue Op, const SDLoc &SL, EVT VT) {
12444 if (VT.bitsLT(VT: Op.getValueType()))
12445 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
12446
12447 switch (ExtType) {
12448 case ISD::SEXTLOAD:
12449 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SL, VT, Operand: Op);
12450 case ISD::ZEXTLOAD:
12451 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT, Operand: Op);
12452 case ISD::EXTLOAD:
12453 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT, Operand: Op);
12454 case ISD::NON_EXTLOAD:
12455 return Op;
12456 }
12457
12458 llvm_unreachable("invalid ext type");
12459}
12460
12461// Try to turn 8 and 16-bit scalar loads into SMEM eligible 32-bit loads.
12462// TODO: Skip this on GFX12 which does have scalar sub-dword loads.
12463SDValue SITargetLowering::widenLoad(LoadSDNode *Ld,
12464 DAGCombinerInfo &DCI) const {
12465 SelectionDAG &DAG = DCI.DAG;
12466 if (Ld->getAlign() < Align(4) || Ld->isDivergent())
12467 return SDValue();
12468
12469 // FIXME: Constant loads should all be marked invariant.
12470 unsigned AS = Ld->getAddressSpace();
12471 if (AS != AMDGPUAS::CONSTANT_ADDRESS &&
12472 AS != AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
12473 (AS != AMDGPUAS::GLOBAL_ADDRESS || !Ld->isInvariant()))
12474 return SDValue();
12475
12476 // Don't do this early, since it may interfere with adjacent load merging for
12477 // illegal types. We can avoid losing alignment information for exotic types
12478 // pre-legalize.
12479 EVT MemVT = Ld->getMemoryVT();
12480 if ((MemVT.isSimple() && !DCI.isAfterLegalizeDAG()) ||
12481 MemVT.getSizeInBits() >= 32)
12482 return SDValue();
12483
12484 SDLoc SL(Ld);
12485
12486 assert((!MemVT.isVector() || Ld->getExtensionType() == ISD::NON_EXTLOAD) &&
12487 "unexpected vector extload");
12488
12489 // TODO: Drop only high part of range.
12490 SDValue Ptr = Ld->getBasePtr();
12491 SDValue NewLoad = DAG.getLoad(
12492 AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT: MVT::i32, dl: SL, Chain: Ld->getChain(), Ptr,
12493 Offset: Ld->getOffset(), PtrInfo: Ld->getPointerInfo(), MemVT: MVT::i32, Alignment: Ld->getAlign(),
12494 MMOFlags: Ld->getMemOperand()->getFlags(), AAInfo: Ld->getAAInfo(),
12495 Ranges: nullptr); // Drop ranges
12496
12497 EVT TruncVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
12498 if (MemVT.isFloatingPoint()) {
12499 assert(Ld->getExtensionType() == ISD::NON_EXTLOAD &&
12500 "unexpected fp extload");
12501 TruncVT = MemVT.changeTypeToInteger();
12502 }
12503
12504 SDValue Cvt = NewLoad;
12505 if (Ld->getExtensionType() == ISD::SEXTLOAD) {
12506 Cvt = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SL, VT: MVT::i32, N1: NewLoad,
12507 N2: DAG.getValueType(TruncVT));
12508 } else if (Ld->getExtensionType() == ISD::ZEXTLOAD ||
12509 Ld->getExtensionType() == ISD::NON_EXTLOAD) {
12510 Cvt = DAG.getZeroExtendInReg(Op: NewLoad, DL: SL, VT: TruncVT);
12511 } else {
12512 assert(Ld->getExtensionType() == ISD::EXTLOAD);
12513 }
12514
12515 EVT VT = Ld->getValueType(ResNo: 0);
12516 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
12517
12518 DCI.AddToWorklist(N: Cvt.getNode());
12519
12520 // We may need to handle exotic cases, such as i16->i64 extloads, so insert
12521 // the appropriate extension from the 32-bit load.
12522 Cvt = getLoadExtOrTrunc(DAG, ExtType: Ld->getExtensionType(), Op: Cvt, SL, VT: IntVT);
12523 DCI.AddToWorklist(N: Cvt.getNode());
12524
12525 // Handle conversion back to floating point if necessary.
12526 Cvt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Cvt);
12527
12528 return DAG.getMergeValues(Ops: {Cvt, NewLoad.getValue(R: 1)}, dl: SL);
12529}
12530
12531static bool addressMayBeAccessedAsPrivate(const MachineMemOperand *MMO,
12532 const SIMachineFunctionInfo &Info) {
12533 // TODO: Should check if the address can definitely not access stack.
12534 if (Info.isEntryFunction())
12535 return Info.getUserSGPRInfo().hasFlatScratchInit();
12536 return true;
12537}
12538
12539SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
12540 SDLoc DL(Op);
12541 LoadSDNode *Load = cast<LoadSDNode>(Val&: Op);
12542 ISD::LoadExtType ExtType = Load->getExtensionType();
12543 EVT MemVT = Load->getMemoryVT();
12544 MachineMemOperand *MMO = Load->getMemOperand();
12545
12546 if (ExtType == ISD::NON_EXTLOAD && MemVT.getSizeInBits() < 32) {
12547 if (MemVT == MVT::i16 && isTypeLegal(VT: MVT::i16))
12548 return SDValue();
12549
12550 // FIXME: Copied from PPC
12551 // First, load into 32 bits, then truncate to 1 bit.
12552
12553 SDValue Chain = Load->getChain();
12554 SDValue BasePtr = Load->getBasePtr();
12555
12556 EVT RealMemVT = (MemVT == MVT::i1) ? MVT::i8 : MVT::i16;
12557
12558 SDValue NewLD = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: MVT::i32, Chain, Ptr: BasePtr,
12559 MemVT: RealMemVT, MMO);
12560
12561 if (!MemVT.isVector()) {
12562 SDValue Ops[] = {DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MemVT, Operand: NewLD),
12563 NewLD.getValue(R: 1)};
12564
12565 return DAG.getMergeValues(Ops, dl: DL);
12566 }
12567
12568 SmallVector<SDValue, 3> Elts;
12569 for (unsigned I = 0, N = MemVT.getVectorNumElements(); I != N; ++I) {
12570 SDValue Elt = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: NewLD,
12571 N2: DAG.getConstant(Val: I, DL, VT: MVT::i32));
12572
12573 Elts.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Elt));
12574 }
12575
12576 SDValue Ops[] = {DAG.getBuildVector(VT: MemVT, DL, Ops: Elts), NewLD.getValue(R: 1)};
12577
12578 return DAG.getMergeValues(Ops, dl: DL);
12579 }
12580
12581 if (!MemVT.isVector())
12582 return SDValue();
12583
12584 assert(Op.getValueType().getVectorElementType() == MVT::i32 &&
12585 "Custom lowering for non-i32 vectors hasn't been implemented.");
12586
12587 Align Alignment = Load->getAlign();
12588 unsigned AS = Load->getAddressSpace();
12589 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
12590 AS == AMDGPUAS::FLAT_ADDRESS &&
12591 Alignment.value() < MemVT.getStoreSize() && MemVT.getSizeInBits() > 32) {
12592 return SplitVectorLoad(Op, DAG);
12593 }
12594
12595 MachineFunction &MF = DAG.getMachineFunction();
12596 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
12597 // If there is a possibility that flat instruction access scratch memory
12598 // then we need to use the same legalization rules we use for private.
12599 if (AS == AMDGPUAS::FLAT_ADDRESS &&
12600 !Subtarget->hasMultiDwordFlatScratchAddressing())
12601 AS = addressMayBeAccessedAsPrivate(MMO: Load->getMemOperand(), Info: *MFI)
12602 ? AMDGPUAS::PRIVATE_ADDRESS
12603 : AMDGPUAS::GLOBAL_ADDRESS;
12604
12605 unsigned NumElements = MemVT.getVectorNumElements();
12606
12607 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
12608 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
12609 (AS == AMDGPUAS::GLOBAL_ADDRESS &&
12610 Subtarget->getScalarizeGlobalBehavior() && Load->isSimple() &&
12611 (Load->isInvariant() || isMemOpHasNoClobberedMemOperand(N: Load)))) {
12612 if ((!Op->isDivergent() || AMDGPU::isUniformMMO(MMO)) &&
12613 Alignment >= Align(4) && NumElements < 32) {
12614 if (MemVT.isPow2VectorType() ||
12615 (Subtarget->hasScalarDwordx3Loads() && NumElements == 3))
12616 return SDValue();
12617 return WidenOrSplitVectorLoad(Op, DAG);
12618 }
12619 // Non-uniform loads will be selected to MUBUF instructions, so they
12620 // have the same legalization requirements as global and private
12621 // loads.
12622 //
12623 }
12624 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
12625 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
12626 AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS) {
12627 if (NumElements > 4)
12628 return SplitVectorLoad(Op, DAG);
12629 // v3 loads not supported on SI.
12630 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
12631 return WidenOrSplitVectorLoad(Op, DAG);
12632
12633 // v3 and v4 loads are supported for private and global memory.
12634 return SDValue();
12635 }
12636 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
12637 // Depending on the setting of the private_element_size field in the
12638 // resource descriptor, we can only make private accesses up to a certain
12639 // size.
12640 switch (Subtarget->getMaxPrivateElementSize()) {
12641 case 4: {
12642 auto [Op0, Op1] = scalarizeVectorLoad(LD: Load, DAG);
12643 return DAG.getMergeValues(Ops: {Op0, Op1}, dl: DL);
12644 }
12645 case 8:
12646 if (NumElements > 2)
12647 return SplitVectorLoad(Op, DAG);
12648 return SDValue();
12649 case 16:
12650 // Same as global/flat
12651 if (NumElements > 4)
12652 return SplitVectorLoad(Op, DAG);
12653 // v3 loads not supported on SI.
12654 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
12655 return WidenOrSplitVectorLoad(Op, DAG);
12656
12657 return SDValue();
12658 default:
12659 llvm_unreachable("unsupported private_element_size");
12660 }
12661 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
12662 unsigned Fast = 0;
12663 auto Flags = Load->getMemOperand()->getFlags();
12664 if (allowsMisalignedMemoryAccessesImpl(Size: MemVT.getSizeInBits(), AddrSpace: AS,
12665 Alignment: Load->getAlign(), Flags, IsFast: &Fast) &&
12666 Fast > 1)
12667 return SDValue();
12668
12669 if (MemVT.isVector())
12670 return SplitVectorLoad(Op, DAG);
12671 }
12672
12673 if (!allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
12674 VT: MemVT, MMO: *Load->getMemOperand())) {
12675 auto [Op0, Op1] = expandUnalignedLoad(LD: Load, DAG);
12676 return DAG.getMergeValues(Ops: {Op0, Op1}, dl: DL);
12677 }
12678
12679 return SDValue();
12680}
12681
12682SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
12683 EVT VT = Op.getValueType();
12684 if (VT.getSizeInBits() == 128 || VT.getSizeInBits() == 256 ||
12685 VT.getSizeInBits() == 512)
12686 return splitTernaryVectorOp(Op, DAG);
12687
12688 assert(VT.getSizeInBits() == 64);
12689
12690 SDLoc DL(Op);
12691 SDValue Cond = DAG.getFreeze(V: Op.getOperand(i: 0));
12692
12693 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
12694 SDValue One = DAG.getConstant(Val: 1, DL, VT: MVT::i32);
12695
12696 SDValue LHS = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
12697 SDValue RHS = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 2));
12698
12699 SDValue Lo0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: LHS, N2: Zero);
12700 SDValue Lo1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: RHS, N2: Zero);
12701
12702 SDValue Lo = DAG.getSelect(DL, VT: MVT::i32, Cond, LHS: Lo0, RHS: Lo1);
12703
12704 SDValue Hi0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: LHS, N2: One);
12705 SDValue Hi1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: RHS, N2: One);
12706
12707 SDValue Hi = DAG.getSelect(DL, VT: MVT::i32, Cond, LHS: Hi0, RHS: Hi1);
12708
12709 SDValue Res = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Lo, Hi});
12710 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Res);
12711}
12712
12713// Catch division cases where we can use shortcuts with rcp and rsq
12714// instructions.
12715SDValue SITargetLowering::lowerFastUnsafeFDIV(SDValue Op,
12716 SelectionDAG &DAG) const {
12717 SDLoc SL(Op);
12718 SDValue LHS = Op.getOperand(i: 0);
12719 SDValue RHS = Op.getOperand(i: 1);
12720 EVT VT = Op.getValueType();
12721 const SDNodeFlags Flags = Op->getFlags();
12722
12723 bool AllowInaccurateRcp = Flags.hasApproximateFuncs();
12724
12725 if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
12726 // Without !fpmath accuracy information, we can't do more because we don't
12727 // know exactly whether rcp is accurate enough to meet !fpmath requirement.
12728 // f16 is always accurate enough
12729 if (!AllowInaccurateRcp && VT != MVT::f16 && VT != MVT::bf16)
12730 return SDValue();
12731
12732 if (CLHS->isExactlyValue(V: 1.0)) {
12733 // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
12734 // the CI documentation has a worst case error of 1 ulp.
12735 // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to
12736 // use it as long as we aren't trying to use denormals.
12737 //
12738 // v_rcp_f16 and v_rsq_f16 DO support denormals and 0.51ulp.
12739
12740 // 1.0 / sqrt(x) -> rsq(x)
12741
12742 // XXX - Is afn sufficient to do this for f64? The maximum ULP
12743 // error seems really high at 2^29 ULP.
12744 // 1.0 / x -> rcp(x)
12745 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: RHS);
12746 }
12747
12748 // Same as for 1.0, but expand the sign out of the constant.
12749 if (CLHS->isExactlyValue(V: -1.0)) {
12750 // -1.0 / x -> rcp (fneg x)
12751 SDValue FNegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
12752 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: FNegRHS);
12753 }
12754 }
12755
12756 // For f16 and bf16 require afn or arcp.
12757 // For f32 require afn.
12758 if (!AllowInaccurateRcp &&
12759 ((VT != MVT::f16 && VT != MVT::bf16) || !Flags.hasAllowReciprocal()))
12760 return SDValue();
12761
12762 // Turn into multiply by the reciprocal.
12763 // x / y -> x * (1.0 / y)
12764 SDValue Recip = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: RHS);
12765 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: LHS, N2: Recip, Flags);
12766}
12767
12768SDValue SITargetLowering::lowerFastUnsafeFDIV64(SDValue Op,
12769 SelectionDAG &DAG) const {
12770 SDLoc SL(Op);
12771 SDValue X = Op.getOperand(i: 0);
12772 SDValue Y = Op.getOperand(i: 1);
12773 EVT VT = Op.getValueType();
12774 const SDNodeFlags Flags = Op->getFlags();
12775
12776 bool AllowInaccurateDiv = Flags.hasApproximateFuncs();
12777 if (!AllowInaccurateDiv)
12778 return SDValue();
12779
12780 SDValue NegY = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Y);
12781 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
12782
12783 SDValue R = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: Y);
12784 SDValue Tmp0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: R, N3: One);
12785
12786 R = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp0, N2: R, N3: R);
12787 SDValue Tmp1 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: R, N3: One);
12788 R = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp1, N2: R, N3: R);
12789 SDValue Ret = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: R);
12790 SDValue Tmp2 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: Ret, N3: X);
12791 return DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp2, N2: R, N3: Ret);
12792}
12793
12794static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
12795 EVT VT, SDValue A, SDValue B, SDValue GlueChain,
12796 SDNodeFlags Flags) {
12797 if (GlueChain->getNumValues() <= 1) {
12798 return DAG.getNode(Opcode, DL: SL, VT, N1: A, N2: B, Flags);
12799 }
12800
12801 assert(GlueChain->getNumValues() == 3);
12802
12803 SDVTList VTList = DAG.getVTList(VT1: VT, VT2: MVT::Other, VT3: MVT::Glue);
12804 switch (Opcode) {
12805 default:
12806 llvm_unreachable("no chain equivalent for opcode");
12807 case ISD::FMUL:
12808 Opcode = AMDGPUISD::FMUL_W_CHAIN;
12809 break;
12810 }
12811
12812 return DAG.getNode(Opcode, DL: SL, VTList,
12813 Ops: {GlueChain.getValue(R: 1), A, B, GlueChain.getValue(R: 2)},
12814 Flags);
12815}
12816
12817static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
12818 EVT VT, SDValue A, SDValue B, SDValue C,
12819 SDValue GlueChain, SDNodeFlags Flags) {
12820 if (GlueChain->getNumValues() <= 1) {
12821 return DAG.getNode(Opcode, DL: SL, VT, Ops: {A, B, C}, Flags);
12822 }
12823
12824 assert(GlueChain->getNumValues() == 3);
12825
12826 SDVTList VTList = DAG.getVTList(VT1: VT, VT2: MVT::Other, VT3: MVT::Glue);
12827 switch (Opcode) {
12828 default:
12829 llvm_unreachable("no chain equivalent for opcode");
12830 case ISD::FMA:
12831 Opcode = AMDGPUISD::FMA_W_CHAIN;
12832 break;
12833 }
12834
12835 return DAG.getNode(Opcode, DL: SL, VTList,
12836 Ops: {GlueChain.getValue(R: 1), A, B, C, GlueChain.getValue(R: 2)},
12837 Flags);
12838}
12839
12840SDValue SITargetLowering::LowerFDIV16(SDValue Op, SelectionDAG &DAG) const {
12841 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
12842 return FastLowered;
12843
12844 SDLoc SL(Op);
12845 EVT VT = Op.getValueType();
12846 SDValue LHS = Op.getOperand(i: 0);
12847 SDValue RHS = Op.getOperand(i: 1);
12848
12849 SDValue LHSExt = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: LHS);
12850 SDValue RHSExt = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: RHS);
12851
12852 if (VT == MVT::bf16) {
12853 SDValue ExtDiv =
12854 DAG.getNode(Opcode: ISD::FDIV, DL: SL, VT: MVT::f32, N1: LHSExt, N2: RHSExt, Flags: Op->getFlags());
12855 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::bf16, N1: ExtDiv,
12856 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32));
12857 }
12858
12859 assert(VT == MVT::f16);
12860
12861 // a32.u = opx(V_CVT_F32_F16, a.u); // CVT to F32
12862 // b32.u = opx(V_CVT_F32_F16, b.u); // CVT to F32
12863 // r32.u = opx(V_RCP_F32, b32.u); // rcp = 1 / d
12864 // q32.u = opx(V_MUL_F32, a32.u, r32.u); // q = n * rcp
12865 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
12866 // q32.u = opx(V_MAD_F32, e32.u, r32.u, q32.u); // q = n * rcp
12867 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
12868 // tmp.u = opx(V_MUL_F32, e32.u, r32.u);
12869 // tmp.u = opx(V_AND_B32, tmp.u, 0xff800000)
12870 // q32.u = opx(V_ADD_F32, tmp.u, q32.u);
12871 // q16.u = opx(V_CVT_F16_F32, q32.u);
12872 // q16.u = opx(V_DIV_FIXUP_F16, q16.u, b.u, a.u); // q = touchup(q, d, n)
12873
12874 // We will use ISD::FMA on targets that don't support ISD::FMAD.
12875 unsigned FMADOpCode =
12876 isOperationLegal(Op: ISD::FMAD, VT: MVT::f32) ? ISD::FMAD : ISD::FMA;
12877 SDValue NegRHSExt = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: RHSExt);
12878 SDValue Rcp =
12879 DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: RHSExt, Flags: Op->getFlags());
12880 SDValue Quot =
12881 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: LHSExt, N2: Rcp, Flags: Op->getFlags());
12882 SDValue Err = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: NegRHSExt, N2: Quot, N3: LHSExt,
12883 Flags: Op->getFlags());
12884 Quot = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: Err, N2: Rcp, N3: Quot, Flags: Op->getFlags());
12885 Err = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: NegRHSExt, N2: Quot, N3: LHSExt,
12886 Flags: Op->getFlags());
12887 SDValue Tmp = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: Err, N2: Rcp, Flags: Op->getFlags());
12888 SDValue TmpCast = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Tmp);
12889 TmpCast = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: TmpCast,
12890 N2: DAG.getConstant(Val: 0xff800000, DL: SL, VT: MVT::i32));
12891 Tmp = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: TmpCast);
12892 Quot = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f32, N1: Tmp, N2: Quot, Flags: Op->getFlags());
12893 SDValue RDst = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::f16, N1: Quot,
12894 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32));
12895 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f16, N1: RDst, N2: RHS, N3: LHS,
12896 Flags: Op->getFlags());
12897}
12898
12899// Faster 2.5 ULP division that does not support denormals.
12900SDValue SITargetLowering::lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const {
12901 SDNodeFlags Flags = Op->getFlags();
12902 SDLoc SL(Op);
12903 SDValue LHS = Op.getOperand(i: 1);
12904 SDValue RHS = Op.getOperand(i: 2);
12905
12906 // TODO: The combiner should probably handle elimination of redundant fabs.
12907 SDValue r1 = DAG.SignBitIsZeroFP(Op: RHS)
12908 ? RHS
12909 : DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: MVT::f32, Operand: RHS, Flags);
12910
12911 const APFloat K0Val(0x1p+96f);
12912 const SDValue K0 = DAG.getConstantFP(Val: K0Val, DL: SL, VT: MVT::f32);
12913
12914 const APFloat K1Val(0x1p-32f);
12915 const SDValue K1 = DAG.getConstantFP(Val: K1Val, DL: SL, VT: MVT::f32);
12916
12917 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f32);
12918
12919 EVT SetCCVT =
12920 getSetCCResultType(DL: DAG.getDataLayout(), Ctx&: *DAG.getContext(), VT: MVT::f32);
12921
12922 SDValue r2 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: r1, RHS: K0, Cond: ISD::SETOGT);
12923
12924 SDValue r3 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f32, N1: r2, N2: K1, N3: One, Flags);
12925
12926 r1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: RHS, N2: r3, Flags);
12927
12928 // rcp does not support denormals.
12929 SDValue r0 = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: r1, Flags);
12930
12931 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: LHS, N2: r0, Flags);
12932
12933 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: r3, N2: Mul, Flags);
12934}
12935
12936// Returns immediate value for setting the F32 denorm mode when using the
12937// S_DENORM_MODE instruction.
12938static SDValue getSPDenormModeValue(uint32_t SPDenormMode, SelectionDAG &DAG,
12939 const SIMachineFunctionInfo *Info,
12940 const GCNSubtarget *ST) {
12941 assert(ST->hasDenormModeInst() && "Requires S_DENORM_MODE");
12942 uint32_t DPDenormModeDefault = Info->getMode().fpDenormModeDPValue();
12943 uint32_t Mode = SPDenormMode | (DPDenormModeDefault << 2);
12944 return DAG.getTargetConstant(Val: Mode, DL: SDLoc(), VT: MVT::i32);
12945}
12946
12947SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const {
12948 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
12949 return FastLowered;
12950
12951 // The selection matcher assumes anything with a chain selecting to a
12952 // mayRaiseFPException machine instruction. Since we're introducing a chain
12953 // here, we need to explicitly report nofpexcept for the regular fdiv
12954 // lowering.
12955 SDNodeFlags Flags = Op->getFlags();
12956 Flags.setNoFPExcept(true);
12957
12958 SDLoc SL(Op);
12959 SDValue LHS = Op.getOperand(i: 0);
12960 SDValue RHS = Op.getOperand(i: 1);
12961
12962 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f32);
12963
12964 SDVTList ScaleVT = DAG.getVTList(VT1: MVT::f32, VT2: MVT::i1);
12965
12966 SDValue DenominatorScaled =
12967 DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, Ops: {RHS, RHS, LHS}, Flags);
12968 SDValue NumeratorScaled =
12969 DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, Ops: {LHS, RHS, LHS}, Flags);
12970
12971 // Denominator is scaled to not be denormal, so using rcp is ok.
12972 SDValue ApproxRcp =
12973 DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: DenominatorScaled, Flags);
12974 SDValue NegDivScale0 =
12975 DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: DenominatorScaled, Flags);
12976
12977 using namespace AMDGPU::Hwreg;
12978 const unsigned Denorm32Reg = HwregEncoding::encode(Values: ID_MODE, Values: 4, Values: 2);
12979 const SDValue BitField = DAG.getTargetConstant(Val: Denorm32Reg, DL: SL, VT: MVT::i32);
12980
12981 const MachineFunction &MF = DAG.getMachineFunction();
12982 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
12983 const DenormalMode DenormMode = Info->getMode().FP32Denormals;
12984
12985 const bool PreservesDenormals = DenormMode == DenormalMode::getIEEE();
12986 const bool HasDynamicDenormals =
12987 (DenormMode.Input == DenormalMode::Dynamic) ||
12988 (DenormMode.Output == DenormalMode::Dynamic);
12989
12990 SDValue SavedDenormMode;
12991
12992 if (!PreservesDenormals) {
12993 // Note we can't use the STRICT_FMA/STRICT_FMUL for the non-strict FDIV
12994 // lowering. The chain dependence is insufficient, and we need glue. We do
12995 // not need the glue variants in a strictfp function.
12996
12997 SDVTList BindParamVTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
12998
12999 SDValue Glue = DAG.getEntryNode();
13000 if (HasDynamicDenormals) {
13001 SDNode *GetReg = DAG.getMachineNode(Opcode: AMDGPU::S_GETREG_B32, dl: SL,
13002 VTs: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Glue),
13003 Ops: {BitField, Glue});
13004 SavedDenormMode = SDValue(GetReg, 0);
13005
13006 Glue = DAG.getMergeValues(
13007 Ops: {DAG.getEntryNode(), SDValue(GetReg, 0), SDValue(GetReg, 1)}, dl: SL);
13008 }
13009
13010 SDNode *EnableDenorm;
13011 if (Subtarget->hasDenormModeInst()) {
13012 const SDValue EnableDenormValue =
13013 getSPDenormModeValue(FP_DENORM_FLUSH_NONE, DAG, Info, ST: Subtarget);
13014
13015 EnableDenorm = DAG.getNode(Opcode: AMDGPUISD::DENORM_MODE, DL: SL, VTList: BindParamVTs, N1: Glue,
13016 N2: EnableDenormValue)
13017 .getNode();
13018 } else {
13019 const SDValue EnableDenormValue =
13020 DAG.getConstant(FP_DENORM_FLUSH_NONE, DL: SL, VT: MVT::i32);
13021 EnableDenorm = DAG.getMachineNode(Opcode: AMDGPU::S_SETREG_B32, dl: SL, VTs: BindParamVTs,
13022 Ops: {EnableDenormValue, BitField, Glue});
13023 }
13024
13025 SDValue Ops[3] = {NegDivScale0, SDValue(EnableDenorm, 0),
13026 SDValue(EnableDenorm, 1)};
13027
13028 NegDivScale0 = DAG.getMergeValues(Ops, dl: SL);
13029 }
13030
13031 SDValue Fma0 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0,
13032 B: ApproxRcp, C: One, GlueChain: NegDivScale0, Flags);
13033
13034 SDValue Fma1 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: Fma0, B: ApproxRcp,
13035 C: ApproxRcp, GlueChain: Fma0, Flags);
13036
13037 SDValue Mul = getFPBinOp(DAG, Opcode: ISD::FMUL, SL, VT: MVT::f32, A: NumeratorScaled, B: Fma1,
13038 GlueChain: Fma1, Flags);
13039
13040 SDValue Fma2 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0, B: Mul,
13041 C: NumeratorScaled, GlueChain: Mul, Flags);
13042
13043 SDValue Fma3 =
13044 getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: Fma2, B: Fma1, C: Mul, GlueChain: Fma2, Flags);
13045
13046 SDValue Fma4 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0, B: Fma3,
13047 C: NumeratorScaled, GlueChain: Fma3, Flags);
13048
13049 if (!PreservesDenormals) {
13050 SDNode *DisableDenorm;
13051 if (!HasDynamicDenormals && Subtarget->hasDenormModeInst()) {
13052 const SDValue DisableDenormValue = getSPDenormModeValue(
13053 FP_DENORM_FLUSH_IN_FLUSH_OUT, DAG, Info, ST: Subtarget);
13054
13055 SDVTList BindParamVTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
13056 DisableDenorm =
13057 DAG.getNode(Opcode: AMDGPUISD::DENORM_MODE, DL: SL, VTList: BindParamVTs,
13058 N1: Fma4.getValue(R: 1), N2: DisableDenormValue, N3: Fma4.getValue(R: 2))
13059 .getNode();
13060 } else {
13061 assert(HasDynamicDenormals == (bool)SavedDenormMode);
13062 const SDValue DisableDenormValue =
13063 HasDynamicDenormals
13064 ? SavedDenormMode
13065 : DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, DL: SL, VT: MVT::i32);
13066
13067 DisableDenorm = DAG.getMachineNode(
13068 Opcode: AMDGPU::S_SETREG_B32, dl: SL, VT: MVT::Other,
13069 Ops: {DisableDenormValue, BitField, Fma4.getValue(R: 1), Fma4.getValue(R: 2)});
13070 }
13071
13072 SDValue OutputChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other,
13073 N1: SDValue(DisableDenorm, 0), N2: DAG.getRoot());
13074 DAG.setRoot(OutputChain);
13075 }
13076
13077 SDValue Scale = NumeratorScaled.getValue(R: 1);
13078 SDValue Fmas = DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL: SL, VT: MVT::f32,
13079 Ops: {Fma4, Fma1, Fma3, Scale}, Flags);
13080
13081 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f32, N1: Fmas, N2: RHS, N3: LHS, Flags);
13082}
13083
13084SDValue SITargetLowering::LowerFDIV64(SDValue Op, SelectionDAG &DAG) const {
13085 if (SDValue FastLowered = lowerFastUnsafeFDIV64(Op, DAG))
13086 return FastLowered;
13087
13088 SDLoc SL(Op);
13089 SDValue X = Op.getOperand(i: 0);
13090 SDValue Y = Op.getOperand(i: 1);
13091
13092 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f64);
13093
13094 SDVTList ScaleVT = DAG.getVTList(VT1: MVT::f64, VT2: MVT::i1);
13095
13096 SDValue DivScale0 = DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, N1: Y, N2: Y, N3: X);
13097
13098 SDValue NegDivScale0 = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f64, Operand: DivScale0);
13099
13100 SDValue Rcp = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f64, Operand: DivScale0);
13101
13102 SDValue Fma0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Rcp, N3: One);
13103
13104 SDValue Fma1 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: Rcp, N2: Fma0, N3: Rcp);
13105
13106 SDValue Fma2 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Fma1, N3: One);
13107
13108 SDValue DivScale1 = DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, N1: X, N2: Y, N3: X);
13109
13110 SDValue Fma3 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: Fma1, N2: Fma2, N3: Fma1);
13111 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f64, N1: DivScale1, N2: Fma3);
13112
13113 SDValue Fma4 =
13114 DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Mul, N3: DivScale1);
13115
13116 SDValue Scale;
13117
13118 if (!Subtarget->hasUsableDivScaleConditionOutput()) {
13119 // Workaround a hardware bug on SI where the condition output from div_scale
13120 // is not usable.
13121
13122 const SDValue Hi = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
13123
13124 // Figure out if the scale to use for div_fmas.
13125 SDValue NumBC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: X);
13126 SDValue DenBC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Y);
13127 SDValue Scale0BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: DivScale0);
13128 SDValue Scale1BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: DivScale1);
13129
13130 SDValue NumHi =
13131 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: NumBC, N2: Hi);
13132 SDValue DenHi =
13133 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: DenBC, N2: Hi);
13134
13135 SDValue Scale0Hi =
13136 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Scale0BC, N2: Hi);
13137 SDValue Scale1Hi =
13138 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Scale1BC, N2: Hi);
13139
13140 SDValue CmpDen = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: DenHi, RHS: Scale0Hi, Cond: ISD::SETEQ);
13141 SDValue CmpNum = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: NumHi, RHS: Scale1Hi, Cond: ISD::SETEQ);
13142 Scale = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i1, N1: CmpNum, N2: CmpDen);
13143 } else {
13144 Scale = DivScale1.getValue(R: 1);
13145 }
13146
13147 SDValue Fmas =
13148 DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL: SL, VT: MVT::f64, N1: Fma4, N2: Fma3, N3: Mul, N4: Scale);
13149
13150 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f64, N1: Fmas, N2: Y, N3: X);
13151}
13152
13153SDValue SITargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const {
13154 EVT VT = Op.getValueType();
13155
13156 if (VT == MVT::f32)
13157 return LowerFDIV32(Op, DAG);
13158
13159 if (VT == MVT::f64)
13160 return LowerFDIV64(Op, DAG);
13161
13162 if (VT == MVT::f16 || VT == MVT::bf16)
13163 return LowerFDIV16(Op, DAG);
13164
13165 llvm_unreachable("Unexpected type for fdiv");
13166}
13167
13168SDValue SITargetLowering::LowerFFREXP(SDValue Op, SelectionDAG &DAG) const {
13169 SDLoc dl(Op);
13170 SDValue Val = Op.getOperand(i: 0);
13171 EVT VT = Val.getValueType();
13172 EVT ResultExpVT = Op->getValueType(ResNo: 1);
13173 EVT InstrExpVT = VT == MVT::f16 ? MVT::i16 : MVT::i32;
13174
13175 SDValue Mant = DAG.getNode(
13176 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT,
13177 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_frexp_mant, DL: dl, VT: MVT::i32), N2: Val);
13178
13179 SDValue Exp = DAG.getNode(
13180 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: InstrExpVT,
13181 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_frexp_exp, DL: dl, VT: MVT::i32), N2: Val);
13182
13183 if (Subtarget->hasFractBug()) {
13184 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Val);
13185 SDValue Inf =
13186 DAG.getConstantFP(Val: APFloat::getInf(Sem: VT.getFltSemantics()), DL: dl, VT);
13187
13188 SDValue IsFinite = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: Fabs, RHS: Inf, Cond: ISD::SETOLT);
13189 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: InstrExpVT);
13190 Exp = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: InstrExpVT, N1: IsFinite, N2: Exp, N3: Zero);
13191 Mant = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: IsFinite, N2: Mant, N3: Val);
13192 }
13193
13194 SDValue CastExp = DAG.getSExtOrTrunc(Op: Exp, DL: dl, VT: ResultExpVT);
13195 return DAG.getMergeValues(Ops: {Mant, CastExp}, dl);
13196}
13197
13198SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
13199 SDLoc DL(Op);
13200 StoreSDNode *Store = cast<StoreSDNode>(Val&: Op);
13201 EVT VT = Store->getMemoryVT();
13202
13203 if (VT == MVT::i1) {
13204 return DAG.getTruncStore(
13205 Chain: Store->getChain(), dl: DL,
13206 Val: DAG.getSExtOrTrunc(Op: Store->getValue(), DL, VT: MVT::i32),
13207 Ptr: Store->getBasePtr(), SVT: MVT::i1, MMO: Store->getMemOperand());
13208 }
13209
13210 assert(VT.isVector() &&
13211 Store->getValue().getValueType().getScalarType() == MVT::i32);
13212
13213 unsigned AS = Store->getAddressSpace();
13214 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
13215 AS == AMDGPUAS::FLAT_ADDRESS &&
13216 Store->getAlign().value() < VT.getStoreSize() &&
13217 VT.getSizeInBits() > 32) {
13218 return SplitVectorStore(Op, DAG);
13219 }
13220
13221 MachineFunction &MF = DAG.getMachineFunction();
13222 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
13223 // If there is a possibility that flat instruction access scratch memory
13224 // then we need to use the same legalization rules we use for private.
13225 if (AS == AMDGPUAS::FLAT_ADDRESS &&
13226 !Subtarget->hasMultiDwordFlatScratchAddressing())
13227 AS = addressMayBeAccessedAsPrivate(MMO: Store->getMemOperand(), Info: *MFI)
13228 ? AMDGPUAS::PRIVATE_ADDRESS
13229 : AMDGPUAS::GLOBAL_ADDRESS;
13230
13231 unsigned NumElements = VT.getVectorNumElements();
13232 if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS) {
13233 if (NumElements > 4)
13234 return SplitVectorStore(Op, DAG);
13235 // v3 stores not supported on SI.
13236 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
13237 return SplitVectorStore(Op, DAG);
13238
13239 if (!allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
13240 VT, MMO: *Store->getMemOperand()))
13241 return expandUnalignedStore(ST: Store, DAG);
13242
13243 return SDValue();
13244 }
13245 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
13246 switch (Subtarget->getMaxPrivateElementSize()) {
13247 case 4:
13248 return scalarizeVectorStore(ST: Store, DAG);
13249 case 8:
13250 if (NumElements > 2)
13251 return SplitVectorStore(Op, DAG);
13252 return SDValue();
13253 case 16:
13254 if (NumElements > 4 ||
13255 (NumElements == 3 && !Subtarget->hasFlatScratchEnabled()))
13256 return SplitVectorStore(Op, DAG);
13257 return SDValue();
13258 default:
13259 llvm_unreachable("unsupported private_element_size");
13260 }
13261 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
13262 unsigned Fast = 0;
13263 auto Flags = Store->getMemOperand()->getFlags();
13264 if (allowsMisalignedMemoryAccessesImpl(Size: VT.getSizeInBits(), AddrSpace: AS,
13265 Alignment: Store->getAlign(), Flags, IsFast: &Fast) &&
13266 Fast > 1)
13267 return SDValue();
13268
13269 if (VT.isVector())
13270 return SplitVectorStore(Op, DAG);
13271
13272 return expandUnalignedStore(ST: Store, DAG);
13273 }
13274
13275 // Probably an invalid store. If so we'll end up emitting a selection error.
13276 return SDValue();
13277}
13278
13279// Avoid the full correct expansion for f32 sqrt when promoting from f16.
13280SDValue SITargetLowering::lowerFSQRTF16(SDValue Op, SelectionDAG &DAG) const {
13281 SDLoc SL(Op);
13282 assert(!Subtarget->has16BitInsts());
13283 SDNodeFlags Flags = Op->getFlags();
13284 SDValue Ext =
13285 DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op.getOperand(i: 0), Flags);
13286
13287 SDValue SqrtID = DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL: SL, VT: MVT::i32);
13288 SDValue Sqrt =
13289 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::f32, N1: SqrtID, N2: Ext, Flags);
13290
13291 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::f16, N1: Sqrt,
13292 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
13293}
13294
13295SDValue SITargetLowering::lowerFSQRTF32(SDValue Op, SelectionDAG &DAG) const {
13296 SDLoc DL(Op);
13297 SDNodeFlags Flags = Op->getFlags();
13298 MVT VT = Op.getValueType().getSimpleVT();
13299 const SDValue X = Op.getOperand(i: 0);
13300
13301 if (allowApproxFunc(DAG, Flags)) {
13302 // Instruction is 1ulp but ignores denormals.
13303 return DAG.getNode(
13304 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT,
13305 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL, VT: MVT::i32), N2: X, Flags);
13306 }
13307
13308 SDValue ScaleThreshold = DAG.getConstantFP(Val: 0x1.0p-96f, DL, VT);
13309 SDValue NeedScale = DAG.getSetCC(DL, VT: MVT::i1, LHS: X, RHS: ScaleThreshold, Cond: ISD::SETOLT);
13310
13311 SDValue ScaleUpFactor = DAG.getConstantFP(Val: 0x1.0p+32f, DL, VT);
13312
13313 SDValue ScaledX = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: X, N2: ScaleUpFactor, Flags);
13314
13315 SDValue SqrtX =
13316 DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: NeedScale, N2: ScaledX, N3: X, Flags);
13317
13318 SDValue SqrtS;
13319 if (needsDenormHandlingF32(DAG, Src: X, Flags)) {
13320 SDValue SqrtID =
13321 DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL, VT: MVT::i32);
13322 SqrtS = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT, N1: SqrtID, N2: SqrtX, Flags);
13323
13324 SDValue SqrtSAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: SqrtS);
13325 SDValue SqrtSNextDownInt =
13326 DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SqrtSAsInt,
13327 N2: DAG.getAllOnesConstant(DL, VT: MVT::i32));
13328 SDValue SqrtSNextDown = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: SqrtSNextDownInt);
13329
13330 SDValue NegSqrtSNextDown =
13331 DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtSNextDown, Flags);
13332
13333 SDValue SqrtVP =
13334 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtSNextDown, N2: SqrtS, N3: SqrtX, Flags);
13335
13336 SDValue SqrtSNextUpInt = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SqrtSAsInt,
13337 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
13338 SDValue SqrtSNextUp = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: SqrtSNextUpInt);
13339
13340 SDValue NegSqrtSNextUp = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtSNextUp, Flags);
13341 SDValue SqrtVS =
13342 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtSNextUp, N2: SqrtS, N3: SqrtX, Flags);
13343
13344 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL, VT);
13345 SDValue SqrtVPLE0 = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtVP, RHS: Zero, Cond: ISD::SETOLE);
13346
13347 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: SqrtVPLE0, N2: SqrtSNextDown, N3: SqrtS,
13348 Flags);
13349
13350 SDValue SqrtVPVSGT0 = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtVS, RHS: Zero, Cond: ISD::SETOGT);
13351 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: SqrtVPVSGT0, N2: SqrtSNextUp, N3: SqrtS,
13352 Flags);
13353 } else {
13354 SDValue SqrtR = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: SqrtX, Flags);
13355
13356 SqrtS = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtX, N2: SqrtR, Flags);
13357
13358 SDValue Half = DAG.getConstantFP(Val: 0.5f, DL, VT);
13359 SDValue SqrtH = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtR, N2: Half, Flags);
13360 SDValue NegSqrtH = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtH, Flags);
13361
13362 SDValue SqrtE = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtH, N2: SqrtS, N3: Half, Flags);
13363 SqrtH = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtH, N2: SqrtE, N3: SqrtH, Flags);
13364 SqrtS = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtS, N2: SqrtE, N3: SqrtS, Flags);
13365
13366 SDValue NegSqrtS = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtS, Flags);
13367 SDValue SqrtD =
13368 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtS, N2: SqrtS, N3: SqrtX, Flags);
13369 SqrtS = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtD, N2: SqrtH, N3: SqrtS, Flags);
13370 }
13371
13372 SDValue ScaleDownFactor = DAG.getConstantFP(Val: 0x1.0p-16f, DL, VT);
13373
13374 SDValue ScaledDown =
13375 DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtS, N2: ScaleDownFactor, Flags);
13376
13377 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: NeedScale, N2: ScaledDown, N3: SqrtS, Flags);
13378 SDValue IsZeroOrInf =
13379 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, N1: SqrtX,
13380 N2: DAG.getTargetConstant(Val: fcZero | fcPosInf, DL, VT: MVT::i32));
13381
13382 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsZeroOrInf, N2: SqrtX, N3: SqrtS, Flags);
13383}
13384
13385SDValue SITargetLowering::lowerFSQRTF64(SDValue Op, SelectionDAG &DAG) const {
13386 // For double type, the SQRT and RSQ instructions don't have required
13387 // precision, we apply Goldschmidt's algorithm to improve the result:
13388 //
13389 // y0 = rsq(x)
13390 // g0 = x * y0
13391 // h0 = 0.5 * y0
13392 //
13393 // r0 = 0.5 - h0 * g0
13394 // g1 = g0 * r0 + g0
13395 // h1 = h0 * r0 + h0
13396 //
13397 // r1 = 0.5 - h1 * g1 => d0 = x - g1 * g1
13398 // g2 = g1 * r1 + g1 g2 = d0 * h1 + g1
13399 // h2 = h1 * r1 + h1
13400 //
13401 // r2 = 0.5 - h2 * g2 => d1 = x - g2 * g2
13402 // g3 = g2 * r2 + g2 g3 = d1 * h1 + g2
13403 //
13404 // sqrt(x) = g3
13405
13406 SDNodeFlags Flags = Op->getFlags();
13407
13408 SDLoc DL(Op);
13409
13410 SDValue X = Op.getOperand(i: 0);
13411 SDValue ZeroInt = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13412
13413 SDValue SqrtX = X;
13414 SDValue Scaling;
13415 if (!Flags.hasApproximateFuncs()) {
13416 SDValue ScaleConstant = DAG.getConstantFP(Val: 0x1.0p-767, DL, VT: MVT::f64);
13417 Scaling = DAG.getSetCC(DL, VT: MVT::i1, LHS: X, RHS: ScaleConstant, Cond: ISD::SETOLT);
13418
13419 // Scale up input if it is too small.
13420 SDValue ScaleUpFactor = DAG.getConstant(Val: 256, DL, VT: MVT::i32);
13421 SDValue ScaleUp =
13422 DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::i32, N1: Scaling, N2: ScaleUpFactor, N3: ZeroInt);
13423 SqrtX = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: X, N2: ScaleUp, Flags);
13424 }
13425
13426 SDValue SqrtY = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT: MVT::f64, Operand: SqrtX);
13427
13428 SDValue SqrtS0 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: SqrtX, N2: SqrtY);
13429
13430 SDValue Half = DAG.getConstantFP(Val: 0.5, DL, VT: MVT::f64);
13431 SDValue SqrtH0 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: SqrtY, N2: Half);
13432
13433 SDValue NegSqrtH0 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtH0);
13434 SDValue SqrtR0 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtH0, N2: SqrtS0, N3: Half);
13435
13436 SDValue SqrtH1 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtH0, N2: SqrtR0, N3: SqrtH0);
13437
13438 SDValue SqrtS1 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtS0, N2: SqrtR0, N3: SqrtS0);
13439
13440 SDValue NegSqrtS1 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtS1);
13441 SDValue SqrtD0 =
13442 DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtS1, N2: SqrtS1, N3: SqrtX);
13443
13444 SDValue SqrtS2 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtD0, N2: SqrtH1, N3: SqrtS1);
13445
13446 SDValue SqrtRet = SqrtS2;
13447 if (!Flags.hasApproximateFuncs()) {
13448 SDValue NegSqrtS2 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtS2);
13449 SDValue SqrtD1 =
13450 DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtS2, N2: SqrtS2, N3: SqrtX);
13451
13452 SqrtRet = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtD1, N2: SqrtH1, N3: SqrtS2);
13453
13454 SDValue ScaleDownFactor = DAG.getSignedConstant(Val: -128, DL, VT: MVT::i32);
13455 SDValue ScaleDown = DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::i32, N1: Scaling,
13456 N2: ScaleDownFactor, N3: ZeroInt);
13457 SqrtRet = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: SqrtRet, N2: ScaleDown, Flags);
13458 }
13459
13460 // TODO: Check for DAZ and expand to subnormals
13461
13462 SDValue IsZeroOrInf;
13463 if (Flags.hasNoInfs()) {
13464 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: MVT::f64);
13465 IsZeroOrInf = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtX, RHS: Zero, Cond: ISD::SETOEQ);
13466 } else {
13467 IsZeroOrInf =
13468 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, N1: SqrtX,
13469 N2: DAG.getTargetConstant(Val: fcZero | fcPosInf, DL, VT: MVT::i32));
13470 }
13471
13472 // If x is +INF, +0, or -0, use its original value
13473 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::f64, N1: IsZeroOrInf, N2: SqrtX, N3: SqrtRet,
13474 Flags);
13475}
13476
13477SDValue SITargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
13478 SDLoc DL(Op);
13479 EVT VT = Op.getValueType();
13480 SDValue Arg = Op.getOperand(i: 0);
13481 SDValue TrigVal;
13482
13483 // Propagate fast-math flags so that the multiply we introduce can be folded
13484 // if Arg is already the result of a multiply by constant.
13485 auto Flags = Op->getFlags();
13486
13487 // AMDGPUISD nodes of vector type must be unrolled here since
13488 // they will not be expanded elsewhere.
13489 auto UnrollIfVec = [&DAG](SDValue V) -> SDValue {
13490 if (!V.getValueType().isVector())
13491 return V;
13492
13493 return DAG.UnrollVectorOp(N: cast<SDNode>(Val&: V));
13494 };
13495
13496 SDValue OneOver2Pi = DAG.getConstantFP(Val: 0.5 * numbers::inv_pi, DL, VT);
13497
13498 if (Subtarget->hasTrigReducedRange()) {
13499 SDValue MulVal = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Arg, N2: OneOver2Pi, Flags);
13500 TrigVal = UnrollIfVec(DAG.getNode(Opcode: AMDGPUISD::FRACT, DL, VT, Operand: MulVal, Flags));
13501 } else {
13502 TrigVal = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Arg, N2: OneOver2Pi, Flags);
13503 }
13504
13505 switch (Op.getOpcode()) {
13506 case ISD::FCOS:
13507 TrigVal = DAG.getNode(Opcode: AMDGPUISD::COS_HW, DL: SDLoc(Op), VT, Operand: TrigVal, Flags);
13508 break;
13509 case ISD::FSIN:
13510 TrigVal = DAG.getNode(Opcode: AMDGPUISD::SIN_HW, DL: SDLoc(Op), VT, Operand: TrigVal, Flags);
13511 break;
13512 default:
13513 llvm_unreachable("Wrong trig opcode");
13514 }
13515
13516 return UnrollIfVec(TrigVal);
13517}
13518
13519SDValue SITargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op,
13520 SelectionDAG &DAG) const {
13521 AtomicSDNode *AtomicNode = cast<AtomicSDNode>(Val&: Op);
13522 assert(AtomicNode->isCompareAndSwap());
13523 unsigned AS = AtomicNode->getAddressSpace();
13524
13525 // No custom lowering required for local address space
13526 if (!AMDGPU::isFlatGlobalAddrSpace(AS))
13527 return Op;
13528
13529 // Non-local address space requires custom lowering for atomic compare
13530 // and swap; cmp and swap should be in a v2i32 or v2i64 in case of _X2
13531 SDLoc DL(Op);
13532 SDValue ChainIn = Op.getOperand(i: 0);
13533 SDValue Addr = Op.getOperand(i: 1);
13534 SDValue Old = Op.getOperand(i: 2);
13535 SDValue New = Op.getOperand(i: 3);
13536 EVT VT = Op.getValueType();
13537 MVT SimpleVT = VT.getSimpleVT();
13538 MVT VecType = MVT::getVectorVT(VT: SimpleVT, NumElements: 2);
13539
13540 SDValue NewOld = DAG.getBuildVector(VT: VecType, DL, Ops: {New, Old});
13541 SDValue Ops[] = {ChainIn, Addr, NewOld};
13542
13543 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::ATOMIC_CMP_SWAP, dl: DL,
13544 VTList: Op->getVTList(), Ops, MemVT: VT,
13545 MMO: AtomicNode->getMemOperand());
13546}
13547
13548//===----------------------------------------------------------------------===//
13549// Custom DAG optimizations
13550//===----------------------------------------------------------------------===//
13551
13552SDValue
13553SITargetLowering::performUCharToFloatCombine(SDNode *N,
13554 DAGCombinerInfo &DCI) const {
13555 EVT VT = N->getValueType(ResNo: 0);
13556 EVT ScalarVT = VT.getScalarType();
13557 if (ScalarVT != MVT::f32 && ScalarVT != MVT::f16)
13558 return SDValue();
13559
13560 SelectionDAG &DAG = DCI.DAG;
13561 SDLoc DL(N);
13562
13563 SDValue Src = N->getOperand(Num: 0);
13564 EVT SrcVT = Src.getValueType();
13565
13566 // TODO: We could try to match extracting the higher bytes, which would be
13567 // easier if i8 vectors weren't promoted to i32 vectors, particularly after
13568 // types are legalized. v4i8 -> v4f32 is probably the only case to worry
13569 // about in practice.
13570 if (DCI.isAfterLegalizeDAG() && SrcVT == MVT::i32) {
13571 if (DAG.MaskedValueIsZero(Op: Src, Mask: APInt::getHighBitsSet(numBits: 32, hiBitsSet: 24))) {
13572 SDValue Cvt = DAG.getNode(Opcode: AMDGPUISD::CVT_F32_UBYTE0, DL, VT: MVT::f32, Operand: Src);
13573 DCI.AddToWorklist(N: Cvt.getNode());
13574
13575 // For the f16 case, fold to a cast to f32 and then cast back to f16.
13576 if (ScalarVT != MVT::f32) {
13577 Cvt = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Cvt,
13578 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
13579 }
13580 return Cvt;
13581 }
13582 }
13583
13584 return SDValue();
13585}
13586
13587SDValue SITargetLowering::performFCopySignCombine(SDNode *N,
13588 DAGCombinerInfo &DCI) const {
13589 SDValue MagnitudeOp = N->getOperand(Num: 0);
13590 SDValue SignOp = N->getOperand(Num: 1);
13591
13592 // The generic combine for fcopysign + fp cast is too conservative with
13593 // vectors, and also gets confused by the splitting we will perform here, so
13594 // peek through FP casts.
13595 if (SignOp.getOpcode() == ISD::FP_EXTEND ||
13596 SignOp.getOpcode() == ISD::FP_ROUND)
13597 SignOp = SignOp.getOperand(i: 0);
13598
13599 SelectionDAG &DAG = DCI.DAG;
13600 SDLoc DL(N);
13601 EVT SignVT = SignOp.getValueType();
13602
13603 // f64 fcopysign is really an f32 copysign on the high bits, so replace the
13604 // lower half with a copy.
13605 // fcopysign f64:x, _:y -> x.lo32, (fcopysign (f32 x.hi32), _:y)
13606 EVT MagVT = MagnitudeOp.getValueType();
13607
13608 unsigned NumElts = MagVT.isVector() ? MagVT.getVectorNumElements() : 1;
13609
13610 if (MagVT.getScalarType() == MVT::f64) {
13611 EVT F32VT = MagVT.isVector()
13612 ? EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: 2 * NumElts)
13613 : MVT::v2f32;
13614
13615 SDValue MagAsVector = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: F32VT, Operand: MagnitudeOp);
13616
13617 SmallVector<SDValue, 8> NewElts;
13618 for (unsigned I = 0; I != NumElts; ++I) {
13619 SDValue MagLo =
13620 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: MagAsVector,
13621 N2: DAG.getConstant(Val: 2 * I, DL, VT: MVT::i32));
13622 SDValue MagHi =
13623 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: MagAsVector,
13624 N2: DAG.getConstant(Val: 2 * I + 1, DL, VT: MVT::i32));
13625
13626 SDValue SignOpElt =
13627 MagVT.isVector()
13628 ? DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: SignVT.getScalarType(),
13629 N1: SignOp, N2: DAG.getConstant(Val: I, DL, VT: MVT::i32))
13630 : SignOp;
13631
13632 SDValue HiOp =
13633 DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: MVT::f32, N1: MagHi, N2: SignOpElt);
13634
13635 SDValue Vector =
13636 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MVT::v2f32, N1: MagLo, N2: HiOp);
13637
13638 SDValue NewElt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: Vector);
13639 NewElts.push_back(Elt: NewElt);
13640 }
13641
13642 if (NewElts.size() == 1)
13643 return NewElts[0];
13644
13645 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MagVT, Ops: NewElts);
13646 }
13647
13648 if (SignVT.getScalarType() != MVT::f64)
13649 return SDValue();
13650
13651 // Reduce width of sign operand, we only need the highest bit.
13652 //
13653 // fcopysign f64:x, f64:y ->
13654 // fcopysign f64:x, (extract_vector_elt (bitcast f64:y to v2f32), 1)
13655 // TODO: In some cases it might make sense to go all the way to f16.
13656
13657 EVT F32VT = MagVT.isVector()
13658 ? EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: 2 * NumElts)
13659 : MVT::v2f32;
13660
13661 SDValue SignAsVector = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: F32VT, Operand: SignOp);
13662
13663 SmallVector<SDValue, 8> F32Signs;
13664 for (unsigned I = 0; I != NumElts; ++I) {
13665 // Take sign from odd elements of cast vector
13666 SDValue SignAsF32 =
13667 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: SignAsVector,
13668 N2: DAG.getConstant(Val: 2 * I + 1, DL, VT: MVT::i32));
13669 F32Signs.push_back(Elt: SignAsF32);
13670 }
13671
13672 SDValue NewSign =
13673 NumElts == 1
13674 ? F32Signs.back()
13675 : DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL,
13676 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: NumElts),
13677 Ops: F32Signs);
13678
13679 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
13680 N2: NewSign);
13681}
13682
13683// (shl (add x, c1), c2) -> add (shl x, c2), (shl c1, c2)
13684// (shl (or x, c1), c2) -> add (shl x, c2), (shl c1, c2) iff x and c1 share no
13685// bits
13686
13687// This is a variant of
13688// (mul (add x, c1), c2) -> add (mul x, c2), (mul c1, c2),
13689//
13690// The normal DAG combiner will do this, but only if the add has one use since
13691// that would increase the number of instructions.
13692//
13693// This prevents us from seeing a constant offset that can be folded into a
13694// memory instruction's addressing mode. If we know the resulting add offset of
13695// a pointer can be folded into an addressing offset, we can replace the pointer
13696// operand with the add of new constant offset. This eliminates one of the uses,
13697// and may allow the remaining use to also be simplified.
13698//
13699SDValue SITargetLowering::performSHLPtrCombine(SDNode *N, unsigned AddrSpace,
13700 EVT MemVT,
13701 DAGCombinerInfo &DCI) const {
13702 SDValue N0 = N->getOperand(Num: 0);
13703 SDValue N1 = N->getOperand(Num: 1);
13704
13705 // We only do this to handle cases where it's profitable when there are
13706 // multiple uses of the add, so defer to the standard combine.
13707 if ((!N0->isAnyAdd() && N0.getOpcode() != ISD::OR) || N0->hasOneUse())
13708 return SDValue();
13709
13710 const ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(Val&: N1);
13711 if (!CN1)
13712 return SDValue();
13713
13714 const ConstantSDNode *CAdd = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
13715 if (!CAdd)
13716 return SDValue();
13717
13718 SelectionDAG &DAG = DCI.DAG;
13719
13720 if (N0->getOpcode() == ISD::OR &&
13721 !DAG.haveNoCommonBitsSet(A: N0.getOperand(i: 0), B: N0.getOperand(i: 1)))
13722 return SDValue();
13723
13724 // If the resulting offset is too large, we can't fold it into the
13725 // addressing mode offset.
13726 APInt Offset = CAdd->getAPIntValue() << CN1->getAPIntValue();
13727 Type *Ty = MemVT.getTypeForEVT(Context&: *DCI.DAG.getContext());
13728
13729 AddrMode AM;
13730 AM.HasBaseReg = true;
13731 AM.BaseOffs = Offset.getSExtValue();
13732 if (!isLegalAddressingMode(DL: DCI.DAG.getDataLayout(), AM, Ty, AS: AddrSpace))
13733 return SDValue();
13734
13735 SDLoc SL(N);
13736 EVT VT = N->getValueType(ResNo: 0);
13737
13738 SDValue ShlX = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT, N1: N0.getOperand(i: 0), N2: N1);
13739 SDValue COffset = DAG.getConstant(Val: Offset, DL: SL, VT);
13740
13741 SDNodeFlags Flags;
13742 Flags.setNoUnsignedWrap(
13743 N->getFlags().hasNoUnsignedWrap() &&
13744 (N0.getOpcode() == ISD::OR || N0->getFlags().hasNoUnsignedWrap()));
13745
13746 // Use ISD::ADD even if the original operation was ISD::PTRADD, since we can't
13747 // be sure that the new left operand is a proper base pointer.
13748 return DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ShlX, N2: COffset, Flags);
13749}
13750
13751/// MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset
13752/// by the chain and intrinsic ID. Theoretically we would also need to check the
13753/// specific intrinsic, but they all place the pointer operand first.
13754static unsigned getBasePtrIndex(const MemSDNode *N) {
13755 switch (N->getOpcode()) {
13756 case ISD::STORE:
13757 case ISD::INTRINSIC_W_CHAIN:
13758 case ISD::INTRINSIC_VOID:
13759 return 2;
13760 default:
13761 return 1;
13762 }
13763}
13764
13765SDValue SITargetLowering::performMemSDNodeCombine(MemSDNode *N,
13766 DAGCombinerInfo &DCI) const {
13767 SelectionDAG &DAG = DCI.DAG;
13768
13769 unsigned PtrIdx = getBasePtrIndex(N);
13770 SDValue Ptr = N->getOperand(Num: PtrIdx);
13771
13772 // TODO: We could also do this for multiplies.
13773 if (Ptr.getOpcode() == ISD::SHL) {
13774 SDValue NewPtr = performSHLPtrCombine(N: Ptr.getNode(), AddrSpace: N->getAddressSpace(),
13775 MemVT: N->getMemoryVT(), DCI);
13776 if (NewPtr) {
13777 SmallVector<SDValue, 8> NewOps(N->ops());
13778
13779 NewOps[PtrIdx] = NewPtr;
13780 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
13781 }
13782 }
13783
13784 return SDValue();
13785}
13786
13787static bool bitOpWithConstantIsReducible(unsigned Opc, uint32_t Val) {
13788 return (Opc == ISD::AND && (Val == 0 || Val == 0xffffffff)) ||
13789 (Opc == ISD::OR && (Val == 0xffffffff || Val == 0)) ||
13790 (Opc == ISD::XOR && Val == 0);
13791}
13792
13793// Break up 64-bit bit operation of a constant into two 32-bit and/or/xor. This
13794// will typically happen anyway for a VALU 64-bit and. This exposes other 32-bit
13795// integer combine opportunities since most 64-bit operations are decomposed
13796// this way. TODO: We won't want this for SALU especially if it is an inline
13797// immediate.
13798SDValue SITargetLowering::splitBinaryBitConstantOp(
13799 DAGCombinerInfo &DCI, const SDLoc &SL, unsigned Opc, SDValue LHS,
13800 const ConstantSDNode *CRHS) const {
13801 uint64_t Val = CRHS->getZExtValue();
13802 uint32_t ValLo = Lo_32(Value: Val);
13803 uint32_t ValHi = Hi_32(Value: Val);
13804 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
13805
13806 if ((bitOpWithConstantIsReducible(Opc, Val: ValLo) ||
13807 bitOpWithConstantIsReducible(Opc, Val: ValHi)) ||
13808 (CRHS->hasOneUse() && !TII->isInlineConstant(Imm: CRHS->getAPIntValue()))) {
13809 // We have 64-bit scalar and/or/xor, but do not have vector forms.
13810 if (Subtarget->has64BitLiterals() && CRHS->hasOneUse() &&
13811 !CRHS->user_begin()->isDivergent())
13812 return SDValue();
13813
13814 // If we need to materialize a 64-bit immediate, it will be split up later
13815 // anyway. Avoid creating the harder to understand 64-bit immediate
13816 // materialization.
13817 return splitBinaryBitConstantOpImpl(DCI, SL, Opc, LHS, ValLo, ValHi);
13818 }
13819
13820 return SDValue();
13821}
13822
13823bool llvm::isBoolSGPR(SDValue V) {
13824 if (V.getValueType() != MVT::i1)
13825 return false;
13826 switch (V.getOpcode()) {
13827 default:
13828 break;
13829 case ISD::SETCC:
13830 case ISD::IS_FPCLASS:
13831 case AMDGPUISD::FP_CLASS:
13832 return true;
13833 case ISD::AND:
13834 case ISD::OR:
13835 case ISD::XOR:
13836 return isBoolSGPR(V: V.getOperand(i: 0)) && isBoolSGPR(V: V.getOperand(i: 1));
13837 case ISD::SADDO:
13838 case ISD::UADDO:
13839 case ISD::SSUBO:
13840 case ISD::USUBO:
13841 case ISD::SMULO:
13842 case ISD::UMULO:
13843 return V.getResNo() == 1;
13844 case ISD::INTRINSIC_WO_CHAIN: {
13845 unsigned IntrinsicID = V.getConstantOperandVal(i: 0);
13846 switch (IntrinsicID) {
13847 case Intrinsic::amdgcn_is_shared:
13848 case Intrinsic::amdgcn_is_private:
13849 return true;
13850 default:
13851 return false;
13852 }
13853
13854 return false;
13855 }
13856 }
13857 return false;
13858}
13859
13860// If a constant has all zeroes or all ones within each byte return it.
13861// Otherwise return 0.
13862static uint32_t getConstantPermuteMask(uint32_t C) {
13863 // 0xff for any zero byte in the mask
13864 uint32_t ZeroByteMask = 0;
13865 if (!(C & 0x000000ff))
13866 ZeroByteMask |= 0x000000ff;
13867 if (!(C & 0x0000ff00))
13868 ZeroByteMask |= 0x0000ff00;
13869 if (!(C & 0x00ff0000))
13870 ZeroByteMask |= 0x00ff0000;
13871 if (!(C & 0xff000000))
13872 ZeroByteMask |= 0xff000000;
13873 uint32_t NonZeroByteMask = ~ZeroByteMask; // 0xff for any non-zero byte
13874 if ((NonZeroByteMask & C) != NonZeroByteMask)
13875 return 0; // Partial bytes selected.
13876 return C;
13877}
13878
13879// Check if a node selects whole bytes from its operand 0 starting at a byte
13880// boundary while masking the rest. Returns select mask as in the v_perm_b32
13881// or -1 if not succeeded.
13882// Note byte select encoding:
13883// value 0-3 selects corresponding source byte;
13884// value 0xc selects zero;
13885// value 0xff selects 0xff.
13886static uint32_t getPermuteMask(SDValue V) {
13887 assert(V.getValueSizeInBits() == 32);
13888
13889 if (V.getNumOperands() != 2)
13890 return ~0;
13891
13892 ConstantSDNode *N1 = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1));
13893 if (!N1)
13894 return ~0;
13895
13896 uint32_t C = N1->getZExtValue();
13897
13898 switch (V.getOpcode()) {
13899 default:
13900 break;
13901 case ISD::AND:
13902 if (uint32_t ConstMask = getConstantPermuteMask(C))
13903 return (0x03020100 & ConstMask) | (0x0c0c0c0c & ~ConstMask);
13904 break;
13905
13906 case ISD::OR:
13907 if (uint32_t ConstMask = getConstantPermuteMask(C))
13908 return (0x03020100 & ~ConstMask) | ConstMask;
13909 break;
13910
13911 case ISD::SHL:
13912 if (C % 8)
13913 return ~0;
13914
13915 return uint32_t((0x030201000c0c0c0cull << C) >> 32);
13916
13917 case ISD::SRL:
13918 if (C % 8)
13919 return ~0;
13920
13921 return uint32_t(0x0c0c0c0c03020100ull >> C);
13922 }
13923
13924 return ~0;
13925}
13926
13927SDValue SITargetLowering::performAndCombine(SDNode *N,
13928 DAGCombinerInfo &DCI) const {
13929 if (DCI.isBeforeLegalize())
13930 return SDValue();
13931
13932 SelectionDAG &DAG = DCI.DAG;
13933 EVT VT = N->getValueType(ResNo: 0);
13934 SDValue LHS = N->getOperand(Num: 0);
13935 SDValue RHS = N->getOperand(Num: 1);
13936
13937 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
13938 if (VT == MVT::i64 && CRHS) {
13939 if (SDValue Split =
13940 splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::AND, LHS, CRHS))
13941 return Split;
13942 }
13943
13944 if (CRHS && VT == MVT::i32) {
13945 // and (srl x, c), mask => shl (bfe x, nb + c, mask >> nb), nb
13946 // nb = number of trailing zeroes in mask
13947 // It can be optimized out using SDWA for GFX8+ in the SDWA peephole pass,
13948 // given that we are selecting 8 or 16 bit fields starting at byte boundary.
13949 uint64_t Mask = CRHS->getZExtValue();
13950 unsigned Bits = llvm::popcount(Value: Mask);
13951 if (getSubtarget()->hasSDWA() && LHS->getOpcode() == ISD::SRL &&
13952 (Bits == 8 || Bits == 16) && isShiftedMask_64(Value: Mask) && !(Mask & 1)) {
13953 if (auto *CShift = dyn_cast<ConstantSDNode>(Val: LHS->getOperand(Num: 1))) {
13954 unsigned Shift = CShift->getZExtValue();
13955 unsigned NB = CRHS->getAPIntValue().countr_zero();
13956 unsigned Offset = NB + Shift;
13957 if ((Offset & (Bits - 1)) == 0) { // Starts at a byte or word boundary.
13958 SDLoc SL(N);
13959 SDValue BFE =
13960 DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT: MVT::i32, N1: LHS->getOperand(Num: 0),
13961 N2: DAG.getConstant(Val: Offset, DL: SL, VT: MVT::i32),
13962 N3: DAG.getConstant(Val: Bits, DL: SL, VT: MVT::i32));
13963 EVT NarrowVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: Bits);
13964 SDValue Ext = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT, N1: BFE,
13965 N2: DAG.getValueType(NarrowVT));
13966 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(LHS), VT, N1: Ext,
13967 N2: DAG.getConstant(Val: NB, DL: SDLoc(CRHS), VT: MVT::i32));
13968 return Shl;
13969 }
13970 }
13971 }
13972
13973 // and (perm x, y, c1), c2 -> perm x, y, permute_mask(c1, c2)
13974 if (LHS.hasOneUse() && LHS.getOpcode() == AMDGPUISD::PERM &&
13975 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2))) {
13976 uint32_t Sel = getConstantPermuteMask(C: Mask);
13977 if (!Sel)
13978 return SDValue();
13979
13980 // Select 0xc for all zero bytes
13981 Sel = (LHS.getConstantOperandVal(i: 2) & Sel) | (~Sel & 0x0c0c0c0c);
13982 SDLoc DL(N);
13983 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
13984 N2: LHS.getOperand(i: 1), N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
13985 }
13986 }
13987
13988 // (and (fcmp ord x, x), (fcmp une (fabs x), inf)) ->
13989 // fp_class x, ~(s_nan | q_nan | n_infinity | p_infinity)
13990 if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == ISD::SETCC) {
13991 ISD::CondCode LCC = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
13992 ISD::CondCode RCC = cast<CondCodeSDNode>(Val: RHS.getOperand(i: 2))->get();
13993
13994 SDValue X = LHS.getOperand(i: 0);
13995 SDValue Y = RHS.getOperand(i: 0);
13996 if (Y.getOpcode() != ISD::FABS || Y.getOperand(i: 0) != X ||
13997 !isTypeLegal(VT: X.getValueType()))
13998 return SDValue();
13999
14000 if (LCC == ISD::SETO) {
14001 if (X != LHS.getOperand(i: 1))
14002 return SDValue();
14003
14004 if (RCC == ISD::SETUNE) {
14005 const ConstantFPSDNode *C1 =
14006 dyn_cast<ConstantFPSDNode>(Val: RHS.getOperand(i: 1));
14007 if (!C1 || !C1->isInfinity() || C1->isNegative())
14008 return SDValue();
14009
14010 const uint32_t Mask = SIInstrFlags::N_NORMAL |
14011 SIInstrFlags::N_SUBNORMAL | SIInstrFlags::N_ZERO |
14012 SIInstrFlags::P_ZERO | SIInstrFlags::P_SUBNORMAL |
14013 SIInstrFlags::P_NORMAL;
14014
14015 static_assert(
14016 ((~(SIInstrFlags::S_NAN | SIInstrFlags::Q_NAN |
14017 SIInstrFlags::N_INFINITY | SIInstrFlags::P_INFINITY)) &
14018 0x3ff) == Mask,
14019 "mask not equal");
14020
14021 SDLoc DL(N);
14022 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: X,
14023 N2: DAG.getConstant(Val: Mask, DL, VT: MVT::i32));
14024 }
14025 }
14026 }
14027
14028 if (RHS.getOpcode() == ISD::SETCC && LHS.getOpcode() == AMDGPUISD::FP_CLASS)
14029 std::swap(a&: LHS, b&: RHS);
14030
14031 if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == AMDGPUISD::FP_CLASS &&
14032 RHS.hasOneUse()) {
14033 ISD::CondCode LCC = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
14034 // and (fcmp seto), (fp_class x, mask) -> fp_class x, mask & ~(p_nan |
14035 // n_nan) and (fcmp setuo), (fp_class x, mask) -> fp_class x, mask & (p_nan
14036 // | n_nan)
14037 const ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i: 1));
14038 if ((LCC == ISD::SETO || LCC == ISD::SETUO) && Mask &&
14039 (RHS.getOperand(i: 0) == LHS.getOperand(i: 0) &&
14040 LHS.getOperand(i: 0) == LHS.getOperand(i: 1))) {
14041 const unsigned OrdMask = SIInstrFlags::S_NAN | SIInstrFlags::Q_NAN;
14042 unsigned NewMask = LCC == ISD::SETO ? Mask->getZExtValue() & ~OrdMask
14043 : Mask->getZExtValue() & OrdMask;
14044
14045 SDLoc DL(N);
14046 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: RHS.getOperand(i: 0),
14047 N2: DAG.getConstant(Val: NewMask, DL, VT: MVT::i32));
14048 }
14049 }
14050
14051 if (VT == MVT::i32 && (RHS.getOpcode() == ISD::SIGN_EXTEND ||
14052 LHS.getOpcode() == ISD::SIGN_EXTEND)) {
14053 // and x, (sext cc from i1) => select cc, x, 0
14054 if (RHS.getOpcode() != ISD::SIGN_EXTEND)
14055 std::swap(a&: LHS, b&: RHS);
14056 if (isBoolSGPR(V: RHS.getOperand(i: 0)))
14057 return DAG.getSelect(DL: SDLoc(N), VT: MVT::i32, Cond: RHS.getOperand(i: 0), LHS,
14058 RHS: DAG.getConstant(Val: 0, DL: SDLoc(N), VT: MVT::i32));
14059 }
14060
14061 // and (op x, c1), (op y, c2) -> perm x, y, permute_mask(c1, c2)
14062 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
14063 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() &&
14064 N->isDivergent() && TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
14065 uint32_t LHSMask = getPermuteMask(V: LHS);
14066 uint32_t RHSMask = getPermuteMask(V: RHS);
14067 if (LHSMask != ~0u && RHSMask != ~0u) {
14068 // Canonicalize the expression in an attempt to have fewer unique masks
14069 // and therefore fewer registers used to hold the masks.
14070 if (LHSMask > RHSMask) {
14071 std::swap(a&: LHSMask, b&: RHSMask);
14072 std::swap(a&: LHS, b&: RHS);
14073 }
14074
14075 // Select 0xc for each lane used from source operand. Zero has 0xc mask
14076 // set, 0xff have 0xff in the mask, actual lanes are in the 0-3 range.
14077 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
14078 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
14079
14080 // Check of we need to combine values from two sources within a byte.
14081 if (!(LHSUsedLanes & RHSUsedLanes) &&
14082 // If we select high and lower word keep it for SDWA.
14083 // TODO: teach SDWA to work with v_perm_b32 and remove the check.
14084 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
14085 // Each byte in each mask is either selector mask 0-3, or has higher
14086 // bits set in either of masks, which can be 0xff for 0xff or 0x0c for
14087 // zero. If 0x0c is in either mask it shall always be 0x0c. Otherwise
14088 // mask which is not 0xff wins. By anding both masks we have a correct
14089 // result except that 0x0c shall be corrected to give 0x0c only.
14090 uint32_t Mask = LHSMask & RHSMask;
14091 for (unsigned I = 0; I < 32; I += 8) {
14092 uint32_t ByteSel = 0xff << I;
14093 if ((LHSMask & ByteSel) == 0x0c || (RHSMask & ByteSel) == 0x0c)
14094 Mask &= (0x0c << I) & 0xffffffff;
14095 }
14096
14097 // Add 4 to each active LHS lane. It will not affect any existing 0xff
14098 // or 0x0c.
14099 uint32_t Sel = Mask | (LHSUsedLanes & 0x04040404);
14100 SDLoc DL(N);
14101
14102 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
14103 N2: RHS.getOperand(i: 0),
14104 N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
14105 }
14106 }
14107 }
14108
14109 return SDValue();
14110}
14111
14112// A key component of v_perm is a mapping between byte position of the src
14113// operands, and the byte position of the dest. To provide such, we need: 1. the
14114// node that provides x byte of the dest of the OR, and 2. the byte of the node
14115// used to provide that x byte. calculateByteProvider finds which node provides
14116// a certain byte of the dest of the OR, and calculateSrcByte takes that node,
14117// and finds an ultimate src and byte position For example: The supported
14118// LoadCombine pattern for vector loads is as follows
14119// t1
14120// or
14121// / \
14122// t2 t3
14123// zext shl
14124// | | \
14125// t4 t5 16
14126// or anyext
14127// / \ |
14128// t6 t7 t8
14129// srl shl or
14130// / | / \ / \
14131// t9 t10 t11 t12 t13 t14
14132// trunc* 8 trunc* 8 and and
14133// | | / | | \
14134// t15 t16 t17 t18 t19 t20
14135// trunc* 255 srl -256
14136// | / \
14137// t15 t15 16
14138//
14139// *In this example, the truncs are from i32->i16
14140//
14141// calculateByteProvider would find t6, t7, t13, and t14 for bytes 0-3
14142// respectively. calculateSrcByte would find (given node) -> ultimate src &
14143// byteposition: t6 -> t15 & 1, t7 -> t16 & 0, t13 -> t15 & 0, t14 -> t15 & 3.
14144// After finding the mapping, we can combine the tree into vperm t15, t16,
14145// 0x05000407
14146
14147// Find the source and byte position from a node.
14148// \p DestByte is the byte position of the dest of the or that the src
14149// ultimately provides. \p SrcIndex is the byte of the src that maps to this
14150// dest of the or byte. \p Depth tracks how many recursive iterations we have
14151// performed.
14152static const std::optional<ByteProvider<SDValue>>
14153calculateSrcByte(const SDValue Op, uint64_t DestByte, uint64_t SrcIndex = 0,
14154 unsigned Depth = 0) {
14155 // We may need to recursively traverse a series of SRLs
14156 if (Depth >= 6)
14157 return std::nullopt;
14158
14159 if (Op.getValueSizeInBits() < 8)
14160 return std::nullopt;
14161
14162 if (Op.getValueType().isVector())
14163 return ByteProvider<SDValue>::getSrc(Val: Op, ByteOffset: DestByte, VectorOffset: SrcIndex);
14164
14165 switch (Op->getOpcode()) {
14166 case ISD::TRUNCATE: {
14167 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex, Depth: Depth + 1);
14168 }
14169
14170 case ISD::ANY_EXTEND:
14171 case ISD::SIGN_EXTEND:
14172 case ISD::ZERO_EXTEND:
14173 case ISD::SIGN_EXTEND_INREG: {
14174 SDValue NarrowOp = Op->getOperand(Num: 0);
14175 auto NarrowVT = NarrowOp.getValueType();
14176 if (Op->getOpcode() == ISD::SIGN_EXTEND_INREG) {
14177 auto *VTSign = cast<VTSDNode>(Val: Op->getOperand(Num: 1));
14178 NarrowVT = VTSign->getVT();
14179 }
14180 if (!NarrowVT.isByteSized())
14181 return std::nullopt;
14182 uint64_t NarrowByteWidth = NarrowVT.getStoreSize();
14183
14184 if (SrcIndex >= NarrowByteWidth)
14185 return std::nullopt;
14186 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex, Depth: Depth + 1);
14187 }
14188
14189 case ISD::SRA:
14190 case ISD::SRL: {
14191 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
14192 if (!ShiftOp)
14193 return std::nullopt;
14194
14195 uint64_t BitShift = ShiftOp->getZExtValue();
14196
14197 if (BitShift % 8 != 0)
14198 return std::nullopt;
14199
14200 SrcIndex += BitShift / 8;
14201
14202 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex, Depth: Depth + 1);
14203 }
14204
14205 default: {
14206 return ByteProvider<SDValue>::getSrc(Val: Op, ByteOffset: DestByte, VectorOffset: SrcIndex);
14207 }
14208 }
14209 llvm_unreachable("fully handled switch");
14210}
14211
14212// For a byte position in the result of an Or, traverse the tree and find the
14213// node (and the byte of the node) which ultimately provides this {Or,
14214// BytePosition}. \p Op is the operand we are currently examining. \p Index is
14215// the byte position of the Op that corresponds with the originally requested
14216// byte of the Or \p Depth tracks how many recursive iterations we have
14217// performed. \p StartingIndex is the originally requested byte of the Or
14218static const std::optional<ByteProvider<SDValue>>
14219calculateByteProvider(const SDValue &Op, unsigned Index, unsigned Depth,
14220 unsigned StartingIndex = 0) {
14221 // Finding Src tree of RHS of or typically requires at least 1 additional
14222 // depth
14223 if (Depth > 6)
14224 return std::nullopt;
14225
14226 unsigned BitWidth = Op.getScalarValueSizeInBits();
14227 if (BitWidth % 8 != 0)
14228 return std::nullopt;
14229 if (Index > BitWidth / 8 - 1)
14230 return std::nullopt;
14231
14232 bool IsVec = Op.getValueType().isVector();
14233 switch (Op.getOpcode()) {
14234 case ISD::OR: {
14235 if (IsVec)
14236 return std::nullopt;
14237
14238 auto RHS = calculateByteProvider(Op: Op.getOperand(i: 1), Index, Depth: Depth + 1,
14239 StartingIndex);
14240 if (!RHS)
14241 return std::nullopt;
14242 auto LHS = calculateByteProvider(Op: Op.getOperand(i: 0), Index, Depth: Depth + 1,
14243 StartingIndex);
14244 if (!LHS)
14245 return std::nullopt;
14246 // A well formed Or will have two ByteProviders for each byte, one of which
14247 // is constant zero
14248 if (!LHS->isConstantZero() && !RHS->isConstantZero())
14249 return std::nullopt;
14250 if (!LHS || LHS->isConstantZero())
14251 return RHS;
14252 if (!RHS || RHS->isConstantZero())
14253 return LHS;
14254 return std::nullopt;
14255 }
14256
14257 case ISD::AND: {
14258 if (IsVec)
14259 return std::nullopt;
14260
14261 auto *BitMaskOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
14262 if (!BitMaskOp)
14263 return std::nullopt;
14264
14265 uint32_t BitMask = BitMaskOp->getZExtValue();
14266 // Bits we expect for our StartingIndex
14267 uint32_t IndexMask = 0xFF << (Index * 8);
14268
14269 if ((IndexMask & BitMask) != IndexMask) {
14270 // If the result of the and partially provides the byte, then it
14271 // is not well formatted
14272 if (IndexMask & BitMask)
14273 return std::nullopt;
14274 return ByteProvider<SDValue>::getConstantZero();
14275 }
14276
14277 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte: StartingIndex, SrcIndex: Index);
14278 }
14279
14280 case ISD::FSHR: {
14281 if (IsVec)
14282 return std::nullopt;
14283
14284 // fshr(X,Y,Z): (X << (BW - (Z % BW))) | (Y >> (Z % BW))
14285 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 2));
14286 if (!ShiftOp || Op.getValueType().isVector())
14287 return std::nullopt;
14288
14289 uint64_t BitsProvided = Op.getValueSizeInBits();
14290 if (BitsProvided % 8 != 0)
14291 return std::nullopt;
14292
14293 uint64_t BitShift = ShiftOp->getAPIntValue().urem(RHS: BitsProvided);
14294 if (BitShift % 8)
14295 return std::nullopt;
14296
14297 uint64_t ConcatSizeInBytes = BitsProvided / 4;
14298 uint64_t ByteShift = BitShift / 8;
14299
14300 uint64_t NewIndex = (Index + ByteShift) % ConcatSizeInBytes;
14301 uint64_t BytesProvided = BitsProvided / 8;
14302 SDValue NextOp = Op.getOperand(i: NewIndex >= BytesProvided ? 0 : 1);
14303 NewIndex %= BytesProvided;
14304 return calculateByteProvider(Op: NextOp, Index: NewIndex, Depth: Depth + 1, StartingIndex);
14305 }
14306
14307 case ISD::SRA:
14308 case ISD::SRL: {
14309 if (IsVec)
14310 return std::nullopt;
14311
14312 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
14313 if (!ShiftOp)
14314 return std::nullopt;
14315
14316 uint64_t BitShift = ShiftOp->getZExtValue();
14317 if (BitShift % 8)
14318 return std::nullopt;
14319
14320 auto BitsProvided = Op.getScalarValueSizeInBits();
14321 if (BitsProvided % 8 != 0)
14322 return std::nullopt;
14323
14324 uint64_t BytesProvided = BitsProvided / 8;
14325 uint64_t ByteShift = BitShift / 8;
14326 // The dest of shift will have good [0 : (BytesProvided - ByteShift)] bytes.
14327 // If the byte we are trying to provide (as tracked by index) falls in this
14328 // range, then the SRL provides the byte. The byte of interest of the src of
14329 // the SRL is Index + ByteShift
14330 return BytesProvided - ByteShift > Index
14331 ? calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte: StartingIndex,
14332 SrcIndex: Index + ByteShift)
14333 : ByteProvider<SDValue>::getConstantZero();
14334 }
14335
14336 case ISD::SHL: {
14337 if (IsVec)
14338 return std::nullopt;
14339
14340 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
14341 if (!ShiftOp)
14342 return std::nullopt;
14343
14344 uint64_t BitShift = ShiftOp->getZExtValue();
14345 if (BitShift % 8 != 0)
14346 return std::nullopt;
14347 uint64_t ByteShift = BitShift / 8;
14348
14349 // If we are shifting by an amount greater than (or equal to)
14350 // the index we are trying to provide, then it provides 0s. If not,
14351 // then this bytes are not definitively 0s, and the corresponding byte
14352 // of interest is Index - ByteShift of the src
14353 return Index < ByteShift
14354 ? ByteProvider<SDValue>::getConstantZero()
14355 : calculateByteProvider(Op: Op.getOperand(i: 0), Index: Index - ByteShift,
14356 Depth: Depth + 1, StartingIndex);
14357 }
14358 case ISD::ANY_EXTEND:
14359 case ISD::SIGN_EXTEND:
14360 case ISD::ZERO_EXTEND:
14361 case ISD::SIGN_EXTEND_INREG:
14362 case ISD::AssertZext:
14363 case ISD::AssertSext: {
14364 if (IsVec)
14365 return std::nullopt;
14366
14367 SDValue NarrowOp = Op->getOperand(Num: 0);
14368 unsigned NarrowBitWidth = NarrowOp.getValueSizeInBits();
14369 if (Op->getOpcode() == ISD::SIGN_EXTEND_INREG ||
14370 Op->getOpcode() == ISD::AssertZext ||
14371 Op->getOpcode() == ISD::AssertSext) {
14372 auto *VTSign = cast<VTSDNode>(Val: Op->getOperand(Num: 1));
14373 NarrowBitWidth = VTSign->getVT().getSizeInBits();
14374 }
14375 if (NarrowBitWidth % 8 != 0)
14376 return std::nullopt;
14377 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
14378
14379 if (Index >= NarrowByteWidth)
14380 return Op.getOpcode() == ISD::ZERO_EXTEND
14381 ? std::optional<ByteProvider<SDValue>>(
14382 ByteProvider<SDValue>::getConstantZero())
14383 : std::nullopt;
14384 return calculateByteProvider(Op: NarrowOp, Index, Depth: Depth + 1, StartingIndex);
14385 }
14386
14387 case ISD::TRUNCATE: {
14388 if (IsVec)
14389 return std::nullopt;
14390
14391 uint64_t NarrowByteWidth = BitWidth / 8;
14392
14393 if (NarrowByteWidth >= Index) {
14394 return calculateByteProvider(Op: Op.getOperand(i: 0), Index, Depth: Depth + 1,
14395 StartingIndex);
14396 }
14397
14398 return std::nullopt;
14399 }
14400
14401 case ISD::CopyFromReg: {
14402 if (BitWidth / 8 > Index)
14403 return calculateSrcByte(Op, DestByte: StartingIndex, SrcIndex: Index);
14404
14405 return std::nullopt;
14406 }
14407
14408 case ISD::LOAD: {
14409 auto *L = cast<LoadSDNode>(Val: Op.getNode());
14410
14411 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
14412 if (NarrowBitWidth % 8 != 0)
14413 return std::nullopt;
14414 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
14415
14416 // If the width of the load does not reach byte we are trying to provide for
14417 // and it is not a ZEXTLOAD, then the load does not provide for the byte in
14418 // question
14419 if (Index >= NarrowByteWidth) {
14420 return L->getExtensionType() == ISD::ZEXTLOAD
14421 ? std::optional<ByteProvider<SDValue>>(
14422 ByteProvider<SDValue>::getConstantZero())
14423 : std::nullopt;
14424 }
14425
14426 if (NarrowByteWidth > Index) {
14427 return calculateSrcByte(Op, DestByte: StartingIndex, SrcIndex: Index);
14428 }
14429
14430 return std::nullopt;
14431 }
14432
14433 case ISD::BSWAP: {
14434 if (IsVec)
14435 return std::nullopt;
14436
14437 return calculateByteProvider(Op: Op->getOperand(Num: 0), Index: BitWidth / 8 - Index - 1,
14438 Depth: Depth + 1, StartingIndex);
14439 }
14440
14441 case ISD::EXTRACT_VECTOR_ELT: {
14442 auto *IdxOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
14443 if (!IdxOp)
14444 return std::nullopt;
14445 auto VecIdx = IdxOp->getZExtValue();
14446 auto ScalarSize = Op.getScalarValueSizeInBits();
14447 if (ScalarSize < 32)
14448 Index = ScalarSize == 8 ? VecIdx : VecIdx * 2 + Index;
14449 return calculateSrcByte(Op: ScalarSize >= 32 ? Op : Op.getOperand(i: 0),
14450 DestByte: StartingIndex, SrcIndex: Index);
14451 }
14452
14453 case AMDGPUISD::PERM: {
14454 if (IsVec)
14455 return std::nullopt;
14456
14457 auto *PermMask = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 2));
14458 if (!PermMask)
14459 return std::nullopt;
14460
14461 auto IdxMask =
14462 (PermMask->getZExtValue() & (0xFF << (Index * 8))) >> (Index * 8);
14463 if (IdxMask > 0x07 && IdxMask != 0x0c)
14464 return std::nullopt;
14465
14466 auto NextOp = Op.getOperand(i: IdxMask > 0x03 ? 0 : 1);
14467 auto NextIndex = IdxMask > 0x03 ? IdxMask % 4 : IdxMask;
14468
14469 return IdxMask != 0x0c ? calculateSrcByte(Op: NextOp, DestByte: StartingIndex, SrcIndex: NextIndex)
14470 : ByteProvider<SDValue>(
14471 ByteProvider<SDValue>::getConstantZero());
14472 }
14473
14474 default: {
14475 return std::nullopt;
14476 }
14477 }
14478
14479 llvm_unreachable("fully handled switch");
14480}
14481
14482// Returns true if the Operand is a scalar and is 16 bits
14483static bool isExtendedFrom16Bits(SDValue &Operand) {
14484
14485 switch (Operand.getOpcode()) {
14486 case ISD::ANY_EXTEND:
14487 case ISD::SIGN_EXTEND:
14488 case ISD::ZERO_EXTEND: {
14489 auto OpVT = Operand.getOperand(i: 0).getValueType();
14490 return !OpVT.isVector() && OpVT.getSizeInBits() == 16;
14491 }
14492 case ISD::LOAD: {
14493 LoadSDNode *L = cast<LoadSDNode>(Val: Operand.getNode());
14494 auto ExtType = cast<LoadSDNode>(Val: L)->getExtensionType();
14495 if (ExtType == ISD::ZEXTLOAD || ExtType == ISD::SEXTLOAD ||
14496 ExtType == ISD::EXTLOAD) {
14497 auto MemVT = L->getMemoryVT();
14498 return !MemVT.isVector() && MemVT.getSizeInBits() == 16;
14499 }
14500 return L->getMemoryVT().getSizeInBits() == 16;
14501 }
14502 default:
14503 return false;
14504 }
14505}
14506
14507// Returns true if the mask matches consecutive bytes, and the first byte
14508// begins at a power of 2 byte offset from 0th byte
14509static bool addresses16Bits(int Mask) {
14510 int Low8 = Mask & 0xff;
14511 int Hi8 = (Mask & 0xff00) >> 8;
14512
14513 assert(Low8 < 8 && Hi8 < 8);
14514 // Are the bytes contiguous in the order of increasing addresses.
14515 bool IsConsecutive = (Hi8 - Low8 == 1);
14516 // Is the first byte at location that is aligned for 16 bit instructions.
14517 // A counter example is taking 2 consecutive bytes starting at the 8th bit.
14518 // In this case, we still need code to extract the 16 bit operand, so it
14519 // is better to use i8 v_perm
14520 bool Is16Aligned = !(Low8 % 2);
14521
14522 return IsConsecutive && Is16Aligned;
14523}
14524
14525// Do not lower into v_perm if the operands are actually 16 bit
14526// and the selected bits (based on PermMask) correspond with two
14527// easily addressable 16 bit operands.
14528static bool hasNon16BitAccesses(uint64_t PermMask, SDValue &Op,
14529 SDValue &OtherOp) {
14530 int Low16 = PermMask & 0xffff;
14531 int Hi16 = (PermMask & 0xffff0000) >> 16;
14532
14533 auto TempOp = peekThroughBitcasts(V: Op);
14534 auto TempOtherOp = peekThroughBitcasts(V: OtherOp);
14535
14536 auto OpIs16Bit =
14537 TempOtherOp.getValueSizeInBits() == 16 || isExtendedFrom16Bits(Operand&: TempOp);
14538 if (!OpIs16Bit)
14539 return true;
14540
14541 auto OtherOpIs16Bit = TempOtherOp.getValueSizeInBits() == 16 ||
14542 isExtendedFrom16Bits(Operand&: TempOtherOp);
14543 if (!OtherOpIs16Bit)
14544 return true;
14545
14546 // Do we cleanly address both
14547 return !addresses16Bits(Mask: Low16) || !addresses16Bits(Mask: Hi16);
14548}
14549
14550static SDValue getDWordFromOffset(SelectionDAG &DAG, SDLoc SL, SDValue Src,
14551 unsigned DWordOffset) {
14552 SDValue Ret;
14553
14554 auto TypeSize = Src.getValueSizeInBits().getFixedValue();
14555 // ByteProvider must be at least 8 bits
14556 assert(Src.getValueSizeInBits().isKnownMultipleOf(8));
14557
14558 if (TypeSize <= 32)
14559 return DAG.getBitcastedAnyExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32);
14560
14561 if (Src.getValueType().isVector()) {
14562 auto ScalarTySize = Src.getScalarValueSizeInBits();
14563 auto ScalarTy = Src.getValueType().getScalarType();
14564 if (ScalarTySize == 32) {
14565 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Src,
14566 N2: DAG.getConstant(Val: DWordOffset, DL: SL, VT: MVT::i32));
14567 }
14568 if (ScalarTySize > 32) {
14569 Ret = DAG.getNode(
14570 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ScalarTy, N1: Src,
14571 N2: DAG.getConstant(Val: DWordOffset / (ScalarTySize / 32), DL: SL, VT: MVT::i32));
14572 auto ShiftVal = 32 * (DWordOffset % (ScalarTySize / 32));
14573 if (ShiftVal)
14574 Ret = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: Ret.getValueType(), N1: Ret,
14575 N2: DAG.getConstant(Val: ShiftVal, DL: SL, VT: MVT::i32));
14576 return DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
14577 }
14578
14579 assert(ScalarTySize < 32);
14580 auto NumElements = TypeSize / ScalarTySize;
14581 auto Trunc32Elements = (ScalarTySize * NumElements) / 32;
14582 auto NormalizedTrunc = Trunc32Elements * 32 / ScalarTySize;
14583 auto NumElementsIn32 = 32 / ScalarTySize;
14584 auto NumAvailElements = DWordOffset < Trunc32Elements
14585 ? NumElementsIn32
14586 : NumElements - NormalizedTrunc;
14587
14588 SmallVector<SDValue, 4> VecSrcs;
14589 DAG.ExtractVectorElements(Op: Src, Args&: VecSrcs, Start: DWordOffset * NumElementsIn32,
14590 Count: NumAvailElements);
14591
14592 Ret = DAG.getBuildVector(
14593 VT: MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ScalarTySize), NumElements: NumAvailElements), DL: SL,
14594 Ops: VecSrcs);
14595 return Ret = DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
14596 }
14597
14598 /// Scalar Type
14599 auto ShiftVal = 32 * DWordOffset;
14600 Ret = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: Src.getValueType(), N1: Src,
14601 N2: DAG.getConstant(Val: ShiftVal, DL: SL, VT: MVT::i32));
14602 return DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
14603}
14604
14605static SDValue matchPERM(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
14606 SelectionDAG &DAG = DCI.DAG;
14607 [[maybe_unused]] EVT VT = N->getValueType(ResNo: 0);
14608 SmallVector<ByteProvider<SDValue>, 8> PermNodes;
14609
14610 // VT is known to be MVT::i32, so we need to provide 4 bytes.
14611 assert(VT == MVT::i32);
14612 for (int i = 0; i < 4; i++) {
14613 // Find the ByteProvider that provides the ith byte of the result of OR
14614 std::optional<ByteProvider<SDValue>> P =
14615 calculateByteProvider(Op: SDValue(N, 0), Index: i, Depth: 0, /*StartingIndex = */ i);
14616 // TODO support constantZero
14617 if (!P || P->isConstantZero())
14618 return SDValue();
14619
14620 PermNodes.push_back(Elt: *P);
14621 }
14622 if (PermNodes.size() != 4)
14623 return SDValue();
14624
14625 std::pair<unsigned, unsigned> FirstSrc(0, PermNodes[0].SrcOffset / 4);
14626 std::optional<std::pair<unsigned, unsigned>> SecondSrc;
14627 uint64_t PermMask = 0x00000000;
14628 for (size_t i = 0; i < PermNodes.size(); i++) {
14629 auto PermOp = PermNodes[i];
14630 // Since the mask is applied to Src1:Src2, Src1 bytes must be offset
14631 // by sizeof(Src2) = 4
14632 int SrcByteAdjust = 4;
14633
14634 // If the Src uses a byte from a different DWORD, then it corresponds
14635 // with a difference source
14636 if (!PermOp.hasSameSrc(Other: PermNodes[FirstSrc.first]) ||
14637 ((PermOp.SrcOffset / 4) != FirstSrc.second)) {
14638 if (SecondSrc)
14639 if (!PermOp.hasSameSrc(Other: PermNodes[SecondSrc->first]) ||
14640 ((PermOp.SrcOffset / 4) != SecondSrc->second))
14641 return SDValue();
14642
14643 // Set the index of the second distinct Src node
14644 SecondSrc = {i, PermNodes[i].SrcOffset / 4};
14645 assert(!(PermNodes[SecondSrc->first].Src->getValueSizeInBits() % 8));
14646 SrcByteAdjust = 0;
14647 }
14648 assert((PermOp.SrcOffset % 4) + SrcByteAdjust < 8);
14649 assert(!DAG.getDataLayout().isBigEndian());
14650 PermMask |= ((PermOp.SrcOffset % 4) + SrcByteAdjust) << (i * 8);
14651 }
14652 SDLoc DL(N);
14653 SDValue Op = *PermNodes[FirstSrc.first].Src;
14654 Op = getDWordFromOffset(DAG, SL: DL, Src: Op, DWordOffset: FirstSrc.second);
14655 assert(Op.getValueSizeInBits() == 32);
14656
14657 // Check that we are not just extracting the bytes in order from an op
14658 if (!SecondSrc) {
14659 int Low16 = PermMask & 0xffff;
14660 int Hi16 = (PermMask & 0xffff0000) >> 16;
14661
14662 bool WellFormedLow = (Low16 == 0x0504) || (Low16 == 0x0100);
14663 bool WellFormedHi = (Hi16 == 0x0706) || (Hi16 == 0x0302);
14664
14665 // The perm op would really just produce Op. So combine into Op
14666 if (WellFormedLow && WellFormedHi)
14667 return DAG.getBitcast(VT: MVT::getIntegerVT(BitWidth: 32), V: Op);
14668 }
14669
14670 SDValue OtherOp = SecondSrc ? *PermNodes[SecondSrc->first].Src : Op;
14671
14672 if (SecondSrc) {
14673 OtherOp = getDWordFromOffset(DAG, SL: DL, Src: OtherOp, DWordOffset: SecondSrc->second);
14674 assert(OtherOp.getValueSizeInBits() == 32);
14675 }
14676
14677 // Check that we haven't just recreated the same FSHR node.
14678 if (N->getOpcode() == ISD::FSHR &&
14679 (N->getOperand(Num: 0) == Op || N->getOperand(Num: 0) == OtherOp) &&
14680 (N->getOperand(Num: 1) == Op || N->getOperand(Num: 1) == OtherOp))
14681 return SDValue();
14682
14683 if (hasNon16BitAccesses(PermMask, Op, OtherOp)) {
14684
14685 assert(Op.getValueType().isByteSized() &&
14686 OtherOp.getValueType().isByteSized());
14687
14688 // If the ultimate src is less than 32 bits, then we will only be
14689 // using bytes 0: Op.getValueSizeInBytes() - 1 in the or.
14690 // CalculateByteProvider would not have returned Op as source if we
14691 // used a byte that is outside its ValueType. Thus, we are free to
14692 // ANY_EXTEND as the extended bits are dont-cares.
14693 Op = DAG.getBitcastedAnyExtOrTrunc(Op, DL, VT: MVT::i32);
14694 OtherOp = DAG.getBitcastedAnyExtOrTrunc(Op: OtherOp, DL, VT: MVT::i32);
14695
14696 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: Op, N2: OtherOp,
14697 N3: DAG.getConstant(Val: PermMask, DL, VT: MVT::i32));
14698 }
14699 return SDValue();
14700}
14701
14702SDValue SITargetLowering::performOrCombine(SDNode *N,
14703 DAGCombinerInfo &DCI) const {
14704 SelectionDAG &DAG = DCI.DAG;
14705 SDValue LHS = N->getOperand(Num: 0);
14706 SDValue RHS = N->getOperand(Num: 1);
14707
14708 EVT VT = N->getValueType(ResNo: 0);
14709 if (VT == MVT::i1) {
14710 // or (fp_class x, c1), (fp_class x, c2) -> fp_class x, (c1 | c2)
14711 if (LHS.getOpcode() == AMDGPUISD::FP_CLASS &&
14712 RHS.getOpcode() == AMDGPUISD::FP_CLASS) {
14713 SDValue Src = LHS.getOperand(i: 0);
14714 if (Src != RHS.getOperand(i: 0))
14715 return SDValue();
14716
14717 const ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
14718 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i: 1));
14719 if (!CLHS || !CRHS)
14720 return SDValue();
14721
14722 // Only 10 bits are used.
14723 static const uint32_t MaxMask = 0x3ff;
14724
14725 uint32_t NewMask =
14726 (CLHS->getZExtValue() | CRHS->getZExtValue()) & MaxMask;
14727 SDLoc DL(N);
14728 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: Src,
14729 N2: DAG.getConstant(Val: NewMask, DL, VT: MVT::i32));
14730 }
14731
14732 return SDValue();
14733 }
14734
14735 // or (perm x, y, c1), c2 -> perm x, y, permute_mask(c1, c2)
14736 if (isa<ConstantSDNode>(Val: RHS) && LHS.hasOneUse() &&
14737 LHS.getOpcode() == AMDGPUISD::PERM &&
14738 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2))) {
14739 uint32_t Sel = getConstantPermuteMask(C: N->getConstantOperandVal(Num: 1));
14740 if (!Sel)
14741 return SDValue();
14742
14743 Sel |= LHS.getConstantOperandVal(i: 2);
14744 SDLoc DL(N);
14745 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
14746 N2: LHS.getOperand(i: 1), N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
14747 }
14748
14749 // or (op x, c1), (op y, c2) -> perm x, y, permute_mask(c1, c2)
14750 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
14751 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() &&
14752 N->isDivergent() && TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
14753
14754 // If all the uses of an or need to extract the individual elements, do not
14755 // attempt to lower into v_perm
14756 auto usesCombinedOperand = [](SDNode *OrUse) {
14757 // If we have any non-vectorized use, then it is a candidate for v_perm
14758 if (OrUse->getOpcode() != ISD::BITCAST ||
14759 !OrUse->getValueType(ResNo: 0).isVector())
14760 return true;
14761
14762 // If we have any non-vectorized use, then it is a candidate for v_perm
14763 for (auto *VUser : OrUse->users()) {
14764 if (!VUser->getValueType(ResNo: 0).isVector())
14765 return true;
14766
14767 // If the use of a vector is a store, then combining via a v_perm
14768 // is beneficial.
14769 // TODO -- whitelist more uses
14770 for (auto VectorwiseOp : {ISD::STORE, ISD::CopyToReg, ISD::CopyFromReg})
14771 if (VUser->getOpcode() == VectorwiseOp)
14772 return true;
14773 }
14774 return false;
14775 };
14776
14777 if (!any_of(Range: N->users(), P: usesCombinedOperand))
14778 return SDValue();
14779
14780 uint32_t LHSMask = getPermuteMask(V: LHS);
14781 uint32_t RHSMask = getPermuteMask(V: RHS);
14782
14783 if (LHSMask != ~0u && RHSMask != ~0u) {
14784 // Canonicalize the expression in an attempt to have fewer unique masks
14785 // and therefore fewer registers used to hold the masks.
14786 if (LHSMask > RHSMask) {
14787 std::swap(a&: LHSMask, b&: RHSMask);
14788 std::swap(a&: LHS, b&: RHS);
14789 }
14790
14791 // Select 0xc for each lane used from source operand. Zero has 0xc mask
14792 // set, 0xff have 0xff in the mask, actual lanes are in the 0-3 range.
14793 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
14794 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
14795
14796 // Check of we need to combine values from two sources within a byte.
14797 if (!(LHSUsedLanes & RHSUsedLanes) &&
14798 // If we select high and lower word keep it for SDWA.
14799 // TODO: teach SDWA to work with v_perm_b32 and remove the check.
14800 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
14801 // Kill zero bytes selected by other mask. Zero value is 0xc.
14802 LHSMask &= ~RHSUsedLanes;
14803 RHSMask &= ~LHSUsedLanes;
14804 // Add 4 to each active LHS lane
14805 LHSMask |= LHSUsedLanes & 0x04040404;
14806 // Combine masks
14807 uint32_t Sel = LHSMask | RHSMask;
14808 SDLoc DL(N);
14809
14810 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
14811 N2: RHS.getOperand(i: 0),
14812 N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
14813 }
14814 }
14815 if (LHSMask == ~0u || RHSMask == ~0u) {
14816 if (SDValue Perm = matchPERM(N, DCI))
14817 return Perm;
14818 }
14819 }
14820
14821 // Detect identity v2i32 OR and replace with identity source node.
14822 // Specifically an Or that has operands constructed from the same source node
14823 // via extract_vector_elt and build_vector. I.E.
14824 // v2i32 or(
14825 // v2i32 build_vector(
14826 // i32 extract_elt(%IdentitySrc, 0),
14827 // i32 0
14828 // ),
14829 // v2i32 build_vector(
14830 // i32 0,
14831 // i32 extract_elt(%IdentitySrc, 1)
14832 // ) )
14833 // =>
14834 // v2i32 %IdentitySrc
14835
14836 if (VT == MVT::v2i32 && LHS->getOpcode() == ISD::BUILD_VECTOR &&
14837 RHS->getOpcode() == ISD::BUILD_VECTOR) {
14838
14839 ConstantSDNode *LC = dyn_cast<ConstantSDNode>(Val: LHS->getOperand(Num: 1));
14840 ConstantSDNode *RC = dyn_cast<ConstantSDNode>(Val: RHS->getOperand(Num: 0));
14841
14842 // Test for and normalise build vectors.
14843 if (LC && RC && LC->getZExtValue() == 0 && RC->getZExtValue() == 0) {
14844
14845 // Get the extract_vector_element operands.
14846 SDValue LEVE = LHS->getOperand(Num: 0);
14847 SDValue REVE = RHS->getOperand(Num: 1);
14848
14849 if (LEVE->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
14850 REVE->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
14851 // Check that different elements from the same vector are
14852 // extracted.
14853 if (LEVE->getOperand(Num: 0) == REVE->getOperand(Num: 0) &&
14854 LEVE->getOperand(Num: 1) != REVE->getOperand(Num: 1)) {
14855 SDValue IdentitySrc = LEVE.getOperand(i: 0);
14856 return IdentitySrc;
14857 }
14858 }
14859 }
14860 }
14861
14862 if (VT != MVT::i64 || DCI.isBeforeLegalizeOps())
14863 return SDValue();
14864
14865 // TODO: This could be a generic combine with a predicate for extracting the
14866 // high half of an integer being free.
14867
14868 // (or i64:x, (zero_extend i32:y)) ->
14869 // i64 (bitcast (v2i32 build_vector (or i32:y, lo_32(x)), hi_32(x)))
14870 if (LHS.getOpcode() == ISD::ZERO_EXTEND &&
14871 RHS.getOpcode() != ISD::ZERO_EXTEND)
14872 std::swap(a&: LHS, b&: RHS);
14873
14874 if (RHS.getOpcode() == ISD::ZERO_EXTEND) {
14875 SDValue ExtSrc = RHS.getOperand(i: 0);
14876 EVT SrcVT = ExtSrc.getValueType();
14877 if (SrcVT == MVT::i32) {
14878 SDLoc SL(N);
14879 auto [LowLHS, HiBits] = split64BitValue(Op: LHS, DAG);
14880 SDValue LowOr = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: LowLHS, N2: ExtSrc);
14881
14882 DCI.AddToWorklist(N: LowOr.getNode());
14883 DCI.AddToWorklist(N: HiBits.getNode());
14884
14885 SDValue Vec =
14886 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: LowOr, N2: HiBits);
14887 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
14888 }
14889 }
14890
14891 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
14892 if (CRHS) {
14893 if (SDValue Split = splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::OR,
14894 LHS: N->getOperand(Num: 0), CRHS))
14895 return Split;
14896 }
14897
14898 return SDValue();
14899}
14900
14901SDValue SITargetLowering::performXorCombine(SDNode *N,
14902 DAGCombinerInfo &DCI) const {
14903 if (SDValue RV = reassociateScalarOps(N, DAG&: DCI.DAG))
14904 return RV;
14905
14906 SDValue LHS = N->getOperand(Num: 0);
14907 SDValue RHS = N->getOperand(Num: 1);
14908
14909 const ConstantSDNode *CRHS = isConstOrConstSplat(N: RHS);
14910 SelectionDAG &DAG = DCI.DAG;
14911
14912 EVT VT = N->getValueType(ResNo: 0);
14913 if (CRHS && VT == MVT::i64) {
14914 if (SDValue Split =
14915 splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::XOR, LHS, CRHS))
14916 return Split;
14917 }
14918
14919 // v2i32 (xor (vselect cc, x, y), K) ->
14920 // (v2i32 svelect cc, (xor x, K), (xor y, K)) This enables the xor to be
14921 // replaced with source modifiers when the select is lowered to CNDMASK.
14922 unsigned Opc = LHS.getOpcode();
14923 if (((Opc == ISD::VSELECT && VT == MVT::v2i32) ||
14924 (Opc == ISD::SELECT && VT == MVT::i64)) &&
14925 CRHS && CRHS->getAPIntValue().isSignMask()) {
14926 SDValue CC = LHS->getOperand(Num: 0);
14927 SDValue TRUE = LHS->getOperand(Num: 1);
14928 SDValue FALSE = LHS->getOperand(Num: 2);
14929 SDValue XTrue = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N), VT, N1: TRUE, N2: RHS);
14930 SDValue XFalse = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N), VT, N1: FALSE, N2: RHS);
14931 SDValue XSelect =
14932 DAG.getNode(Opcode: ISD::VSELECT, DL: SDLoc(N), VT, N1: CC, N2: XTrue, N3: XFalse);
14933 return XSelect;
14934 }
14935
14936 // Make sure to apply the 64-bit constant splitting fold before trying to fold
14937 // fneg-like xors into 64-bit select.
14938 if (LHS.getOpcode() == ISD::SELECT && VT == MVT::i32) {
14939 // This looks like an fneg, try to fold as a source modifier.
14940 if (CRHS && CRHS->getAPIntValue().isSignMask() &&
14941 shouldFoldFNegIntoSrc(FNeg: N, FNegSrc: LHS)) {
14942 // xor (select c, a, b), 0x80000000 ->
14943 // bitcast (select c, (fneg (bitcast a)), (fneg (bitcast b)))
14944 SDLoc DL(N);
14945 SDValue CastLHS =
14946 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: LHS->getOperand(Num: 1));
14947 SDValue CastRHS =
14948 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: LHS->getOperand(Num: 2));
14949 SDValue FNegLHS = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f32, Operand: CastLHS);
14950 SDValue FNegRHS = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f32, Operand: CastRHS);
14951 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::f32,
14952 N1: LHS->getOperand(Num: 0), N2: FNegLHS, N3: FNegRHS);
14953 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: NewSelect);
14954 }
14955 }
14956
14957 return SDValue();
14958}
14959
14960SDValue
14961SITargetLowering::performZeroOrAnyExtendCombine(SDNode *N,
14962 DAGCombinerInfo &DCI) const {
14963 if (!Subtarget->has16BitInsts() ||
14964 DCI.getDAGCombineLevel() < AfterLegalizeTypes)
14965 return SDValue();
14966
14967 EVT VT = N->getValueType(ResNo: 0);
14968 if (VT != MVT::i32)
14969 return SDValue();
14970
14971 SDValue Src = N->getOperand(Num: 0);
14972 if (Src.getValueType() != MVT::i16)
14973 return SDValue();
14974
14975 if (!Src->hasOneUse())
14976 return SDValue();
14977
14978 // TODO: We bail out below if SrcOffset is not in the first dword (>= 4). It's
14979 // possible we're missing out on some combine opportunities, but we'd need to
14980 // weigh the cost of extracting the byte from the upper dwords.
14981
14982 std::optional<ByteProvider<SDValue>> BP0 =
14983 calculateByteProvider(Op: SDValue(N, 0), Index: 0, Depth: 0, StartingIndex: 0);
14984 if (!BP0 || BP0->SrcOffset >= 4 || !BP0->Src)
14985 return SDValue();
14986 SDValue V0 = *BP0->Src;
14987
14988 std::optional<ByteProvider<SDValue>> BP1 =
14989 calculateByteProvider(Op: SDValue(N, 0), Index: 1, Depth: 0, StartingIndex: 1);
14990 if (!BP1 || BP1->SrcOffset >= 4 || !BP1->Src)
14991 return SDValue();
14992
14993 SDValue V1 = *BP1->Src;
14994
14995 if (V0 == V1)
14996 return SDValue();
14997
14998 SelectionDAG &DAG = DCI.DAG;
14999 SDLoc DL(N);
15000 uint32_t PermMask = 0x0c0c0c0c;
15001 if (V0) {
15002 V0 = DAG.getBitcastedAnyExtOrTrunc(Op: V0, DL, VT: MVT::i32);
15003 PermMask = (PermMask & ~0xFF) | (BP0->SrcOffset + 4);
15004 }
15005
15006 if (V1) {
15007 V1 = DAG.getBitcastedAnyExtOrTrunc(Op: V1, DL, VT: MVT::i32);
15008 PermMask = (PermMask & ~(0xFF << 8)) | (BP1->SrcOffset << 8);
15009 }
15010
15011 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: V0, N2: V1,
15012 N3: DAG.getConstant(Val: PermMask, DL, VT: MVT::i32));
15013}
15014
15015SDValue
15016SITargetLowering::performSignExtendInRegCombine(SDNode *N,
15017 DAGCombinerInfo &DCI) const {
15018 SDValue Src = N->getOperand(Num: 0);
15019 auto *VTSign = cast<VTSDNode>(Val: N->getOperand(Num: 1));
15020
15021 // Combine s_buffer_load_u8 or s_buffer_load_u16 with sext and replace them
15022 // with s_buffer_load_i8 and s_buffer_load_i16 respectively.
15023 if (((Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE &&
15024 VTSign->getVT() == MVT::i8) ||
15025 (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_USHORT &&
15026 VTSign->getVT() == MVT::i16))) {
15027 assert(Subtarget->hasScalarSubwordLoads() &&
15028 "s_buffer_load_{u8, i8} are supported "
15029 "in GFX12 (or newer) architectures.");
15030 EVT VT = Src.getValueType();
15031 unsigned Opc = (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE)
15032 ? AMDGPUISD::SBUFFER_LOAD_BYTE
15033 : AMDGPUISD::SBUFFER_LOAD_SHORT;
15034 SDLoc DL(N);
15035 SDVTList ResList = DCI.DAG.getVTList(VT: MVT::i32);
15036 SDValue Ops[] = {
15037 Src.getOperand(i: 0), // source register
15038 Src.getOperand(i: 1), // offset
15039 Src.getOperand(i: 2) // cachePolicy
15040 };
15041 auto *M = cast<MemSDNode>(Val&: Src);
15042 SDValue BufferLoad = DCI.DAG.getMemIntrinsicNode(
15043 Opcode: Opc, dl: DL, VTList: ResList, Ops, MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
15044 SDValue LoadVal = DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: BufferLoad);
15045 return LoadVal;
15046 }
15047 if (((Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE &&
15048 VTSign->getVT() == MVT::i8) ||
15049 (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_USHORT &&
15050 VTSign->getVT() == MVT::i16)) &&
15051 Src.hasOneUse()) {
15052 auto *M = cast<MemSDNode>(Val&: Src);
15053 SDValue Ops[] = {Src.getOperand(i: 0), // Chain
15054 Src.getOperand(i: 1), // rsrc
15055 Src.getOperand(i: 2), // vindex
15056 Src.getOperand(i: 3), // voffset
15057 Src.getOperand(i: 4), // soffset
15058 Src.getOperand(i: 5), // offset
15059 Src.getOperand(i: 6), Src.getOperand(i: 7)};
15060 // replace with BUFFER_LOAD_BYTE/SHORT
15061 SDVTList ResList =
15062 DCI.DAG.getVTList(VT1: MVT::i32, VT2: Src.getOperand(i: 0).getValueType());
15063 unsigned Opc = (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE)
15064 ? AMDGPUISD::BUFFER_LOAD_BYTE
15065 : AMDGPUISD::BUFFER_LOAD_SHORT;
15066 SDValue BufferLoadSignExt = DCI.DAG.getMemIntrinsicNode(
15067 Opcode: Opc, dl: SDLoc(N), VTList: ResList, Ops, MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
15068 return DCI.DAG.getMergeValues(
15069 Ops: {BufferLoadSignExt, BufferLoadSignExt.getValue(R: 1)}, dl: SDLoc(N));
15070 }
15071 return SDValue();
15072}
15073
15074SDValue SITargetLowering::performClassCombine(SDNode *N,
15075 DAGCombinerInfo &DCI) const {
15076 SelectionDAG &DAG = DCI.DAG;
15077 SDValue Mask = N->getOperand(Num: 1);
15078
15079 // fp_class x, 0 -> false
15080 if (isNullConstant(V: Mask))
15081 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT: MVT::i1);
15082
15083 if (N->getOperand(Num: 0).isUndef())
15084 return DAG.getUNDEF(VT: MVT::i1);
15085
15086 return SDValue();
15087}
15088
15089SDValue SITargetLowering::performRcpCombine(SDNode *N,
15090 DAGCombinerInfo &DCI) const {
15091 EVT VT = N->getValueType(ResNo: 0);
15092 SDValue N0 = N->getOperand(Num: 0);
15093
15094 if (N0.isUndef()) {
15095 return DCI.DAG.getConstantFP(Val: APFloat::getQNaN(Sem: VT.getFltSemantics()),
15096 DL: SDLoc(N), VT);
15097 }
15098
15099 if (VT == MVT::f32 && (N0.getOpcode() == ISD::UINT_TO_FP ||
15100 N0.getOpcode() == ISD::SINT_TO_FP)) {
15101 return DCI.DAG.getNode(Opcode: AMDGPUISD::RCP_IFLAG, DL: SDLoc(N), VT, Operand: N0,
15102 Flags: N->getFlags());
15103 }
15104
15105 // TODO: Could handle f32 + amdgcn.sqrt but probably never reaches here.
15106 if ((VT == MVT::f16 && N0.getOpcode() == ISD::FSQRT) &&
15107 N->getFlags().hasAllowContract() && N0->getFlags().hasAllowContract()) {
15108 return DCI.DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SDLoc(N), VT, Operand: N0.getOperand(i: 0),
15109 Flags: N->getFlags());
15110 }
15111
15112 return AMDGPUTargetLowering::performRcpCombine(N, DCI);
15113}
15114
15115bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
15116 SDNodeFlags UserFlags,
15117 unsigned MaxDepth) const {
15118 unsigned Opcode = Op.getOpcode();
15119 if (Opcode == ISD::FCANONICALIZE)
15120 return true;
15121
15122 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
15123 const auto &F = CFP->getValueAPF();
15124 if (F.isNaN() && F.isSignaling())
15125 return false;
15126 if (!F.isDenormal())
15127 return true;
15128
15129 DenormalMode Mode =
15130 DAG.getMachineFunction().getDenormalMode(FPType: F.getSemantics());
15131 return Mode == DenormalMode::getIEEE();
15132 }
15133
15134 // If source is a result of another standard FP operation it is already in
15135 // canonical form.
15136 if (MaxDepth == 0)
15137 return false;
15138
15139 switch (Opcode) {
15140 // These will flush denorms if required.
15141 case ISD::FADD:
15142 case ISD::FSUB:
15143 case ISD::FMUL:
15144 case ISD::FCEIL:
15145 case ISD::FFLOOR:
15146 case ISD::FMA:
15147 case ISD::FMAD:
15148 case ISD::FSQRT:
15149 case ISD::FDIV:
15150 case ISD::FREM:
15151 case ISD::FP_ROUND:
15152 case ISD::FP_EXTEND:
15153 case ISD::FP16_TO_FP:
15154 case ISD::FP_TO_FP16:
15155 case ISD::BF16_TO_FP:
15156 case ISD::FP_TO_BF16:
15157 case ISD::FLDEXP:
15158 case AMDGPUISD::FMUL_LEGACY:
15159 case AMDGPUISD::FMAD_FTZ:
15160 case AMDGPUISD::RCP:
15161 case AMDGPUISD::RSQ:
15162 case AMDGPUISD::RSQ_CLAMP:
15163 case AMDGPUISD::RCP_LEGACY:
15164 case AMDGPUISD::RCP_IFLAG:
15165 case AMDGPUISD::LOG:
15166 case AMDGPUISD::EXP:
15167 case AMDGPUISD::DIV_SCALE:
15168 case AMDGPUISD::DIV_FMAS:
15169 case AMDGPUISD::DIV_FIXUP:
15170 case AMDGPUISD::FRACT:
15171 case AMDGPUISD::CVT_PKRTZ_F16_F32:
15172 case AMDGPUISD::CVT_F32_UBYTE0:
15173 case AMDGPUISD::CVT_F32_UBYTE1:
15174 case AMDGPUISD::CVT_F32_UBYTE2:
15175 case AMDGPUISD::CVT_F32_UBYTE3:
15176 case AMDGPUISD::FP_TO_FP16:
15177 case AMDGPUISD::SIN_HW:
15178 case AMDGPUISD::COS_HW:
15179 return true;
15180
15181 // It can/will be lowered or combined as a bit operation.
15182 // Need to check their input recursively to handle.
15183 case ISD::FNEG:
15184 case ISD::FABS:
15185 case ISD::FCOPYSIGN:
15186 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), UserFlags: MaxDepth - 1);
15187
15188 case ISD::AND:
15189 if (Op.getValueType() == MVT::i32) {
15190 // Be careful as we only know it is a bitcast floating point type. It
15191 // could be f32, v2f16, we have no way of knowing. Luckily the constant
15192 // value that we optimize for, which comes up in fp32 to bf16 conversions,
15193 // is valid to optimize for all types.
15194 if (auto *RHS = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
15195 if (RHS->getZExtValue() == 0xffff0000) {
15196 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), UserFlags: MaxDepth - 1);
15197 }
15198 }
15199 }
15200 break;
15201
15202 case ISD::FSIN:
15203 case ISD::FCOS:
15204 case ISD::FSINCOS:
15205 return Op.getValueType().getScalarType() != MVT::f16;
15206
15207 case ISD::FMINNUM:
15208 case ISD::FMAXNUM:
15209 case ISD::FMINNUM_IEEE:
15210 case ISD::FMAXNUM_IEEE:
15211 case ISD::FMINIMUM:
15212 case ISD::FMAXIMUM:
15213 case ISD::FMINIMUMNUM:
15214 case ISD::FMAXIMUMNUM:
15215 case AMDGPUISD::CLAMP:
15216 case AMDGPUISD::FMED3:
15217 case AMDGPUISD::FMAX3:
15218 case AMDGPUISD::FMIN3:
15219 case AMDGPUISD::FMAXIMUM3:
15220 case AMDGPUISD::FMINIMUM3: {
15221 // FIXME: Shouldn't treat the generic operations different based these.
15222 // However, we aren't really required to flush the result from
15223 // minnum/maxnum..
15224
15225 // snans will be quieted, so we only need to worry about denormals.
15226 if (Subtarget->supportsMinMaxDenormModes() ||
15227 // FIXME: denormalsEnabledForType is broken for dynamic
15228 denormalsEnabledForType(DAG, VT: Op.getValueType()))
15229 return true;
15230
15231 // Flushing may be required.
15232 // In pre-GFX9 targets V_MIN_F32 and others do not flush denorms. For such
15233 // targets need to check their input recursively.
15234
15235 // FIXME: Does this apply with clamp? It's implemented with max.
15236 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
15237 if (!isCanonicalized(DAG, Op: Op.getOperand(i: I), UserFlags: MaxDepth - 1))
15238 return false;
15239 }
15240
15241 return true;
15242 }
15243 case ISD::SELECT: {
15244 return isCanonicalized(DAG, Op: Op.getOperand(i: 1), UserFlags: MaxDepth - 1) &&
15245 isCanonicalized(DAG, Op: Op.getOperand(i: 2), UserFlags: MaxDepth - 1);
15246 }
15247 case ISD::BUILD_VECTOR: {
15248 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
15249 SDValue SrcOp = Op.getOperand(i);
15250 if (!isCanonicalized(DAG, Op: SrcOp, UserFlags: MaxDepth - 1))
15251 return false;
15252 }
15253
15254 return true;
15255 }
15256 case ISD::EXTRACT_VECTOR_ELT:
15257 case ISD::EXTRACT_SUBVECTOR: {
15258 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), UserFlags: MaxDepth - 1);
15259 }
15260 case ISD::INSERT_VECTOR_ELT: {
15261 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), UserFlags: MaxDepth - 1) &&
15262 isCanonicalized(DAG, Op: Op.getOperand(i: 1), UserFlags: MaxDepth - 1);
15263 }
15264 case ISD::UNDEF:
15265 // Could be anything.
15266 return false;
15267
15268 case ISD::BITCAST:
15269 // TODO: This is incorrect as it loses track of the operand's type. We may
15270 // end up effectively bitcasting from f32 to v2f16 or vice versa, and the
15271 // same bits that are canonicalized in one type need not be in the other.
15272 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), UserFlags: MaxDepth - 1);
15273 case ISD::TRUNCATE: {
15274 // Hack round the mess we make when legalizing extract_vector_elt
15275 if (Op.getValueType() == MVT::i16) {
15276 SDValue TruncSrc = Op.getOperand(i: 0);
15277 if (TruncSrc.getValueType() == MVT::i32 &&
15278 TruncSrc.getOpcode() == ISD::BITCAST &&
15279 TruncSrc.getOperand(i: 0).getValueType() == MVT::v2f16) {
15280 return isCanonicalized(DAG, Op: TruncSrc.getOperand(i: 0), UserFlags: MaxDepth - 1);
15281 }
15282 }
15283 return false;
15284 }
15285 case ISD::INTRINSIC_WO_CHAIN: {
15286 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
15287 // TODO: Handle more intrinsics
15288 switch (IntrinsicID) {
15289 case Intrinsic::amdgcn_cvt_pkrtz:
15290 case Intrinsic::amdgcn_cubeid:
15291 case Intrinsic::amdgcn_frexp_mant:
15292 case Intrinsic::amdgcn_fdot2:
15293 case Intrinsic::amdgcn_rcp:
15294 case Intrinsic::amdgcn_rsq:
15295 case Intrinsic::amdgcn_rsq_clamp:
15296 case Intrinsic::amdgcn_rcp_legacy:
15297 case Intrinsic::amdgcn_rsq_legacy:
15298 case Intrinsic::amdgcn_trig_preop:
15299 case Intrinsic::amdgcn_tanh:
15300 case Intrinsic::amdgcn_log:
15301 case Intrinsic::amdgcn_exp2:
15302 case Intrinsic::amdgcn_sqrt:
15303 return true;
15304 default:
15305 break;
15306 }
15307
15308 break;
15309 }
15310 default:
15311 break;
15312 }
15313
15314 // FIXME: denormalsEnabledForType is broken for dynamic
15315 return denormalsEnabledForType(DAG, VT: Op.getValueType()) &&
15316 (UserFlags.hasNoNaNs() || DAG.isKnownNeverSNaN(Op));
15317}
15318
15319bool SITargetLowering::isCanonicalized(Register Reg, const MachineFunction &MF,
15320 unsigned MaxDepth) const {
15321 const MachineRegisterInfo &MRI = MF.getRegInfo();
15322 MachineInstr *MI = MRI.getVRegDef(Reg);
15323 unsigned Opcode = MI->getOpcode();
15324
15325 if (Opcode == AMDGPU::G_FCANONICALIZE)
15326 return true;
15327
15328 std::optional<FPValueAndVReg> FCR;
15329 // Constant splat (can be padded with undef) or scalar constant.
15330 if (mi_match(R: Reg, MRI, P: MIPatternMatch::m_GFCstOrSplat(FPValReg&: FCR))) {
15331 if (FCR->Value.isSignaling())
15332 return false;
15333 if (!FCR->Value.isDenormal())
15334 return true;
15335
15336 DenormalMode Mode = MF.getDenormalMode(FPType: FCR->Value.getSemantics());
15337 return Mode == DenormalMode::getIEEE();
15338 }
15339
15340 if (MaxDepth == 0)
15341 return false;
15342
15343 switch (Opcode) {
15344 case AMDGPU::G_FADD:
15345 case AMDGPU::G_FSUB:
15346 case AMDGPU::G_FMUL:
15347 case AMDGPU::G_FCEIL:
15348 case AMDGPU::G_FFLOOR:
15349 case AMDGPU::G_FRINT:
15350 case AMDGPU::G_FNEARBYINT:
15351 case AMDGPU::G_INTRINSIC_FPTRUNC_ROUND:
15352 case AMDGPU::G_INTRINSIC_TRUNC:
15353 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
15354 case AMDGPU::G_FMA:
15355 case AMDGPU::G_FMAD:
15356 case AMDGPU::G_FSQRT:
15357 case AMDGPU::G_FDIV:
15358 case AMDGPU::G_FREM:
15359 case AMDGPU::G_FPOW:
15360 case AMDGPU::G_FPEXT:
15361 case AMDGPU::G_FLOG:
15362 case AMDGPU::G_FLOG2:
15363 case AMDGPU::G_FLOG10:
15364 case AMDGPU::G_FPTRUNC:
15365 case AMDGPU::G_AMDGPU_RCP_IFLAG:
15366 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
15367 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
15368 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
15369 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
15370 return true;
15371 case AMDGPU::G_FNEG:
15372 case AMDGPU::G_FABS:
15373 case AMDGPU::G_FCOPYSIGN:
15374 return isCanonicalized(Reg: MI->getOperand(i: 1).getReg(), MF, MaxDepth: MaxDepth - 1);
15375 case AMDGPU::G_FMINNUM:
15376 case AMDGPU::G_FMAXNUM:
15377 case AMDGPU::G_FMINNUM_IEEE:
15378 case AMDGPU::G_FMAXNUM_IEEE:
15379 case AMDGPU::G_FMINIMUM:
15380 case AMDGPU::G_FMAXIMUM:
15381 case AMDGPU::G_FMINIMUMNUM:
15382 case AMDGPU::G_FMAXIMUMNUM: {
15383 if (Subtarget->supportsMinMaxDenormModes() ||
15384 // FIXME: denormalsEnabledForType is broken for dynamic
15385 denormalsEnabledForType(Ty: MRI.getType(Reg), MF))
15386 return true;
15387
15388 [[fallthrough]];
15389 }
15390 case AMDGPU::G_BUILD_VECTOR:
15391 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands()))
15392 if (!isCanonicalized(Reg: MO.getReg(), MF, MaxDepth: MaxDepth - 1))
15393 return false;
15394 return true;
15395 case AMDGPU::G_INTRINSIC:
15396 case AMDGPU::G_INTRINSIC_CONVERGENT:
15397 switch (cast<GIntrinsic>(Val: MI)->getIntrinsicID()) {
15398 case Intrinsic::amdgcn_fmul_legacy:
15399 case Intrinsic::amdgcn_fmad_ftz:
15400 case Intrinsic::amdgcn_sqrt:
15401 case Intrinsic::amdgcn_fmed3:
15402 case Intrinsic::amdgcn_sin:
15403 case Intrinsic::amdgcn_cos:
15404 case Intrinsic::amdgcn_log:
15405 case Intrinsic::amdgcn_exp2:
15406 case Intrinsic::amdgcn_log_clamp:
15407 case Intrinsic::amdgcn_rcp:
15408 case Intrinsic::amdgcn_rcp_legacy:
15409 case Intrinsic::amdgcn_rsq:
15410 case Intrinsic::amdgcn_rsq_clamp:
15411 case Intrinsic::amdgcn_rsq_legacy:
15412 case Intrinsic::amdgcn_div_scale:
15413 case Intrinsic::amdgcn_div_fmas:
15414 case Intrinsic::amdgcn_div_fixup:
15415 case Intrinsic::amdgcn_fract:
15416 case Intrinsic::amdgcn_cvt_pkrtz:
15417 case Intrinsic::amdgcn_cubeid:
15418 case Intrinsic::amdgcn_cubema:
15419 case Intrinsic::amdgcn_cubesc:
15420 case Intrinsic::amdgcn_cubetc:
15421 case Intrinsic::amdgcn_frexp_mant:
15422 case Intrinsic::amdgcn_fdot2:
15423 case Intrinsic::amdgcn_trig_preop:
15424 case Intrinsic::amdgcn_tanh:
15425 return true;
15426 default:
15427 break;
15428 }
15429
15430 [[fallthrough]];
15431 default:
15432 return false;
15433 }
15434
15435 llvm_unreachable("invalid operation");
15436}
15437
15438// Constant fold canonicalize.
15439SDValue SITargetLowering::getCanonicalConstantFP(SelectionDAG &DAG,
15440 const SDLoc &SL, EVT VT,
15441 const APFloat &C) const {
15442 // Flush denormals to 0 if not enabled.
15443 if (C.isDenormal()) {
15444 DenormalMode Mode =
15445 DAG.getMachineFunction().getDenormalMode(FPType: C.getSemantics());
15446 if (Mode == DenormalMode::getPreserveSign()) {
15447 return DAG.getConstantFP(
15448 Val: APFloat::getZero(Sem: C.getSemantics(), Negative: C.isNegative()), DL: SL, VT);
15449 }
15450
15451 if (Mode != DenormalMode::getIEEE())
15452 return SDValue();
15453 }
15454
15455 if (C.isNaN()) {
15456 APFloat CanonicalQNaN = APFloat::getQNaN(Sem: C.getSemantics());
15457 if (C.isSignaling()) {
15458 // Quiet a signaling NaN.
15459 // FIXME: Is this supposed to preserve payload bits?
15460 return DAG.getConstantFP(Val: CanonicalQNaN, DL: SL, VT);
15461 }
15462
15463 // Make sure it is the canonical NaN bitpattern.
15464 //
15465 // TODO: Can we use -1 as the canonical NaN value since it's an inline
15466 // immediate?
15467 if (C.bitcastToAPInt() != CanonicalQNaN.bitcastToAPInt())
15468 return DAG.getConstantFP(Val: CanonicalQNaN, DL: SL, VT);
15469 }
15470
15471 // Already canonical.
15472 return DAG.getConstantFP(Val: C, DL: SL, VT);
15473}
15474
15475static bool vectorEltWillFoldAway(SDValue Op) {
15476 return Op.isUndef() || isa<ConstantFPSDNode>(Val: Op);
15477}
15478
15479SDValue
15480SITargetLowering::performFCanonicalizeCombine(SDNode *N,
15481 DAGCombinerInfo &DCI) const {
15482 SelectionDAG &DAG = DCI.DAG;
15483 SDValue N0 = N->getOperand(Num: 0);
15484 EVT VT = N->getValueType(ResNo: 0);
15485
15486 // fcanonicalize undef -> qnan
15487 if (N0.isUndef()) {
15488 APFloat QNaN = APFloat::getQNaN(Sem: VT.getFltSemantics());
15489 return DAG.getConstantFP(Val: QNaN, DL: SDLoc(N), VT);
15490 }
15491
15492 if (ConstantFPSDNode *CFP = isConstOrConstSplatFP(N: N0)) {
15493 EVT VT = N->getValueType(ResNo: 0);
15494 return getCanonicalConstantFP(DAG, SL: SDLoc(N), VT, C: CFP->getValueAPF());
15495 }
15496
15497 // fcanonicalize (build_vector x, k) -> build_vector (fcanonicalize x),
15498 // (fcanonicalize k)
15499 //
15500 // fcanonicalize (build_vector x, undef) -> build_vector (fcanonicalize x), 0
15501
15502 // TODO: This could be better with wider vectors that will be split to v2f16,
15503 // and to consider uses since there aren't that many packed operations.
15504 if (N0.getOpcode() == ISD::BUILD_VECTOR && VT == MVT::v2f16 &&
15505 isTypeLegal(VT: MVT::v2f16)) {
15506 SDLoc SL(N);
15507 SDValue NewElts[2];
15508 SDValue Lo = N0.getOperand(i: 0);
15509 SDValue Hi = N0.getOperand(i: 1);
15510 EVT EltVT = Lo.getValueType();
15511
15512 if (vectorEltWillFoldAway(Op: Lo) || vectorEltWillFoldAway(Op: Hi)) {
15513 for (unsigned I = 0; I != 2; ++I) {
15514 SDValue Op = N0.getOperand(i: I);
15515 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
15516 NewElts[I] =
15517 getCanonicalConstantFP(DAG, SL, VT: EltVT, C: CFP->getValueAPF());
15518 } else if (Op.isUndef()) {
15519 // Handled below based on what the other operand is.
15520 NewElts[I] = Op;
15521 } else {
15522 NewElts[I] = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: SL, VT: EltVT, Operand: Op);
15523 }
15524 }
15525
15526 // If one half is undef, and one is constant, prefer a splat vector rather
15527 // than the normal qNaN. If it's a register, prefer 0.0 since that's
15528 // cheaper to use and may be free with a packed operation.
15529 if (NewElts[0].isUndef()) {
15530 if (isa<ConstantFPSDNode>(Val: NewElts[1]))
15531 NewElts[0] = isa<ConstantFPSDNode>(Val: NewElts[1])
15532 ? NewElts[1]
15533 : DAG.getConstantFP(Val: 0.0f, DL: SL, VT: EltVT);
15534 }
15535
15536 if (NewElts[1].isUndef()) {
15537 NewElts[1] = isa<ConstantFPSDNode>(Val: NewElts[0])
15538 ? NewElts[0]
15539 : DAG.getConstantFP(Val: 0.0f, DL: SL, VT: EltVT);
15540 }
15541
15542 return DAG.getBuildVector(VT, DL: SL, Ops: NewElts);
15543 }
15544 }
15545
15546 return SDValue();
15547}
15548
15549static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc) {
15550 switch (Opc) {
15551 case ISD::FMAXNUM:
15552 case ISD::FMAXNUM_IEEE:
15553 case ISD::FMAXIMUMNUM:
15554 return AMDGPUISD::FMAX3;
15555 case ISD::FMAXIMUM:
15556 return AMDGPUISD::FMAXIMUM3;
15557 case ISD::SMAX:
15558 return AMDGPUISD::SMAX3;
15559 case ISD::UMAX:
15560 return AMDGPUISD::UMAX3;
15561 case ISD::FMINNUM:
15562 case ISD::FMINNUM_IEEE:
15563 case ISD::FMINIMUMNUM:
15564 return AMDGPUISD::FMIN3;
15565 case ISD::FMINIMUM:
15566 return AMDGPUISD::FMINIMUM3;
15567 case ISD::SMIN:
15568 return AMDGPUISD::SMIN3;
15569 case ISD::UMIN:
15570 return AMDGPUISD::UMIN3;
15571 default:
15572 llvm_unreachable("Not a min/max opcode");
15573 }
15574}
15575
15576SDValue SITargetLowering::performIntMed3ImmCombine(SelectionDAG &DAG,
15577 const SDLoc &SL, SDValue Src,
15578 SDValue MinVal,
15579 SDValue MaxVal,
15580 bool Signed) const {
15581
15582 // med3 comes from
15583 // min(max(x, K0), K1), K0 < K1
15584 // max(min(x, K0), K1), K1 < K0
15585 //
15586 // "MinVal" and "MaxVal" respectively refer to the rhs of the
15587 // min/max op.
15588 ConstantSDNode *MinK = dyn_cast<ConstantSDNode>(Val&: MinVal);
15589 ConstantSDNode *MaxK = dyn_cast<ConstantSDNode>(Val&: MaxVal);
15590
15591 if (!MinK || !MaxK)
15592 return SDValue();
15593
15594 if (Signed) {
15595 if (MaxK->getAPIntValue().sge(RHS: MinK->getAPIntValue()))
15596 return SDValue();
15597 } else {
15598 if (MaxK->getAPIntValue().uge(RHS: MinK->getAPIntValue()))
15599 return SDValue();
15600 }
15601
15602 EVT VT = MinK->getValueType(ResNo: 0);
15603 unsigned Med3Opc = Signed ? AMDGPUISD::SMED3 : AMDGPUISD::UMED3;
15604 if (VT == MVT::i32 || (VT == MVT::i16 && Subtarget->hasMed3_16()))
15605 return DAG.getNode(Opcode: Med3Opc, DL: SL, VT, N1: Src, N2: MaxVal, N3: MinVal);
15606
15607 // Note: we could also extend to i32 and use i32 med3 if i16 med3 is
15608 // not available, but this is unlikely to be profitable as constants
15609 // will often need to be materialized & extended, especially on
15610 // pre-GFX10 where VOP3 instructions couldn't take literal operands.
15611 return SDValue();
15612}
15613
15614static ConstantFPSDNode *getSplatConstantFP(SDValue Op) {
15615 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op))
15616 return C;
15617
15618 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: Op)) {
15619 if (ConstantFPSDNode *C = BV->getConstantFPSplatNode())
15620 return C;
15621 }
15622
15623 return nullptr;
15624}
15625
15626SDValue SITargetLowering::performFPMed3ImmCombine(SelectionDAG &DAG,
15627 const SDLoc &SL, SDValue Op0,
15628 SDValue Op1,
15629 bool IsKnownNoNaNs) const {
15630 ConstantFPSDNode *K1 = getSplatConstantFP(Op: Op1);
15631 if (!K1)
15632 return SDValue();
15633
15634 ConstantFPSDNode *K0 = getSplatConstantFP(Op: Op0.getOperand(i: 1));
15635 if (!K0)
15636 return SDValue();
15637
15638 // Ordered >= (although NaN inputs should have folded away by now).
15639 if (K0->getValueAPF() > K1->getValueAPF())
15640 return SDValue();
15641
15642 // med3 with a nan input acts like
15643 // v_min_f32(v_min_f32(S0.f32, S1.f32), S2.f32)
15644 //
15645 // So the result depends on whether the IEEE mode bit is enabled or not with a
15646 // signaling nan input.
15647 // ieee=1
15648 // s0 snan: yields s2
15649 // s1 snan: yields s2
15650 // s2 snan: qnan
15651
15652 // s0 qnan: min(s1, s2)
15653 // s1 qnan: min(s0, s2)
15654 // s2 qnan: min(s0, s1)
15655
15656 // ieee=0
15657 // s0 snan: min(s1, s2)
15658 // s1 snan: min(s0, s2)
15659 // s2 snan: qnan
15660
15661 // s0 qnan: min(s1, s2)
15662 // s1 qnan: min(s0, s2)
15663 // s2 qnan: min(s0, s1)
15664 const MachineFunction &MF = DAG.getMachineFunction();
15665 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
15666
15667 // TODO: Check IEEE bit enabled. We can form fmed3 with IEEE=0 regardless of
15668 // whether the input is a signaling nan if op0 is fmaximum or fmaximumnum. We
15669 // can only form if op0 is fmaxnum_ieee if IEEE=1.
15670 EVT VT = Op0.getValueType();
15671 if (Info->getMode().DX10Clamp) {
15672 // If dx10_clamp is enabled, NaNs clamp to 0.0. This is the same as the
15673 // hardware fmed3 behavior converting to a min.
15674 // FIXME: Should this be allowing -0.0?
15675 if (K1->isExactlyValue(V: 1.0) && K0->isExactlyValue(V: 0.0))
15676 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Op0.getOperand(i: 0));
15677 }
15678
15679 // med3 for f16 is only available on gfx9+, and not available for v2f16.
15680 if (VT == MVT::f32 || (VT == MVT::f16 && Subtarget->hasMed3_16())) {
15681 // This isn't safe with signaling NaNs because in IEEE mode, min/max on a
15682 // signaling NaN gives a quiet NaN. The quiet NaN input to the min would
15683 // then give the other result, which is different from med3 with a NaN
15684 // input.
15685 SDValue Var = Op0.getOperand(i: 0);
15686 if (!IsKnownNoNaNs && !DAG.isKnownNeverSNaN(Op: Var))
15687 return SDValue();
15688
15689 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
15690
15691 if ((!K0->hasOneUse() || TII->isInlineConstant(Imm: K0->getValueAPF())) &&
15692 (!K1->hasOneUse() || TII->isInlineConstant(Imm: K1->getValueAPF()))) {
15693 return DAG.getNode(Opcode: AMDGPUISD::FMED3, DL: SL, VT: K0->getValueType(ResNo: 0), N1: Var,
15694 N2: SDValue(K0, 0), N3: SDValue(K1, 0));
15695 }
15696 }
15697
15698 return SDValue();
15699}
15700
15701/// \return true if the subtarget supports minimum3 and maximum3 with the given
15702/// base min/max opcode \p Opc for type \p VT.
15703static bool supportsMin3Max3(const GCNSubtarget &Subtarget, unsigned Opc,
15704 EVT VT) {
15705 switch (Opc) {
15706 case ISD::FMINNUM:
15707 case ISD::FMAXNUM:
15708 case ISD::FMINNUM_IEEE:
15709 case ISD::FMAXNUM_IEEE:
15710 case ISD::FMINIMUMNUM:
15711 case ISD::FMAXIMUMNUM:
15712 case AMDGPUISD::FMIN_LEGACY:
15713 case AMDGPUISD::FMAX_LEGACY:
15714 return (VT == MVT::f32) || (VT == MVT::f16 && Subtarget.hasMin3Max3_16()) ||
15715 (VT == MVT::v2f16 && Subtarget.hasMin3Max3PKF16());
15716 case ISD::FMINIMUM:
15717 case ISD::FMAXIMUM:
15718 return (VT == MVT::f32 && Subtarget.hasMinimum3Maximum3F32()) ||
15719 (VT == MVT::f16 && Subtarget.hasMinimum3Maximum3F16()) ||
15720 (VT == MVT::v2f16 && Subtarget.hasMinimum3Maximum3PKF16());
15721 case ISD::SMAX:
15722 case ISD::SMIN:
15723 case ISD::UMAX:
15724 case ISD::UMIN:
15725 return (VT == MVT::i32) || (VT == MVT::i16 && Subtarget.hasMin3Max3_16());
15726 default:
15727 return false;
15728 }
15729
15730 llvm_unreachable("not a min/max opcode");
15731}
15732
15733SDValue SITargetLowering::performMinMaxCombine(SDNode *N,
15734 DAGCombinerInfo &DCI) const {
15735 SelectionDAG &DAG = DCI.DAG;
15736
15737 EVT VT = N->getValueType(ResNo: 0);
15738 unsigned Opc = N->getOpcode();
15739 SDValue Op0 = N->getOperand(Num: 0);
15740 SDValue Op1 = N->getOperand(Num: 1);
15741
15742 // Only do this if the inner op has one use since this will just increases
15743 // register pressure for no benefit.
15744
15745 if (supportsMin3Max3(Subtarget: *Subtarget, Opc, VT)) {
15746 // max(max(a, b), c) -> max3(a, b, c)
15747 // min(min(a, b), c) -> min3(a, b, c)
15748 if (Op0.getOpcode() == Opc && Op0.hasOneUse()) {
15749 SDLoc DL(N);
15750 return DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT: N->getValueType(ResNo: 0),
15751 N1: Op0.getOperand(i: 0), N2: Op0.getOperand(i: 1), N3: Op1);
15752 }
15753
15754 // Try commuted.
15755 // max(a, max(b, c)) -> max3(a, b, c)
15756 // min(a, min(b, c)) -> min3(a, b, c)
15757 if (Op1.getOpcode() == Opc && Op1.hasOneUse()) {
15758 SDLoc DL(N);
15759 return DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT: N->getValueType(ResNo: 0),
15760 N1: Op0, N2: Op1.getOperand(i: 0), N3: Op1.getOperand(i: 1));
15761 }
15762 }
15763
15764 // umin(sffbh(x), bitwidth) -> sffbh(x) if x is known to be not 0 or -1.
15765 SDValue FfbhSrc;
15766 uint64_t Clamp = 0;
15767 if (Opc == ISD::UMIN &&
15768 sd_match(N: Op0,
15769 P: m_IntrinsicWOChain<Intrinsic::amdgcn_sffbh>(Opnds: m_Value(N&: FfbhSrc))) &&
15770 sd_match(N: Op1, P: m_ConstInt(V&: Clamp))) {
15771 unsigned BitWidth = FfbhSrc.getValueType().getScalarSizeInBits();
15772 if (Clamp >= BitWidth) {
15773 KnownBits Known = DAG.computeKnownBits(Op: FfbhSrc);
15774 if (Known.isNonZero() && !Known.isAllOnes())
15775 return Op0;
15776 }
15777 }
15778
15779 // min(max(x, K0), K1), K0 < K1 -> med3(x, K0, K1)
15780 // max(min(x, K0), K1), K1 < K0 -> med3(x, K1, K0)
15781 if (Opc == ISD::SMIN && Op0.getOpcode() == ISD::SMAX && Op0.hasOneUse()) {
15782 if (SDValue Med3 = performIntMed3ImmCombine(
15783 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op1, MaxVal: Op0->getOperand(Num: 1), Signed: true))
15784 return Med3;
15785 }
15786 if (Opc == ISD::SMAX && Op0.getOpcode() == ISD::SMIN && Op0.hasOneUse()) {
15787 if (SDValue Med3 = performIntMed3ImmCombine(
15788 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op0->getOperand(Num: 1), MaxVal: Op1, Signed: true))
15789 return Med3;
15790 }
15791
15792 if (Opc == ISD::UMIN && Op0.getOpcode() == ISD::UMAX && Op0.hasOneUse()) {
15793 if (SDValue Med3 = performIntMed3ImmCombine(
15794 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op1, MaxVal: Op0->getOperand(Num: 1), Signed: false))
15795 return Med3;
15796 }
15797 if (Opc == ISD::UMAX && Op0.getOpcode() == ISD::UMIN && Op0.hasOneUse()) {
15798 if (SDValue Med3 = performIntMed3ImmCombine(
15799 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op0->getOperand(Num: 1), MaxVal: Op1, Signed: false))
15800 return Med3;
15801 }
15802
15803 // if !is_snan(x):
15804 // fminnum(fmaxnum(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
15805 // fminnum_ieee(fmaxnum_ieee(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
15806 // fminnumnum(fmaxnumnum(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
15807 // fmin_legacy(fmax_legacy(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
15808 if (((Opc == ISD::FMINNUM && Op0.getOpcode() == ISD::FMAXNUM) ||
15809 (Opc == ISD::FMINNUM_IEEE && Op0.getOpcode() == ISD::FMAXNUM_IEEE) ||
15810 (Opc == ISD::FMINIMUMNUM && Op0.getOpcode() == ISD::FMAXIMUMNUM) ||
15811 (Opc == AMDGPUISD::FMIN_LEGACY &&
15812 Op0.getOpcode() == AMDGPUISD::FMAX_LEGACY)) &&
15813 (VT == MVT::f32 || VT == MVT::f64 ||
15814 (VT == MVT::f16 && Subtarget->has16BitInsts()) ||
15815 (VT == MVT::bf16 && Subtarget->hasBF16PackedInsts()) ||
15816 (VT == MVT::v2bf16 && Subtarget->hasBF16PackedInsts()) ||
15817 (VT == MVT::v2f16 && Subtarget->hasVOP3PInsts())) &&
15818 Op0.hasOneUse()) {
15819 if (SDValue Res = performFPMed3ImmCombine(DAG, SL: SDLoc(N), Op0, Op1,
15820 IsKnownNoNaNs: N->getFlags().hasNoNaNs()))
15821 return Res;
15822 }
15823
15824 // Prefer fminnum_ieee over fminimum. For gfx950, minimum/maximum are legal
15825 // for some types, but at a higher cost since it's implemented with a 3
15826 // operand form.
15827 const SDNodeFlags Flags = N->getFlags();
15828 if ((Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM) && Flags.hasNoNaNs() &&
15829 !Subtarget->hasIEEEMinimumMaximumInsts() &&
15830 isOperationLegal(Op: ISD::FMINNUM_IEEE, VT: VT.getScalarType())) {
15831 unsigned NewOpc =
15832 Opc == ISD::FMINIMUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
15833 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(N), VT, N1: Op0, N2: Op1, Flags);
15834 }
15835
15836 return SDValue();
15837}
15838
15839static bool isClampZeroToOne(SDValue A, SDValue B) {
15840 if (ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A)) {
15841 if (ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B)) {
15842 // FIXME: Should this be allowing -0.0?
15843 return (CA->isExactlyValue(V: 0.0) && CB->isExactlyValue(V: 1.0)) ||
15844 (CA->isExactlyValue(V: 1.0) && CB->isExactlyValue(V: 0.0));
15845 }
15846 }
15847
15848 return false;
15849}
15850
15851// FIXME: Should only worry about snans for version with chain.
15852SDValue SITargetLowering::performFMed3Combine(SDNode *N,
15853 DAGCombinerInfo &DCI) const {
15854 EVT VT = N->getValueType(ResNo: 0);
15855 // v_med3_f32 and v_max_f32 behave identically wrt denorms, exceptions and
15856 // NaNs. With a NaN input, the order of the operands may change the result.
15857
15858 SelectionDAG &DAG = DCI.DAG;
15859 SDLoc SL(N);
15860
15861 SDValue Src0 = N->getOperand(Num: 0);
15862 SDValue Src1 = N->getOperand(Num: 1);
15863 SDValue Src2 = N->getOperand(Num: 2);
15864
15865 if (isClampZeroToOne(A: Src0, B: Src1)) {
15866 // const_a, const_b, x -> clamp is safe in all cases including signaling
15867 // nans.
15868 // FIXME: Should this be allowing -0.0?
15869 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Src2);
15870 }
15871
15872 const MachineFunction &MF = DAG.getMachineFunction();
15873 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
15874
15875 // FIXME: dx10_clamp behavior assumed in instcombine. Should we really bother
15876 // handling no dx10-clamp?
15877 if (Info->getMode().DX10Clamp) {
15878 // If NaNs is clamped to 0, we are free to reorder the inputs.
15879
15880 if (isa<ConstantFPSDNode>(Val: Src0) && !isa<ConstantFPSDNode>(Val: Src1))
15881 std::swap(a&: Src0, b&: Src1);
15882
15883 if (isa<ConstantFPSDNode>(Val: Src1) && !isa<ConstantFPSDNode>(Val: Src2))
15884 std::swap(a&: Src1, b&: Src2);
15885
15886 if (isa<ConstantFPSDNode>(Val: Src0) && !isa<ConstantFPSDNode>(Val: Src1))
15887 std::swap(a&: Src0, b&: Src1);
15888
15889 if (isClampZeroToOne(A: Src1, B: Src2))
15890 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Src0);
15891 }
15892
15893 return SDValue();
15894}
15895
15896SDValue SITargetLowering::performCvtPkRTZCombine(SDNode *N,
15897 DAGCombinerInfo &DCI) const {
15898 SDValue Src0 = N->getOperand(Num: 0);
15899 SDValue Src1 = N->getOperand(Num: 1);
15900 if (Src0.isUndef() && Src1.isUndef())
15901 return DCI.DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
15902 return SDValue();
15903}
15904
15905// Check if EXTRACT_VECTOR_ELT/INSERT_VECTOR_ELT (<n x e>, var-idx) should be
15906// expanded into a set of cmp/select instructions.
15907bool SITargetLowering::shouldExpandVectorDynExt(unsigned EltSize,
15908 unsigned NumElem,
15909 bool IsDivergentIdx,
15910 const GCNSubtarget *Subtarget) {
15911 if (UseDivergentRegisterIndexing)
15912 return false;
15913
15914 unsigned VecSize = EltSize * NumElem;
15915
15916 // Sub-dword vectors of size 2 dword or less have better implementation.
15917 if (VecSize <= 64 && EltSize < 32)
15918 return false;
15919
15920 // Always expand the rest of sub-dword instructions, otherwise it will be
15921 // lowered via memory.
15922 if (EltSize < 32)
15923 return true;
15924
15925 // Always do this if var-idx is divergent, otherwise it will become a loop.
15926 if (IsDivergentIdx)
15927 return true;
15928
15929 // Large vectors would yield too many compares and v_cndmask_b32 instructions.
15930 unsigned NumInsts = NumElem /* Number of compares */ +
15931 ((EltSize + 31) / 32) * NumElem /* Number of cndmasks */;
15932
15933 // On some architectures (GFX9) movrel is not available and it's better
15934 // to expand.
15935 if (Subtarget->useVGPRIndexMode())
15936 return NumInsts <= 16;
15937
15938 // If movrel is available, use it instead of expanding for vector of 8
15939 // elements.
15940 if (Subtarget->hasMovrel())
15941 return NumInsts <= 15;
15942
15943 return true;
15944}
15945
15946bool SITargetLowering::shouldExpandVectorDynExt(SDNode *N) const {
15947 SDValue Idx = N->getOperand(Num: N->getNumOperands() - 1);
15948 if (isa<ConstantSDNode>(Val: Idx))
15949 return false;
15950
15951 SDValue Vec = N->getOperand(Num: 0);
15952 EVT VecVT = Vec.getValueType();
15953 EVT EltVT = VecVT.getVectorElementType();
15954 unsigned EltSize = EltVT.getSizeInBits();
15955 unsigned NumElem = VecVT.getVectorNumElements();
15956
15957 return SITargetLowering::shouldExpandVectorDynExt(
15958 EltSize, NumElem, IsDivergentIdx: Idx->isDivergent(), Subtarget: getSubtarget());
15959}
15960
15961SDValue
15962SITargetLowering::performExtractVectorEltCombine(SDNode *N,
15963 DAGCombinerInfo &DCI) const {
15964 SDValue Vec = N->getOperand(Num: 0);
15965 SelectionDAG &DAG = DCI.DAG;
15966
15967 EVT VecVT = Vec.getValueType();
15968 EVT VecEltVT = VecVT.getVectorElementType();
15969 EVT ResVT = N->getValueType(ResNo: 0);
15970
15971 unsigned VecSize = VecVT.getSizeInBits();
15972 unsigned VecEltSize = VecEltVT.getSizeInBits();
15973
15974 if ((Vec.getOpcode() == ISD::FNEG || Vec.getOpcode() == ISD::FABS) &&
15975 allUsesHaveSourceMods(N)) {
15976 SDLoc SL(N);
15977 SDValue Idx = N->getOperand(Num: 1);
15978 SDValue Elt =
15979 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT, N1: Vec.getOperand(i: 0), N2: Idx);
15980 return DAG.getNode(Opcode: Vec.getOpcode(), DL: SL, VT: ResVT, Operand: Elt);
15981 }
15982
15983 // (extract_vector_element (and {y0, y1}, (build_vector 0x1f, 0x1f)), index)
15984 // -> (and (extract_vector_element {y0, y1}, index), 0x1f)
15985 // There are optimisations to transform 64-bit shifts into 32-bit shifts
15986 // depending on the shift operand. See e.g. performSraCombine().
15987 // This combine ensures that the optimisation is compatible with v2i32
15988 // legalised AND.
15989 if (VecVT == MVT::v2i32 && Vec->getOpcode() == ISD::AND &&
15990 Vec->getOperand(Num: 1)->getOpcode() == ISD::BUILD_VECTOR) {
15991
15992 const ConstantSDNode *C = isConstOrConstSplat(N: Vec.getOperand(i: 1));
15993 if (!C || C->getZExtValue() != 0x1f)
15994 return SDValue();
15995
15996 SDLoc SL(N);
15997 SDValue AndMask = DAG.getConstant(Val: 0x1f, DL: SL, VT: MVT::i32);
15998 SDValue EVE = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32,
15999 N1: Vec->getOperand(Num: 0), N2: N->getOperand(Num: 1));
16000 SDValue A = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: EVE, N2: AndMask);
16001 DAG.ReplaceAllUsesWith(From: N, To: A.getNode());
16002 }
16003
16004 // ScalarRes = EXTRACT_VECTOR_ELT ((vector-BINOP Vec1, Vec2), Idx)
16005 // =>
16006 // Vec1Elt = EXTRACT_VECTOR_ELT(Vec1, Idx)
16007 // Vec2Elt = EXTRACT_VECTOR_ELT(Vec2, Idx)
16008 // ScalarRes = scalar-BINOP Vec1Elt, Vec2Elt
16009 if (Vec.hasOneUse() && DCI.isBeforeLegalize() && VecEltVT == ResVT) {
16010 SDLoc SL(N);
16011 SDValue Idx = N->getOperand(Num: 1);
16012 unsigned Opc = Vec.getOpcode();
16013
16014 switch (Opc) {
16015 default:
16016 break;
16017 // TODO: Support other binary operations.
16018 case ISD::FADD:
16019 case ISD::FSUB:
16020 case ISD::FMUL:
16021 case ISD::ADD:
16022 case ISD::UMIN:
16023 case ISD::UMAX:
16024 case ISD::SMIN:
16025 case ISD::SMAX:
16026 case ISD::FMAXNUM:
16027 case ISD::FMINNUM:
16028 case ISD::FMAXNUM_IEEE:
16029 case ISD::FMINNUM_IEEE:
16030 case ISD::FMAXIMUM:
16031 case ISD::FMINIMUM: {
16032 SDValue Elt0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT,
16033 N1: Vec.getOperand(i: 0), N2: Idx);
16034 SDValue Elt1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT,
16035 N1: Vec.getOperand(i: 1), N2: Idx);
16036
16037 DCI.AddToWorklist(N: Elt0.getNode());
16038 DCI.AddToWorklist(N: Elt1.getNode());
16039 return DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT, N1: Elt0, N2: Elt1, Flags: Vec->getFlags());
16040 }
16041 }
16042 }
16043
16044 // EXTRACT_VECTOR_ELT (<n x e>, var-idx) => n x select (e, const-idx)
16045 if (shouldExpandVectorDynExt(N)) {
16046 SDLoc SL(N);
16047 SDValue Idx = N->getOperand(Num: 1);
16048 SDValue V;
16049 for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
16050 SDValue IC = DAG.getVectorIdxConstant(Val: I, DL: SL);
16051 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT, N1: Vec, N2: IC);
16052 if (I == 0)
16053 V = Elt;
16054 else
16055 V = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IC, True: Elt, False: V, Cond: ISD::SETEQ);
16056 }
16057 return V;
16058 }
16059
16060 // EXTRACT_VECTOR_ELT (v2i32 bitcast (i64/f64:k), Idx)
16061 // =>
16062 // i32:Lo(k) if Idx == 0, or
16063 // i32:Hi(k) if Idx == 1
16064 auto *Idx = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
16065 if (Vec.getOpcode() == ISD::BITCAST && VecVT == MVT::v2i32 && Idx) {
16066 SDLoc SL(N);
16067 SDValue PeekThrough = Vec.getOperand(i: 0);
16068 auto *KImm = dyn_cast<ConstantSDNode>(Val&: PeekThrough);
16069 if (KImm && KImm->getValueType(ResNo: 0).getSizeInBits() == 64) {
16070 uint64_t KImmValue = KImm->getZExtValue();
16071 return DAG.getConstant(
16072 Val: (KImmValue >> (32 * Idx->getZExtValue())) & 0xffffffff, DL: SL, VT: MVT::i32);
16073 }
16074 auto *KFPImm = dyn_cast<ConstantFPSDNode>(Val&: PeekThrough);
16075 if (KFPImm && KFPImm->getValueType(ResNo: 0).getSizeInBits() == 64) {
16076 uint64_t KFPImmValue =
16077 KFPImm->getValueAPF().bitcastToAPInt().getZExtValue();
16078 return DAG.getConstant(Val: (KFPImmValue >> (32 * Idx->getZExtValue())) &
16079 0xffffffff,
16080 DL: SL, VT: MVT::i32);
16081 }
16082 }
16083
16084 if (!DCI.isBeforeLegalize())
16085 return SDValue();
16086
16087 // Try to turn sub-dword accesses of vectors into accesses of the same 32-bit
16088 // elements. This exposes more load reduction opportunities by replacing
16089 // multiple small extract_vector_elements with a single 32-bit extract.
16090 if (isa<MemSDNode>(Val: Vec) && VecEltSize <= 16 && VecEltVT.isByteSized() &&
16091 VecSize > 32 && VecSize % 32 == 0 && Idx) {
16092 EVT NewVT = getEquivalentMemType(Context&: *DAG.getContext(), VT: VecVT);
16093
16094 unsigned BitIndex = Idx->getZExtValue() * VecEltSize;
16095 unsigned EltIdx = BitIndex / 32;
16096 unsigned LeftoverBitIdx = BitIndex % 32;
16097 SDLoc SL(N);
16098
16099 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: Vec);
16100 DCI.AddToWorklist(N: Cast.getNode());
16101
16102 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Cast,
16103 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
16104 DCI.AddToWorklist(N: Elt.getNode());
16105 SDValue Srl = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: Elt,
16106 N2: DAG.getConstant(Val: LeftoverBitIdx, DL: SL, VT: MVT::i32));
16107 DCI.AddToWorklist(N: Srl.getNode());
16108
16109 EVT VecEltAsIntVT = VecEltVT.changeTypeToInteger();
16110 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: VecEltAsIntVT, Operand: Srl);
16111 DCI.AddToWorklist(N: Trunc.getNode());
16112
16113 if (VecEltVT == ResVT) {
16114 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecEltVT, Operand: Trunc);
16115 }
16116
16117 assert(ResVT.isScalarInteger());
16118 return DAG.getAnyExtOrTrunc(Op: Trunc, DL: SL, VT: ResVT);
16119 }
16120
16121 return SDValue();
16122}
16123
16124SDValue
16125SITargetLowering::performInsertVectorEltCombine(SDNode *N,
16126 DAGCombinerInfo &DCI) const {
16127 SDValue Vec = N->getOperand(Num: 0);
16128 SDValue Idx = N->getOperand(Num: 2);
16129 EVT VecVT = Vec.getValueType();
16130 EVT EltVT = VecVT.getVectorElementType();
16131
16132 // INSERT_VECTOR_ELT (<n x e>, var-idx)
16133 // => BUILD_VECTOR n x select (e, const-idx)
16134 if (!shouldExpandVectorDynExt(N))
16135 return SDValue();
16136
16137 SelectionDAG &DAG = DCI.DAG;
16138 SDLoc SL(N);
16139 SDValue Ins = N->getOperand(Num: 1);
16140 EVT IdxVT = Idx.getValueType();
16141
16142 SmallVector<SDValue, 16> Ops;
16143 for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
16144 SDValue IC = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
16145 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec, N2: IC);
16146 SDValue V = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IC, True: Ins, False: Elt, Cond: ISD::SETEQ);
16147 Ops.push_back(Elt: V);
16148 }
16149
16150 return DAG.getBuildVector(VT: VecVT, DL: SL, Ops);
16151}
16152
16153/// Return the source of an fp_extend from f16 to f32, or a converted FP
16154/// constant.
16155static SDValue strictFPExtFromF16(SelectionDAG &DAG, SDValue Src) {
16156 if (Src.getOpcode() == ISD::FP_EXTEND &&
16157 Src.getOperand(i: 0).getValueType() == MVT::f16) {
16158 return Src.getOperand(i: 0);
16159 }
16160
16161 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Src)) {
16162 APFloat Val = CFP->getValueAPF();
16163 bool LosesInfo = true;
16164 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
16165 if (!LosesInfo)
16166 return DAG.getConstantFP(Val, DL: SDLoc(Src), VT: MVT::f16);
16167 }
16168
16169 return SDValue();
16170}
16171
16172SDValue SITargetLowering::performFPRoundCombine(SDNode *N,
16173 DAGCombinerInfo &DCI) const {
16174 assert(Subtarget->has16BitInsts() && !Subtarget->hasMed3_16() &&
16175 "combine only useful on gfx8");
16176
16177 SDValue TruncSrc = N->getOperand(Num: 0);
16178 EVT VT = N->getValueType(ResNo: 0);
16179 if (VT != MVT::f16)
16180 return SDValue();
16181
16182 if (TruncSrc.getOpcode() != AMDGPUISD::FMED3 ||
16183 TruncSrc.getValueType() != MVT::f32 || !TruncSrc.hasOneUse())
16184 return SDValue();
16185
16186 SelectionDAG &DAG = DCI.DAG;
16187 SDLoc SL(N);
16188
16189 // Optimize f16 fmed3 pattern performed on f32. On gfx8 there is no f16 fmed3,
16190 // and expanding it with min/max saves 1 instruction vs. casting to f32 and
16191 // casting back.
16192
16193 // fptrunc (f32 (fmed3 (fpext f16:a, fpext f16:b, fpext f16:c))) =>
16194 // fmin(fmax(a, b), fmax(fmin(a, b), c))
16195 SDValue A = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 0));
16196 if (!A)
16197 return SDValue();
16198
16199 SDValue B = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 1));
16200 if (!B)
16201 return SDValue();
16202
16203 SDValue C = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 2));
16204 if (!C)
16205 return SDValue();
16206
16207 // This changes signaling nan behavior. If an input is a signaling nan, it
16208 // would have been quieted by the fpext originally. We don't care because
16209 // these are unconstrained ops. If we needed to insert quieting canonicalizes
16210 // we would be worse off than just doing the promotion.
16211 SDValue A1 = DAG.getNode(Opcode: ISD::FMINNUM_IEEE, DL: SL, VT, N1: A, N2: B);
16212 SDValue B1 = DAG.getNode(Opcode: ISD::FMAXNUM_IEEE, DL: SL, VT, N1: A, N2: B);
16213 SDValue C1 = DAG.getNode(Opcode: ISD::FMAXNUM_IEEE, DL: SL, VT, N1: A1, N2: C);
16214 return DAG.getNode(Opcode: ISD::FMINNUM_IEEE, DL: SL, VT, N1: B1, N2: C1);
16215}
16216
16217unsigned SITargetLowering::getFusedOpcode(const SelectionDAG &DAG,
16218 const SDNode *N0,
16219 const SDNode *N1) const {
16220 EVT VT = N0->getValueType(ResNo: 0);
16221
16222 // Only do this if we are not trying to support denormals. v_mad_f32 does not
16223 // support denormals ever.
16224 if (((VT == MVT::f32 &&
16225 denormalModeIsFlushAllF32(MF: DAG.getMachineFunction())) ||
16226 (VT == MVT::f16 && Subtarget->hasMadF16() &&
16227 denormalModeIsFlushAllF64F16(MF: DAG.getMachineFunction()))) &&
16228 isOperationLegal(Op: ISD::FMAD, VT))
16229 return ISD::FMAD;
16230
16231 const TargetOptions &Options = DAG.getTarget().Options;
16232 if ((Options.AllowFPOpFusion == FPOpFusion::Fast ||
16233 (N0->getFlags().hasAllowContract() &&
16234 N1->getFlags().hasAllowContract())) &&
16235 isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
16236 return ISD::FMA;
16237 }
16238
16239 return 0;
16240}
16241
16242// For a reassociatable opcode perform:
16243// op x, (op y, z) -> op (op x, z), y, if x and z are uniform
16244SDValue SITargetLowering::reassociateScalarOps(SDNode *N,
16245 SelectionDAG &DAG) const {
16246 EVT VT = N->getValueType(ResNo: 0);
16247 if (VT != MVT::i32 && VT != MVT::i64)
16248 return SDValue();
16249
16250 if (DAG.isBaseWithConstantOffset(Op: SDValue(N, 0)))
16251 return SDValue();
16252
16253 unsigned Opc = N->getOpcode();
16254 SDValue Op0 = N->getOperand(Num: 0);
16255 SDValue Op1 = N->getOperand(Num: 1);
16256
16257 if (!(Op0->isDivergent() ^ Op1->isDivergent()))
16258 return SDValue();
16259
16260 if (Op0->isDivergent())
16261 std::swap(a&: Op0, b&: Op1);
16262
16263 if (Op1.getOpcode() != Opc || !Op1.hasOneUse())
16264 return SDValue();
16265
16266 SDValue Op2 = Op1.getOperand(i: 1);
16267 Op1 = Op1.getOperand(i: 0);
16268 if (!(Op1->isDivergent() ^ Op2->isDivergent()))
16269 return SDValue();
16270
16271 if (Op1->isDivergent())
16272 std::swap(a&: Op1, b&: Op2);
16273
16274 SDLoc SL(N);
16275 SDValue Add1 = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: Op0, N2: Op1);
16276 return DAG.getNode(Opcode: Opc, DL: SL, VT, N1: Add1, N2: Op2);
16277}
16278
16279static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL, EVT VT,
16280 SDValue N0, SDValue N1, SDValue N2, bool Signed) {
16281 unsigned MadOpc = Signed ? AMDGPUISD::MAD_I64_I32 : AMDGPUISD::MAD_U64_U32;
16282 SDVTList VTs = DAG.getVTList(VT1: MVT::i64, VT2: MVT::i1);
16283 SDValue Mad = DAG.getNode(Opcode: MadOpc, DL: SL, VTList: VTs, N1: N0, N2: N1, N3: N2);
16284 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Mad);
16285}
16286
16287// Fold
16288// y = lshr i64 x, 32
16289// res = add (mul i64 y, Const), x where "Const" is a 64-bit constant
16290// with Const.hi == -1
16291// To
16292// res = mad_u64_u32 y.lo ,Const.lo, x.lo
16293static SDValue tryFoldMADwithSRL(SelectionDAG &DAG, const SDLoc &SL,
16294 SDValue MulLHS, SDValue MulRHS,
16295 SDValue AddRHS) {
16296 if (MulRHS.getOpcode() == ISD::SRL)
16297 std::swap(a&: MulLHS, b&: MulRHS);
16298
16299 if (MulLHS.getValueType() != MVT::i64 || MulLHS.getOpcode() != ISD::SRL)
16300 return SDValue();
16301
16302 ConstantSDNode *ShiftVal = dyn_cast<ConstantSDNode>(Val: MulLHS.getOperand(i: 1));
16303 if (!ShiftVal || ShiftVal->getAsZExtVal() != 32 ||
16304 MulLHS.getOperand(i: 0) != AddRHS)
16305 return SDValue();
16306
16307 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val: MulRHS.getNode());
16308 if (!Const || Hi_32(Value: Const->getZExtValue()) != uint32_t(-1))
16309 return SDValue();
16310
16311 SDValue ConstMul =
16312 DAG.getConstant(Val: Lo_32(Value: Const->getZExtValue()), DL: SL, VT: MVT::i32);
16313 return getMad64_32(DAG, SL, VT: MVT::i64,
16314 N0: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulLHS), N1: ConstMul,
16315 N2: DAG.getZeroExtendInReg(Op: AddRHS, DL: SL, VT: MVT::i32), Signed: false);
16316}
16317
16318// Fold (add (mul x, y), z) --> (mad_[iu]64_[iu]32 x, y, z) plus high
16319// multiplies, if any.
16320//
16321// Full 64-bit multiplies that feed into an addition are lowered here instead
16322// of using the generic expansion. The generic expansion ends up with
16323// a tree of ADD nodes that prevents us from using the "add" part of the
16324// MAD instruction. The expansion produced here results in a chain of ADDs
16325// instead of a tree.
16326SDValue SITargetLowering::tryFoldToMad64_32(SDNode *N,
16327 DAGCombinerInfo &DCI) const {
16328 assert(N->isAnyAdd());
16329
16330 SelectionDAG &DAG = DCI.DAG;
16331 EVT VT = N->getValueType(ResNo: 0);
16332 SDLoc SL(N);
16333 SDValue LHS = N->getOperand(Num: 0);
16334 SDValue RHS = N->getOperand(Num: 1);
16335
16336 if (VT.isVector())
16337 return SDValue();
16338
16339 // S_MUL_HI_[IU]32 was added in gfx9, which allows us to keep the overall
16340 // result in scalar registers for uniform values.
16341 if (!N->isDivergent() && Subtarget->hasSMulHi())
16342 return SDValue();
16343
16344 unsigned NumBits = VT.getScalarSizeInBits();
16345 if (NumBits <= 32 || NumBits > 64)
16346 return SDValue();
16347
16348 if (LHS.getOpcode() != ISD::MUL) {
16349 assert(RHS.getOpcode() == ISD::MUL);
16350 std::swap(a&: LHS, b&: RHS);
16351 }
16352
16353 // Avoid the fold if it would unduly increase the number of multiplies due to
16354 // multiple uses, except on hardware with full-rate multiply-add (which is
16355 // part of full-rate 64-bit ops).
16356 if (!Subtarget->hasFullRate64Ops()) {
16357 unsigned NumUsers = 0;
16358 for (SDNode *User : LHS->users()) {
16359 // There is a use that does not feed into addition, so the multiply can't
16360 // be removed. We prefer MUL + ADD + ADDC over MAD + MUL.
16361 if (!User->isAnyAdd())
16362 return SDValue();
16363
16364 // We prefer 2xMAD over MUL + 2xADD + 2xADDC (code density), and prefer
16365 // MUL + 3xADD + 3xADDC over 3xMAD.
16366 ++NumUsers;
16367 if (NumUsers >= 3)
16368 return SDValue();
16369 }
16370 }
16371
16372 SDValue MulLHS = LHS.getOperand(i: 0);
16373 SDValue MulRHS = LHS.getOperand(i: 1);
16374 SDValue AddRHS = RHS;
16375
16376 if (SDValue FoldedMAD = tryFoldMADwithSRL(DAG, SL, MulLHS, MulRHS, AddRHS))
16377 return FoldedMAD;
16378
16379 // Always check whether operands are small unsigned values, since that
16380 // knowledge is useful in more cases. Check for small signed values only if
16381 // doing so can unlock a shorter code sequence.
16382 bool MulLHSUnsigned32 = numBitsUnsigned(Op: MulLHS, DAG) <= 32;
16383 bool MulRHSUnsigned32 = numBitsUnsigned(Op: MulRHS, DAG) <= 32;
16384
16385 bool MulSignedLo = false;
16386 if (!MulLHSUnsigned32 || !MulRHSUnsigned32) {
16387 MulSignedLo =
16388 numBitsSigned(Op: MulLHS, DAG) <= 32 && numBitsSigned(Op: MulRHS, DAG) <= 32;
16389 }
16390
16391 // The operands and final result all have the same number of bits. If
16392 // operands need to be extended, they can be extended with garbage. The
16393 // resulting garbage in the high bits of the mad_[iu]64_[iu]32 result is
16394 // truncated away in the end.
16395 if (VT != MVT::i64) {
16396 MulLHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: MulLHS);
16397 MulRHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: MulRHS);
16398 AddRHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: AddRHS);
16399 }
16400
16401 // The basic code generated is conceptually straightforward. Pseudo code:
16402 //
16403 // accum = mad_64_32 lhs.lo, rhs.lo, accum
16404 // accum.hi = add (mul lhs.hi, rhs.lo), accum.hi
16405 // accum.hi = add (mul lhs.lo, rhs.hi), accum.hi
16406 //
16407 // The second and third lines are optional, depending on whether the factors
16408 // are {sign,zero}-extended or not.
16409 //
16410 // The actual DAG is noisier than the pseudo code, but only due to
16411 // instructions that disassemble values into low and high parts, and
16412 // assemble the final result.
16413 SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
16414
16415 auto MulLHSLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulLHS);
16416 auto MulRHSLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulRHS);
16417 SDValue Accum =
16418 getMad64_32(DAG, SL, VT: MVT::i64, N0: MulLHSLo, N1: MulRHSLo, N2: AddRHS, Signed: MulSignedLo);
16419
16420 if (!MulSignedLo && (!MulLHSUnsigned32 || !MulRHSUnsigned32)) {
16421 auto [AccumLo, AccumHi] = DAG.SplitScalar(N: Accum, DL: SL, LoVT: MVT::i32, HiVT: MVT::i32);
16422
16423 if (!MulLHSUnsigned32) {
16424 auto MulLHSHi =
16425 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SL, VT: MVT::i32, N1: MulLHS, N2: One);
16426 SDValue MulHi = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: MVT::i32, N1: MulLHSHi, N2: MulRHSLo);
16427 AccumHi = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: MulHi, N2: AccumHi);
16428 }
16429
16430 if (!MulRHSUnsigned32) {
16431 auto MulRHSHi =
16432 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SL, VT: MVT::i32, N1: MulRHS, N2: One);
16433 SDValue MulHi = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: MVT::i32, N1: MulLHSLo, N2: MulRHSHi);
16434 AccumHi = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: MulHi, N2: AccumHi);
16435 }
16436
16437 Accum = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {AccumLo, AccumHi});
16438 Accum = DAG.getBitcast(VT: MVT::i64, V: Accum);
16439 }
16440
16441 if (VT != MVT::i64)
16442 Accum = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Accum);
16443 return Accum;
16444}
16445
16446SDValue
16447SITargetLowering::foldAddSub64WithZeroLowBitsTo32(SDNode *N,
16448 DAGCombinerInfo &DCI) const {
16449 SDValue RHS = N->getOperand(Num: 1);
16450 auto *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
16451 if (!CRHS)
16452 return SDValue();
16453
16454 // TODO: Worth using computeKnownBits? Maybe expensive since it's so
16455 // common.
16456 uint64_t Val = CRHS->getZExtValue();
16457 if (countr_zero(Val) >= 32) {
16458 SelectionDAG &DAG = DCI.DAG;
16459 SDLoc SL(N);
16460 SDValue LHS = N->getOperand(Num: 0);
16461
16462 // Avoid carry machinery if we know the low half of the add does not
16463 // contribute to the final result.
16464 //
16465 // add i64:x, K if computeTrailingZeros(K) >= 32
16466 // => build_pair (add x.hi, K.hi), x.lo
16467
16468 // Breaking the 64-bit add here with this strange constant is unlikely
16469 // to interfere with addressing mode patterns.
16470
16471 SDValue Hi = getHiHalf64(Op: LHS, DAG);
16472 SDValue ConstHi32 = DAG.getConstant(Val: Hi_32(Value: Val), DL: SL, VT: MVT::i32);
16473 unsigned Opcode = N->getOpcode();
16474 if (Opcode == ISD::PTRADD)
16475 Opcode = ISD::ADD;
16476 SDValue AddHi =
16477 DAG.getNode(Opcode, DL: SL, VT: MVT::i32, N1: Hi, N2: ConstHi32, Flags: N->getFlags());
16478
16479 SDValue Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: LHS);
16480 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i64, N1: Lo, N2: AddHi);
16481 }
16482
16483 return SDValue();
16484}
16485
16486// Collect the ultimate src of each of the mul node's operands, and confirm
16487// each operand is 8 bytes.
16488static std::optional<ByteProvider<SDValue>>
16489handleMulOperand(const SDValue &MulOperand) {
16490 auto Byte0 = calculateByteProvider(Op: MulOperand, Index: 0, Depth: 0);
16491 if (!Byte0 || Byte0->isConstantZero()) {
16492 return std::nullopt;
16493 }
16494 auto Byte1 = calculateByteProvider(Op: MulOperand, Index: 1, Depth: 0);
16495 if (Byte1 && !Byte1->isConstantZero()) {
16496 return std::nullopt;
16497 }
16498 return Byte0;
16499}
16500
16501static unsigned addPermMasks(unsigned First, unsigned Second) {
16502 unsigned FirstCs = First & 0x0c0c0c0c;
16503 unsigned SecondCs = Second & 0x0c0c0c0c;
16504 unsigned FirstNoCs = First & ~0x0c0c0c0c;
16505 unsigned SecondNoCs = Second & ~0x0c0c0c0c;
16506
16507 assert((FirstCs & 0xFF) | (SecondCs & 0xFF));
16508 assert((FirstCs & 0xFF00) | (SecondCs & 0xFF00));
16509 assert((FirstCs & 0xFF0000) | (SecondCs & 0xFF0000));
16510 assert((FirstCs & 0xFF000000) | (SecondCs & 0xFF000000));
16511
16512 return (FirstNoCs | SecondNoCs) | (FirstCs & SecondCs);
16513}
16514
16515struct DotSrc {
16516 SDValue SrcOp;
16517 int64_t PermMask;
16518 int64_t DWordOffset;
16519};
16520
16521static void placeSources(ByteProvider<SDValue> &Src0,
16522 ByteProvider<SDValue> &Src1,
16523 SmallVectorImpl<DotSrc> &Src0s,
16524 SmallVectorImpl<DotSrc> &Src1s, int Step) {
16525
16526 assert(Src0.Src.has_value() && Src1.Src.has_value());
16527 // Src0s and Src1s are empty, just place arbitrarily.
16528 if (Step == 0) {
16529 Src0s.push_back(Elt: {.SrcOp: *Src0.Src, .PermMask: ((Src0.SrcOffset % 4) << 24) + 0x0c0c0c,
16530 .DWordOffset: Src0.SrcOffset / 4});
16531 Src1s.push_back(Elt: {.SrcOp: *Src1.Src, .PermMask: ((Src1.SrcOffset % 4) << 24) + 0x0c0c0c,
16532 .DWordOffset: Src1.SrcOffset / 4});
16533 return;
16534 }
16535
16536 for (int BPI = 0; BPI < 2; BPI++) {
16537 std::pair<ByteProvider<SDValue>, ByteProvider<SDValue>> BPP = {Src0, Src1};
16538 if (BPI == 1) {
16539 BPP = {Src1, Src0};
16540 }
16541 unsigned ZeroMask = 0x0c0c0c0c;
16542 unsigned FMask = 0xFF << (8 * (3 - Step));
16543
16544 unsigned FirstMask =
16545 (BPP.first.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
16546 unsigned SecondMask =
16547 (BPP.second.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
16548 // Attempt to find Src vector which contains our SDValue, if so, add our
16549 // perm mask to the existing one. If we are unable to find a match for the
16550 // first SDValue, attempt to find match for the second.
16551 int FirstGroup = -1;
16552 for (int I = 0; I < 2; I++) {
16553 SmallVectorImpl<DotSrc> &Srcs = I == 0 ? Src0s : Src1s;
16554 auto MatchesFirst = [&BPP](DotSrc &IterElt) {
16555 return IterElt.SrcOp == *BPP.first.Src &&
16556 (IterElt.DWordOffset == (BPP.first.SrcOffset / 4));
16557 };
16558
16559 auto *Match = llvm::find_if(Range&: Srcs, P: MatchesFirst);
16560 if (Match != Srcs.end()) {
16561 Match->PermMask = addPermMasks(First: FirstMask, Second: Match->PermMask);
16562 FirstGroup = I;
16563 break;
16564 }
16565 }
16566 if (FirstGroup != -1) {
16567 SmallVectorImpl<DotSrc> &Srcs = FirstGroup == 1 ? Src0s : Src1s;
16568 auto MatchesSecond = [&BPP](DotSrc &IterElt) {
16569 return IterElt.SrcOp == *BPP.second.Src &&
16570 (IterElt.DWordOffset == (BPP.second.SrcOffset / 4));
16571 };
16572 auto *Match = llvm::find_if(Range&: Srcs, P: MatchesSecond);
16573 if (Match != Srcs.end()) {
16574 Match->PermMask = addPermMasks(First: SecondMask, Second: Match->PermMask);
16575 } else
16576 Srcs.push_back(Elt: {.SrcOp: *BPP.second.Src, .PermMask: SecondMask, .DWordOffset: BPP.second.SrcOffset / 4});
16577 return;
16578 }
16579 }
16580
16581 // If we have made it here, then we could not find a match in Src0s or Src1s
16582 // for either Src0 or Src1, so just place them arbitrarily.
16583
16584 unsigned ZeroMask = 0x0c0c0c0c;
16585 unsigned FMask = 0xFF << (8 * (3 - Step));
16586
16587 Src0s.push_back(
16588 Elt: {.SrcOp: *Src0.Src,
16589 .PermMask: ((Src0.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
16590 .DWordOffset: Src0.SrcOffset / 4});
16591 Src1s.push_back(
16592 Elt: {.SrcOp: *Src1.Src,
16593 .PermMask: ((Src1.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
16594 .DWordOffset: Src1.SrcOffset / 4});
16595}
16596
16597static SDValue resolveSources(SelectionDAG &DAG, SDLoc SL,
16598 SmallVectorImpl<DotSrc> &Srcs, bool IsSigned,
16599 bool IsAny) {
16600
16601 // If we just have one source, just permute it accordingly.
16602 if (Srcs.size() == 1) {
16603 auto *Elt = Srcs.begin();
16604 auto EltOp = getDWordFromOffset(DAG, SL, Src: Elt->SrcOp, DWordOffset: Elt->DWordOffset);
16605
16606 // v_perm will produce the original value
16607 if (Elt->PermMask == 0x3020100)
16608 return EltOp;
16609
16610 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: EltOp, N2: EltOp,
16611 N3: DAG.getConstant(Val: Elt->PermMask, DL: SL, VT: MVT::i32));
16612 }
16613
16614 auto *FirstElt = Srcs.begin();
16615 auto *SecondElt = std::next(x: FirstElt);
16616
16617 SmallVector<SDValue, 2> Perms;
16618
16619 // If we have multiple sources in the chain, combine them via perms (using
16620 // calculated perm mask) and Ors.
16621 while (true) {
16622 auto FirstMask = FirstElt->PermMask;
16623 auto SecondMask = SecondElt->PermMask;
16624
16625 unsigned FirstCs = FirstMask & 0x0c0c0c0c;
16626 unsigned FirstPlusFour = FirstMask | 0x04040404;
16627 // 0x0c + 0x04 = 0x10, so anding with 0x0F will produced 0x00 for any
16628 // original 0x0C.
16629 FirstMask = (FirstPlusFour & 0x0F0F0F0F) | FirstCs;
16630
16631 auto PermMask = addPermMasks(First: FirstMask, Second: SecondMask);
16632 auto FirstVal =
16633 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
16634 auto SecondVal =
16635 getDWordFromOffset(DAG, SL, Src: SecondElt->SrcOp, DWordOffset: SecondElt->DWordOffset);
16636
16637 Perms.push_back(Elt: DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: FirstVal,
16638 N2: SecondVal,
16639 N3: DAG.getConstant(Val: PermMask, DL: SL, VT: MVT::i32)));
16640
16641 FirstElt = std::next(x: SecondElt);
16642 if (FirstElt == Srcs.end())
16643 break;
16644
16645 SecondElt = std::next(x: FirstElt);
16646 // If we only have a FirstElt, then just combine that into the cumulative
16647 // source node.
16648 if (SecondElt == Srcs.end()) {
16649 auto EltOp =
16650 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
16651
16652 Perms.push_back(
16653 Elt: DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: EltOp, N2: EltOp,
16654 N3: DAG.getConstant(Val: FirstElt->PermMask, DL: SL, VT: MVT::i32)));
16655 break;
16656 }
16657 }
16658
16659 assert(Perms.size() == 1 || Perms.size() == 2);
16660 return Perms.size() == 2
16661 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Perms[0], N2: Perms[1])
16662 : Perms[0];
16663}
16664
16665static void fixMasks(SmallVectorImpl<DotSrc> &Srcs, unsigned ChainLength) {
16666 for (auto &[EntryVal, EntryMask, EntryOffset] : Srcs) {
16667 EntryMask = EntryMask >> ((4 - ChainLength) * 8);
16668 auto ZeroMask = ChainLength == 2 ? 0x0c0c0000 : 0x0c000000;
16669 EntryMask += ZeroMask;
16670 }
16671}
16672
16673static bool isMul(const SDValue Op) {
16674 auto Opcode = Op.getOpcode();
16675
16676 return (Opcode == ISD::MUL || Opcode == AMDGPUISD::MUL_U24 ||
16677 Opcode == AMDGPUISD::MUL_I24);
16678}
16679
16680static std::optional<bool>
16681checkDot4MulSignedness(const SDValue &N, ByteProvider<SDValue> &Src0,
16682 ByteProvider<SDValue> &Src1, const SDValue &S0Op,
16683 const SDValue &S1Op, const SelectionDAG &DAG) {
16684 // If we both ops are i8s (pre legalize-dag), then the signedness semantics
16685 // of the dot4 is irrelevant.
16686 if (S0Op.getValueSizeInBits() == 8 && S1Op.getValueSizeInBits() == 8)
16687 return false;
16688
16689 auto Known0 = DAG.computeKnownBits(Op: S0Op, Depth: 0);
16690 bool S0IsUnsigned = Known0.countMinLeadingZeros() > 0;
16691 bool S0IsSigned = Known0.countMinLeadingOnes() > 0;
16692 auto Known1 = DAG.computeKnownBits(Op: S1Op, Depth: 0);
16693 bool S1IsUnsigned = Known1.countMinLeadingZeros() > 0;
16694 bool S1IsSigned = Known1.countMinLeadingOnes() > 0;
16695
16696 assert(!(S0IsUnsigned && S0IsSigned));
16697 assert(!(S1IsUnsigned && S1IsSigned));
16698
16699 // There are 9 possible permutations of
16700 // {S0IsUnsigned, S0IsSigned, S1IsUnsigned, S1IsSigned}
16701
16702 // In two permutations, the sign bits are known to be the same for both Ops,
16703 // so simply return Signed / Unsigned corresponding to the MSB
16704
16705 if ((S0IsUnsigned && S1IsUnsigned) || (S0IsSigned && S1IsSigned))
16706 return S0IsSigned;
16707
16708 // In another two permutations, the sign bits are known to be opposite. In
16709 // this case return std::nullopt to indicate a bad match.
16710
16711 if ((S0IsUnsigned && S1IsSigned) || (S0IsSigned && S1IsUnsigned))
16712 return std::nullopt;
16713
16714 // In the remaining five permutations, we don't know the value of the sign
16715 // bit for at least one Op. Since we have a valid ByteProvider, we know that
16716 // the upper bits must be extension bits. Thus, the only ways for the sign
16717 // bit to be unknown is if it was sign extended from unknown value, or if it
16718 // was any extended. In either case, it is correct to use the signed
16719 // version of the signedness semantics of dot4
16720
16721 // In two of such permutations, we known the sign bit is set for
16722 // one op, and the other is unknown. It is okay to used signed version of
16723 // dot4.
16724 if ((S0IsSigned && !(S1IsSigned || S1IsUnsigned)) ||
16725 ((S1IsSigned && !(S0IsSigned || S0IsUnsigned))))
16726 return true;
16727
16728 // In one such permutation, we don't know either of the sign bits. It is okay
16729 // to used the signed version of dot4.
16730 if ((!(S1IsSigned || S1IsUnsigned) && !(S0IsSigned || S0IsUnsigned)))
16731 return true;
16732
16733 // In two of such permutations, we known the sign bit is unset for
16734 // one op, and the other is unknown. Return std::nullopt to indicate a
16735 // bad match.
16736 if ((S0IsUnsigned && !(S1IsSigned || S1IsUnsigned)) ||
16737 ((S1IsUnsigned && !(S0IsSigned || S0IsUnsigned))))
16738 return std::nullopt;
16739
16740 llvm_unreachable("Fully covered condition");
16741}
16742
16743SDValue SITargetLowering::performAddCombine(SDNode *N,
16744 DAGCombinerInfo &DCI) const {
16745 SelectionDAG &DAG = DCI.DAG;
16746 EVT VT = N->getValueType(ResNo: 0);
16747 SDLoc SL(N);
16748 SDValue LHS = N->getOperand(Num: 0);
16749 SDValue RHS = N->getOperand(Num: 1);
16750
16751 if (LHS.getOpcode() == ISD::MUL || RHS.getOpcode() == ISD::MUL) {
16752 if (Subtarget->hasMad64_32()) {
16753 if (SDValue Folded = tryFoldToMad64_32(N, DCI))
16754 return Folded;
16755 }
16756 }
16757
16758 if (SDValue V = reassociateScalarOps(N, DAG)) {
16759 return V;
16760 }
16761
16762 if (VT == MVT::i64) {
16763 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
16764 return Folded;
16765 }
16766
16767 if ((isMul(Op: LHS) || isMul(Op: RHS)) && Subtarget->hasDot7Insts() &&
16768 (Subtarget->hasDot1Insts() || Subtarget->hasDot8Insts())) {
16769 SDValue TempNode(N, 0);
16770 std::optional<bool> IsSigned;
16771 SmallVector<DotSrc, 4> Src0s;
16772 SmallVector<DotSrc, 4> Src1s;
16773 SmallVector<SDValue, 4> Src2s;
16774
16775 // Match the v_dot4 tree, while collecting src nodes.
16776 int ChainLength = 0;
16777 for (int I = 0; I < 4; I++) {
16778 auto MulIdx = isMul(Op: LHS) ? 0 : isMul(Op: RHS) ? 1 : -1;
16779 if (MulIdx == -1)
16780 break;
16781 auto Src0 = handleMulOperand(MulOperand: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 0));
16782 if (!Src0)
16783 break;
16784 auto Src1 = handleMulOperand(MulOperand: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 1));
16785 if (!Src1)
16786 break;
16787
16788 auto IterIsSigned = checkDot4MulSignedness(
16789 N: TempNode->getOperand(Num: MulIdx), Src0&: *Src0, Src1&: *Src1,
16790 S0Op: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 0),
16791 S1Op: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 1), DAG);
16792 if (!IterIsSigned)
16793 break;
16794 if (!IsSigned)
16795 IsSigned = *IterIsSigned;
16796 if (*IterIsSigned != *IsSigned)
16797 break;
16798 placeSources(Src0&: *Src0, Src1&: *Src1, Src0s, Src1s, Step: I);
16799 auto AddIdx = 1 - MulIdx;
16800 // Allow the special case where add (add (mul24, 0), mul24) became ->
16801 // add (mul24, mul24).
16802 if (I == 2 && isMul(Op: TempNode->getOperand(Num: AddIdx))) {
16803 Src2s.push_back(Elt: TempNode->getOperand(Num: AddIdx));
16804 auto Src0 =
16805 handleMulOperand(MulOperand: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 0));
16806 if (!Src0)
16807 break;
16808 auto Src1 =
16809 handleMulOperand(MulOperand: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 1));
16810 if (!Src1)
16811 break;
16812 auto IterIsSigned = checkDot4MulSignedness(
16813 N: TempNode->getOperand(Num: AddIdx), Src0&: *Src0, Src1&: *Src1,
16814 S0Op: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 0),
16815 S1Op: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 1), DAG);
16816 if (!IterIsSigned)
16817 break;
16818 assert(IsSigned);
16819 if (*IterIsSigned != *IsSigned)
16820 break;
16821 placeSources(Src0&: *Src0, Src1&: *Src1, Src0s, Src1s, Step: I + 1);
16822 Src2s.push_back(Elt: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
16823 ChainLength = I + 2;
16824 break;
16825 }
16826
16827 TempNode = TempNode->getOperand(Num: AddIdx);
16828 Src2s.push_back(Elt: TempNode);
16829 ChainLength = I + 1;
16830 if (TempNode->getNumOperands() < 2)
16831 break;
16832 LHS = TempNode->getOperand(Num: 0);
16833 RHS = TempNode->getOperand(Num: 1);
16834 }
16835
16836 if (ChainLength < 2)
16837 return SDValue();
16838
16839 // Masks were constructed with assumption that we would find a chain of
16840 // length 4. If not, then we need to 0 out the MSB bits (via perm mask of
16841 // 0x0c) so they do not affect dot calculation.
16842 if (ChainLength < 4) {
16843 fixMasks(Srcs&: Src0s, ChainLength);
16844 fixMasks(Srcs&: Src1s, ChainLength);
16845 }
16846
16847 SDValue Src0, Src1;
16848
16849 // If we are just using a single source for both, and have permuted the
16850 // bytes consistently, we can just use the sources without permuting
16851 // (commutation).
16852 bool UseOriginalSrc = false;
16853 if (ChainLength == 4 && Src0s.size() == 1 && Src1s.size() == 1 &&
16854 Src0s.begin()->PermMask == Src1s.begin()->PermMask &&
16855 Src0s.begin()->SrcOp.getValueSizeInBits() >= 32 &&
16856 Src1s.begin()->SrcOp.getValueSizeInBits() >= 32) {
16857 SmallVector<unsigned, 4> SrcBytes;
16858 auto Src0Mask = Src0s.begin()->PermMask;
16859 SrcBytes.push_back(Elt: Src0Mask & 0xFF000000);
16860 bool UniqueEntries = true;
16861 for (auto I = 1; I < 4; I++) {
16862 auto NextByte = Src0Mask & (0xFF << ((3 - I) * 8));
16863
16864 if (is_contained(Range&: SrcBytes, Element: NextByte)) {
16865 UniqueEntries = false;
16866 break;
16867 }
16868 SrcBytes.push_back(Elt: NextByte);
16869 }
16870
16871 if (UniqueEntries) {
16872 UseOriginalSrc = true;
16873
16874 auto *FirstElt = Src0s.begin();
16875 auto FirstEltOp =
16876 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
16877
16878 auto *SecondElt = Src1s.begin();
16879 auto SecondEltOp = getDWordFromOffset(DAG, SL, Src: SecondElt->SrcOp,
16880 DWordOffset: SecondElt->DWordOffset);
16881
16882 Src0 = DAG.getBitcastedAnyExtOrTrunc(Op: FirstEltOp, DL: SL,
16883 VT: MVT::getIntegerVT(BitWidth: 32));
16884 Src1 = DAG.getBitcastedAnyExtOrTrunc(Op: SecondEltOp, DL: SL,
16885 VT: MVT::getIntegerVT(BitWidth: 32));
16886 }
16887 }
16888
16889 if (!UseOriginalSrc) {
16890 Src0 = resolveSources(DAG, SL, Srcs&: Src0s, IsSigned: false, IsAny: true);
16891 Src1 = resolveSources(DAG, SL, Srcs&: Src1s, IsSigned: false, IsAny: true);
16892 }
16893
16894 assert(IsSigned);
16895 SDValue Src2 =
16896 DAG.getExtOrTrunc(IsSigned: *IsSigned, Op: Src2s[ChainLength - 1], DL: SL, VT: MVT::i32);
16897
16898 SDValue IID = DAG.getTargetConstant(Val: *IsSigned ? Intrinsic::amdgcn_sdot4
16899 : Intrinsic::amdgcn_udot4,
16900 DL: SL, VT: MVT::i64);
16901
16902 assert(!VT.isVector());
16903 auto Dot = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32, N1: IID, N2: Src0,
16904 N3: Src1, N4: Src2, N5: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i1));
16905
16906 return DAG.getExtOrTrunc(IsSigned: *IsSigned, Op: Dot, DL: SL, VT);
16907 }
16908
16909 if (VT != MVT::i32 || !DCI.isAfterLegalizeDAG())
16910 return SDValue();
16911
16912 // add x, zext (setcc) => uaddo_carry x, 0, setcc
16913 // add x, sext (setcc) => usubo_carry x, 0, setcc
16914 unsigned Opc = LHS.getOpcode();
16915 if (Opc == ISD::ZERO_EXTEND || Opc == ISD::SIGN_EXTEND ||
16916 Opc == ISD::ANY_EXTEND || Opc == ISD::UADDO_CARRY)
16917 std::swap(a&: RHS, b&: LHS);
16918
16919 Opc = RHS.getOpcode();
16920 switch (Opc) {
16921 default:
16922 break;
16923 case ISD::ZERO_EXTEND:
16924 case ISD::SIGN_EXTEND:
16925 case ISD::ANY_EXTEND: {
16926 auto Cond = RHS.getOperand(i: 0);
16927 // If this won't be a real VOPC output, we would still need to insert an
16928 // extra instruction anyway.
16929 if (!isBoolSGPR(V: Cond))
16930 break;
16931 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1);
16932 SDValue Args[] = {LHS, DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond};
16933 Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::USUBO_CARRY : ISD::UADDO_CARRY;
16934 return DAG.getNode(Opcode: Opc, DL: SL, VTList, Ops: Args);
16935 }
16936 case ISD::UADDO_CARRY: {
16937 // add x, (uaddo_carry y, 0, cc) => uaddo_carry x, y, cc
16938 if (!isNullConstant(V: RHS.getOperand(i: 1)))
16939 break;
16940 SDValue Args[] = {LHS, RHS.getOperand(i: 0), RHS.getOperand(i: 2)};
16941 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: SDLoc(N), VTList: RHS->getVTList(), Ops: Args);
16942 }
16943 }
16944 return SDValue();
16945}
16946
16947SDValue SITargetLowering::performPtrAddCombine(SDNode *N,
16948 DAGCombinerInfo &DCI) const {
16949 SelectionDAG &DAG = DCI.DAG;
16950 SDLoc DL(N);
16951 EVT VT = N->getValueType(ResNo: 0);
16952 SDValue N0 = N->getOperand(Num: 0);
16953 SDValue N1 = N->getOperand(Num: 1);
16954
16955 // The following folds transform PTRADDs into regular arithmetic in cases
16956 // where the PTRADD wouldn't be folded as an immediate offset into memory
16957 // instructions anyway. They are target-specific in that other targets might
16958 // prefer to not lose information about the pointer arithmetic.
16959
16960 // Fold (ptradd x, shl(0 - v, k)) -> sub(x, shl(v, k)).
16961 // Adapted from DAGCombiner::visitADDLikeCommutative.
16962 SDValue V, K;
16963 if (sd_match(N: N1, P: m_Shl(L: m_Neg(V: m_Value(N&: V)), R: m_Value(N&: K)))) {
16964 SDNodeFlags ShlFlags = N1->getFlags();
16965 // If the original shl is NUW and NSW, the first k+1 bits of 0-v are all 0,
16966 // so v is either 0 or the first k+1 bits of v are all 1 -> NSW can be
16967 // preserved.
16968 SDNodeFlags NewShlFlags =
16969 ShlFlags.hasNoUnsignedWrap() && ShlFlags.hasNoSignedWrap()
16970 ? SDNodeFlags::NoSignedWrap
16971 : SDNodeFlags();
16972 SDValue Inner = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: V, N2: K, Flags: NewShlFlags);
16973 DCI.AddToWorklist(N: Inner.getNode());
16974 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: Inner);
16975 }
16976
16977 // Fold into Mad64 if the right-hand side is a MUL. Analogous to a fold in
16978 // performAddCombine.
16979 if (N1.getOpcode() == ISD::MUL) {
16980 if (Subtarget->hasMad64_32()) {
16981 if (SDValue Folded = tryFoldToMad64_32(N, DCI))
16982 return Folded;
16983 }
16984 }
16985
16986 // If the 32 low bits of the constant are all zero, there is nothing to fold
16987 // into an immediate offset, so it's better to eliminate the unnecessary
16988 // addition for the lower 32 bits than to preserve the PTRADD.
16989 // Analogous to a fold in performAddCombine.
16990 if (VT == MVT::i64) {
16991 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
16992 return Folded;
16993 }
16994
16995 if (N1.getOpcode() != ISD::ADD || !N1.hasOneUse())
16996 return SDValue();
16997
16998 SDValue X = N0;
16999 SDValue Y = N1.getOperand(i: 0);
17000 SDValue Z = N1.getOperand(i: 1);
17001 bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(N: Y);
17002 bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(N: Z);
17003
17004 if (!YIsConstant && !ZIsConstant && !X->isDivergent() &&
17005 Y->isDivergent() != Z->isDivergent()) {
17006 // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if x and
17007 // y are uniform and z isn't.
17008 // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if x and
17009 // z are uniform and y isn't.
17010 // The goal is to push uniform operands up in the computation, so that they
17011 // can be handled with scalar operations. We can't use reassociateScalarOps
17012 // for this since it requires two identical commutative operations to
17013 // reassociate.
17014 if (Y->isDivergent())
17015 std::swap(a&: Y, b&: Z);
17016 // If both additions in the original were NUW, reassociation preserves that.
17017 SDNodeFlags ReassocFlags =
17018 (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap;
17019 SDValue UniformInner = DAG.getMemBasePlusOffset(Base: X, Offset: Y, DL, Flags: ReassocFlags);
17020 DCI.AddToWorklist(N: UniformInner.getNode());
17021 return DAG.getMemBasePlusOffset(Base: UniformInner, Offset: Z, DL, Flags: ReassocFlags);
17022 }
17023
17024 return SDValue();
17025}
17026
17027static bool isCtlzOpc(unsigned Opc) {
17028 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_UNDEF;
17029}
17030
17031SDValue SITargetLowering::performSubCombine(SDNode *N,
17032 DAGCombinerInfo &DCI) const {
17033 SelectionDAG &DAG = DCI.DAG;
17034 EVT VT = N->getValueType(ResNo: 0);
17035
17036 if (VT == MVT::i64) {
17037 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
17038 return Folded;
17039 }
17040
17041 if (VT != MVT::i32)
17042 return SDValue();
17043
17044 SDLoc SL(N);
17045 SDValue LHS = N->getOperand(Num: 0);
17046 SDValue RHS = N->getOperand(Num: 1);
17047
17048 // sub x, zext (setcc) => usubo_carry x, 0, setcc
17049 // sub x, sext (setcc) => uaddo_carry x, 0, setcc
17050 unsigned Opc = RHS.getOpcode();
17051 switch (Opc) {
17052 default:
17053 break;
17054 case ISD::ZERO_EXTEND:
17055 case ISD::SIGN_EXTEND:
17056 case ISD::ANY_EXTEND: {
17057 auto Cond = RHS.getOperand(i: 0);
17058 // If this won't be a real VOPC output, we would still need to insert an
17059 // extra instruction anyway.
17060 if (!isBoolSGPR(V: Cond))
17061 break;
17062 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1);
17063 SDValue Args[] = {LHS, DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond};
17064 Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
17065 return DAG.getNode(Opcode: Opc, DL: SL, VTList, Ops: Args);
17066 }
17067 }
17068
17069 if (LHS.getOpcode() == ISD::USUBO_CARRY) {
17070 // sub (usubo_carry x, 0, cc), y => usubo_carry x, y, cc
17071 if (!isNullConstant(V: LHS.getOperand(i: 1)))
17072 return SDValue();
17073 SDValue Args[] = {LHS.getOperand(i: 0), RHS, LHS.getOperand(i: 2)};
17074 return DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: SDLoc(N), VTList: LHS->getVTList(), Ops: Args);
17075 }
17076
17077 // sub (ctlz (xor x, (sra x, 31))), 1 -> ctls x.
17078 if (isOneConstant(V: RHS) && isCtlzOpc(Opc: LHS.getOpcode())) {
17079 SDValue CtlzSrc = LHS.getOperand(i: 0);
17080 // Check for xor x, (sra x, 31) pattern.
17081 if (CtlzSrc.getOpcode() == ISD::XOR) {
17082 SDValue X = CtlzSrc.getOperand(i: 0);
17083 SDValue SignExt = CtlzSrc.getOperand(i: 1);
17084 // Try both ordering of XOR operands.
17085 if (SignExt.getOpcode() != ISD::SRA)
17086 std::swap(a&: X, b&: SignExt);
17087 if (SignExt.getOpcode() == ISD::SRA && SignExt.getOperand(i: 0) == X) {
17088 ConstantSDNode *ShiftAmt =
17089 dyn_cast<ConstantSDNode>(Val: SignExt.getOperand(i: 1));
17090 unsigned BitWidth = X.getValueType().getScalarSizeInBits();
17091 if (ShiftAmt && ShiftAmt->getZExtValue() == BitWidth - 1)
17092 return DAG.getNode(Opcode: ISD::CTLS, DL: SL, VT, Operand: X);
17093 }
17094 }
17095 }
17096
17097 return SDValue();
17098}
17099
17100SDValue
17101SITargetLowering::performAddCarrySubCarryCombine(SDNode *N,
17102 DAGCombinerInfo &DCI) const {
17103
17104 if (N->getValueType(ResNo: 0) != MVT::i32)
17105 return SDValue();
17106
17107 if (!isNullConstant(V: N->getOperand(Num: 1)))
17108 return SDValue();
17109
17110 SelectionDAG &DAG = DCI.DAG;
17111 SDValue LHS = N->getOperand(Num: 0);
17112
17113 // uaddo_carry (add x, y), 0, cc => uaddo_carry x, y, cc
17114 // usubo_carry (sub x, y), 0, cc => usubo_carry x, y, cc
17115 unsigned LHSOpc = LHS.getOpcode();
17116 unsigned Opc = N->getOpcode();
17117 if ((LHSOpc == ISD::ADD && Opc == ISD::UADDO_CARRY) ||
17118 (LHSOpc == ISD::SUB && Opc == ISD::USUBO_CARRY)) {
17119 SDValue Args[] = {LHS.getOperand(i: 0), LHS.getOperand(i: 1), N->getOperand(Num: 2)};
17120 return DAG.getNode(Opcode: Opc, DL: SDLoc(N), VTList: N->getVTList(), Ops: Args);
17121 }
17122 return SDValue();
17123}
17124
17125SDValue SITargetLowering::performFAddCombine(SDNode *N,
17126 DAGCombinerInfo &DCI) const {
17127 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
17128 return SDValue();
17129
17130 SelectionDAG &DAG = DCI.DAG;
17131 EVT VT = N->getValueType(ResNo: 0);
17132
17133 SDLoc SL(N);
17134 SDValue LHS = N->getOperand(Num: 0);
17135 SDValue RHS = N->getOperand(Num: 1);
17136
17137 // These should really be instruction patterns, but writing patterns with
17138 // source modifiers is a pain.
17139
17140 // fadd (fadd (a, a), b) -> mad 2.0, a, b
17141 if (LHS.getOpcode() == ISD::FADD) {
17142 SDValue A = LHS.getOperand(i: 0);
17143 if (A == LHS.getOperand(i: 1)) {
17144 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: LHS.getNode());
17145 if (FusedOp != 0) {
17146 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
17147 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: RHS);
17148 }
17149 }
17150 }
17151
17152 // fadd (b, fadd (a, a)) -> mad 2.0, a, b
17153 if (RHS.getOpcode() == ISD::FADD) {
17154 SDValue A = RHS.getOperand(i: 0);
17155 if (A == RHS.getOperand(i: 1)) {
17156 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: RHS.getNode());
17157 if (FusedOp != 0) {
17158 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
17159 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: LHS);
17160 }
17161 }
17162 }
17163
17164 return SDValue();
17165}
17166
17167SDValue SITargetLowering::performFSubCombine(SDNode *N,
17168 DAGCombinerInfo &DCI) const {
17169 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
17170 return SDValue();
17171
17172 SelectionDAG &DAG = DCI.DAG;
17173 SDLoc SL(N);
17174 EVT VT = N->getValueType(ResNo: 0);
17175 assert(!VT.isVector());
17176
17177 // Try to get the fneg to fold into the source modifier. This undoes generic
17178 // DAG combines and folds them into the mad.
17179 //
17180 // Only do this if we are not trying to support denormals. v_mad_f32 does
17181 // not support denormals ever.
17182 SDValue LHS = N->getOperand(Num: 0);
17183 SDValue RHS = N->getOperand(Num: 1);
17184 if (LHS.getOpcode() == ISD::FADD) {
17185 // (fsub (fadd a, a), c) -> mad 2.0, a, (fneg c)
17186 SDValue A = LHS.getOperand(i: 0);
17187 if (A == LHS.getOperand(i: 1)) {
17188 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: LHS.getNode());
17189 if (FusedOp != 0) {
17190 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
17191 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
17192
17193 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: NegRHS);
17194 }
17195 }
17196 }
17197
17198 if (RHS.getOpcode() == ISD::FADD) {
17199 // (fsub c, (fadd a, a)) -> mad -2.0, a, c
17200
17201 SDValue A = RHS.getOperand(i: 0);
17202 if (A == RHS.getOperand(i: 1)) {
17203 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: RHS.getNode());
17204 if (FusedOp != 0) {
17205 const SDValue NegTwo = DAG.getConstantFP(Val: -2.0, DL: SL, VT);
17206 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: NegTwo, N3: LHS);
17207 }
17208 }
17209 }
17210
17211 return SDValue();
17212}
17213
17214SDValue SITargetLowering::performFDivCombine(SDNode *N,
17215 DAGCombinerInfo &DCI) const {
17216 SelectionDAG &DAG = DCI.DAG;
17217 SDLoc SL(N);
17218 EVT VT = N->getValueType(ResNo: 0);
17219
17220 // fsqrt legality correlates to rsq availability.
17221 if ((VT != MVT::f16 && VT != MVT::bf16) || !isOperationLegal(Op: ISD::FSQRT, VT))
17222 return SDValue();
17223
17224 SDValue LHS = N->getOperand(Num: 0);
17225 SDValue RHS = N->getOperand(Num: 1);
17226
17227 SDNodeFlags Flags = N->getFlags();
17228 SDNodeFlags RHSFlags = RHS->getFlags();
17229 if (!Flags.hasAllowContract() || !RHSFlags.hasAllowContract() ||
17230 !RHS->hasOneUse())
17231 return SDValue();
17232
17233 if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
17234 bool IsNegative = false;
17235 if (CLHS->isExactlyValue(V: 1.0) ||
17236 (IsNegative = CLHS->isExactlyValue(V: -1.0))) {
17237 // fdiv contract 1.0, (sqrt contract x) -> rsq for f16
17238 // fdiv contract -1.0, (sqrt contract x) -> fneg(rsq) for f16
17239 if (RHS.getOpcode() == ISD::FSQRT) {
17240 // TODO: Or in RHS flags, somehow missing from SDNodeFlags
17241 SDValue Rsq =
17242 DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SL, VT, Operand: RHS.getOperand(i: 0), Flags);
17243 return IsNegative ? DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Rsq, Flags) : Rsq;
17244 }
17245 }
17246 }
17247
17248 return SDValue();
17249}
17250
17251SDValue SITargetLowering::performFMulCombine(SDNode *N,
17252 DAGCombinerInfo &DCI) const {
17253 SelectionDAG &DAG = DCI.DAG;
17254 EVT VT = N->getValueType(ResNo: 0);
17255 EVT ScalarVT = VT.getScalarType();
17256 EVT IntVT = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
17257
17258 if (!N->isDivergent() && getSubtarget()->hasSALUFloatInsts() &&
17259 (ScalarVT == MVT::f32 || ScalarVT == MVT::f16)) {
17260 // Prefer to use s_mul_f16/f32 instead of v_ldexp_f16/f32.
17261 return SDValue();
17262 }
17263
17264 SDValue LHS = N->getOperand(Num: 0);
17265 SDValue RHS = N->getOperand(Num: 1);
17266
17267 // It is cheaper to realize i32 inline constants as compared against
17268 // materializing f16 or f64 (or even non-inline f32) values,
17269 // possible via ldexp usage, as shown below :
17270 //
17271 // Given : A = 2^a & B = 2^b ; where a and b are integers.
17272 // fmul x, (select y, A, B) -> ldexp( x, (select i32 y, a, b) )
17273 // fmul x, (select y, -A, -B) -> ldexp( (fneg x), (select i32 y, a, b) )
17274 if ((ScalarVT == MVT::f64 || ScalarVT == MVT::f32 || ScalarVT == MVT::f16) &&
17275 (RHS.hasOneUse() && RHS.getOpcode() == ISD::SELECT)) {
17276 const ConstantFPSDNode *TrueNode = isConstOrConstSplatFP(N: RHS.getOperand(i: 1));
17277 if (!TrueNode)
17278 return SDValue();
17279 const ConstantFPSDNode *FalseNode =
17280 isConstOrConstSplatFP(N: RHS.getOperand(i: 2));
17281 if (!FalseNode)
17282 return SDValue();
17283
17284 if (TrueNode->isNegative() != FalseNode->isNegative())
17285 return SDValue();
17286
17287 // For f32, only non-inline constants should be transformed.
17288 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
17289 if (ScalarVT == MVT::f32 &&
17290 TII->isInlineConstant(Imm: TrueNode->getValueAPF()) &&
17291 TII->isInlineConstant(Imm: FalseNode->getValueAPF()))
17292 return SDValue();
17293
17294 int TrueNodeExpVal = TrueNode->getValueAPF().getExactLog2Abs();
17295 if (TrueNodeExpVal == INT_MIN)
17296 return SDValue();
17297 int FalseNodeExpVal = FalseNode->getValueAPF().getExactLog2Abs();
17298 if (FalseNodeExpVal == INT_MIN)
17299 return SDValue();
17300
17301 SDLoc SL(N);
17302 SDValue SelectNode =
17303 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: IntVT, N1: RHS.getOperand(i: 0),
17304 N2: DAG.getSignedConstant(Val: TrueNodeExpVal, DL: SL, VT: IntVT),
17305 N3: DAG.getSignedConstant(Val: FalseNodeExpVal, DL: SL, VT: IntVT));
17306
17307 LHS = TrueNode->isNegative()
17308 ? DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS, Flags: LHS->getFlags())
17309 : LHS;
17310
17311 return DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT, N1: LHS, N2: SelectNode, Flags: N->getFlags());
17312 }
17313
17314 return SDValue();
17315}
17316
17317SDValue SITargetLowering::performFMACombine(SDNode *N,
17318 DAGCombinerInfo &DCI) const {
17319 SelectionDAG &DAG = DCI.DAG;
17320 EVT VT = N->getValueType(ResNo: 0);
17321 SDLoc SL(N);
17322
17323 if (!Subtarget->hasDot10Insts() || VT != MVT::f32)
17324 return SDValue();
17325
17326 // FMA((F32)S0.x, (F32)S1. x, FMA((F32)S0.y, (F32)S1.y, (F32)z)) ->
17327 // FDOT2((V2F16)S0, (V2F16)S1, (F32)z))
17328 SDValue Op1 = N->getOperand(Num: 0);
17329 SDValue Op2 = N->getOperand(Num: 1);
17330 SDValue FMA = N->getOperand(Num: 2);
17331
17332 if (FMA.getOpcode() != ISD::FMA || Op1.getOpcode() != ISD::FP_EXTEND ||
17333 Op2.getOpcode() != ISD::FP_EXTEND)
17334 return SDValue();
17335
17336 // fdot2_f32_f16 always flushes fp32 denormal operand and output to zero,
17337 // regardless of the denorm mode setting. Therefore,
17338 // fp-contract is sufficient to allow generating fdot2.
17339 const TargetOptions &Options = DAG.getTarget().Options;
17340 if (Options.AllowFPOpFusion == FPOpFusion::Fast ||
17341 (N->getFlags().hasAllowContract() &&
17342 FMA->getFlags().hasAllowContract())) {
17343 Op1 = Op1.getOperand(i: 0);
17344 Op2 = Op2.getOperand(i: 0);
17345 if (Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
17346 Op2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
17347 return SDValue();
17348
17349 SDValue Vec1 = Op1.getOperand(i: 0);
17350 SDValue Idx1 = Op1.getOperand(i: 1);
17351 SDValue Vec2 = Op2.getOperand(i: 0);
17352
17353 SDValue FMAOp1 = FMA.getOperand(i: 0);
17354 SDValue FMAOp2 = FMA.getOperand(i: 1);
17355 SDValue FMAAcc = FMA.getOperand(i: 2);
17356
17357 if (FMAOp1.getOpcode() != ISD::FP_EXTEND ||
17358 FMAOp2.getOpcode() != ISD::FP_EXTEND)
17359 return SDValue();
17360
17361 FMAOp1 = FMAOp1.getOperand(i: 0);
17362 FMAOp2 = FMAOp2.getOperand(i: 0);
17363 if (FMAOp1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
17364 FMAOp2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
17365 return SDValue();
17366
17367 SDValue Vec3 = FMAOp1.getOperand(i: 0);
17368 SDValue Vec4 = FMAOp2.getOperand(i: 0);
17369 SDValue Idx2 = FMAOp1.getOperand(i: 1);
17370
17371 if (Idx1 != Op2.getOperand(i: 1) || Idx2 != FMAOp2.getOperand(i: 1) ||
17372 // Idx1 and Idx2 cannot be the same.
17373 Idx1 == Idx2)
17374 return SDValue();
17375
17376 if (Vec1 == Vec2 || Vec3 == Vec4)
17377 return SDValue();
17378
17379 if (Vec1.getValueType() != MVT::v2f16 || Vec2.getValueType() != MVT::v2f16)
17380 return SDValue();
17381
17382 if ((Vec1 == Vec3 && Vec2 == Vec4) || (Vec1 == Vec4 && Vec2 == Vec3)) {
17383 return DAG.getNode(Opcode: AMDGPUISD::FDOT2, DL: SL, VT: MVT::f32, N1: Vec1, N2: Vec2, N3: FMAAcc,
17384 N4: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i1));
17385 }
17386 }
17387 return SDValue();
17388}
17389
17390SDValue SITargetLowering::performSetCCCombine(SDNode *N,
17391 DAGCombinerInfo &DCI) const {
17392 SelectionDAG &DAG = DCI.DAG;
17393 SDLoc SL(N);
17394
17395 SDValue LHS = N->getOperand(Num: 0);
17396 SDValue RHS = N->getOperand(Num: 1);
17397 EVT VT = LHS.getValueType();
17398 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
17399
17400 auto *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
17401 if (!CRHS) {
17402 CRHS = dyn_cast<ConstantSDNode>(Val&: LHS);
17403 if (CRHS) {
17404 std::swap(a&: LHS, b&: RHS);
17405 CC = getSetCCSwappedOperands(Operation: CC);
17406 }
17407 }
17408
17409 if (CRHS) {
17410 if (VT == MVT::i32 && LHS.getOpcode() == ISD::SIGN_EXTEND &&
17411 isBoolSGPR(V: LHS.getOperand(i: 0))) {
17412 // setcc (sext from i1 cc), -1, ne|sgt|ult) => not cc => xor cc, -1
17413 // setcc (sext from i1 cc), -1, eq|sle|uge) => cc
17414 // setcc (sext from i1 cc), 0, eq|sge|ule) => not cc => xor cc, -1
17415 // setcc (sext from i1 cc), 0, ne|ugt|slt) => cc
17416 if ((CRHS->isAllOnes() &&
17417 (CC == ISD::SETNE || CC == ISD::SETGT || CC == ISD::SETULT)) ||
17418 (CRHS->isZero() &&
17419 (CC == ISD::SETEQ || CC == ISD::SETGE || CC == ISD::SETULE)))
17420 return DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i1, N1: LHS.getOperand(i: 0),
17421 N2: DAG.getAllOnesConstant(DL: SL, VT: MVT::i1));
17422 if ((CRHS->isAllOnes() &&
17423 (CC == ISD::SETEQ || CC == ISD::SETLE || CC == ISD::SETUGE)) ||
17424 (CRHS->isZero() &&
17425 (CC == ISD::SETNE || CC == ISD::SETUGT || CC == ISD::SETLT)))
17426 return LHS.getOperand(i: 0);
17427 }
17428
17429 const APInt &CRHSVal = CRHS->getAPIntValue();
17430 if ((CC == ISD::SETEQ || CC == ISD::SETNE) &&
17431 LHS.getOpcode() == ISD::SELECT &&
17432 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
17433 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2)) &&
17434 isBoolSGPR(V: LHS.getOperand(i: 0))) {
17435 // Given CT != FT:
17436 // setcc (select cc, CT, CF), CF, eq => xor cc, -1
17437 // setcc (select cc, CT, CF), CF, ne => cc
17438 // setcc (select cc, CT, CF), CT, ne => xor cc, -1
17439 // setcc (select cc, CT, CF), CT, eq => cc
17440 const APInt &CT = LHS.getConstantOperandAPInt(i: 1);
17441 const APInt &CF = LHS.getConstantOperandAPInt(i: 2);
17442
17443 if (CT != CF) {
17444 if ((CF == CRHSVal && CC == ISD::SETEQ) ||
17445 (CT == CRHSVal && CC == ISD::SETNE))
17446 return DAG.getNOT(DL: SL, Val: LHS.getOperand(i: 0), VT: MVT::i1);
17447 if ((CF == CRHSVal && CC == ISD::SETNE) ||
17448 (CT == CRHSVal && CC == ISD::SETEQ))
17449 return LHS.getOperand(i: 0);
17450 }
17451 }
17452 }
17453
17454 // Truncate 64-bit setcc to test only upper 32-bits of its operands in the
17455 // following cases where information about the lower 32-bits of its operands
17456 // is known:
17457 //
17458 // If LHS.lo32 == RHS.lo32:
17459 // setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, eq/ne
17460 // If LHS.lo32 != RHS.lo32:
17461 // setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, false/true
17462 // If LHS.lo32 >= RHS.lo32 (unsigned):
17463 // setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]lt/ge
17464 // If LHS.lo32 > RHS.lo32 (unsigned):
17465 // setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]lt/ge
17466 // If LHS.lo32 <= RHS.lo32 (unsigned):
17467 // setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]le/gt
17468 // If LHS.lo32 < RHS.lo32 (unsigned):
17469 // setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]le/gt
17470 if (VT == MVT::i64) {
17471 const KnownBits LHSKnownLo32 = DAG.computeKnownBits(Op: LHS).trunc(BitWidth: 32);
17472 const KnownBits RHSKnownLo32 = DAG.computeKnownBits(Op: RHS).trunc(BitWidth: 32);
17473
17474 // NewCC is valid iff we can truncate the setcc to only test the upper 32
17475 // bits
17476 ISD::CondCode NewCC = ISD::SETCC_INVALID;
17477
17478 switch (CC) {
17479 default:
17480 break;
17481 case ISD::SETEQ: {
17482 const std::optional<bool> KnownEq =
17483 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
17484 if (KnownEq)
17485 NewCC = *KnownEq ? ISD::SETEQ : ISD::SETFALSE;
17486
17487 break;
17488 }
17489 case ISD::SETNE: {
17490 const std::optional<bool> KnownEq =
17491 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
17492 if (KnownEq)
17493 NewCC = *KnownEq ? ISD::SETNE : ISD::SETTRUE;
17494
17495 break;
17496 }
17497 case ISD::SETULT:
17498 case ISD::SETUGE:
17499 case ISD::SETLT:
17500 case ISD::SETGE: {
17501 const std::optional<bool> KnownUge =
17502 KnownBits::uge(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
17503 if (KnownUge) {
17504 if (*KnownUge) {
17505 // LHS.lo32 uge RHS.lo32, so LHS >= RHS iff LHS.hi32 >= RHS.hi32
17506 NewCC = CC;
17507 } else {
17508 // LHS.lo32 ult RHS.lo32, so LHS >= RHS iff LHS.hi32 > RHS.hi32
17509 NewCC = CC == ISD::SETULT ? ISD::SETULE
17510 : CC == ISD::SETUGE ? ISD::SETUGT
17511 : CC == ISD::SETLT ? ISD::SETLE
17512 : ISD::SETGT;
17513 }
17514 }
17515 break;
17516 }
17517 case ISD::SETULE:
17518 case ISD::SETUGT:
17519 case ISD::SETLE:
17520 case ISD::SETGT: {
17521 const std::optional<bool> KnownUle =
17522 KnownBits::ule(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
17523 if (KnownUle) {
17524 if (*KnownUle) {
17525 // LHS.lo32 ule RHS.lo32, so LHS <= RHS iff LHS.hi32 <= RHS.hi32
17526 NewCC = CC;
17527 } else {
17528 // LHS.lo32 ugt RHS.lo32, so LHS <= RHS iff LHS.hi32 < RHS.hi32
17529 NewCC = CC == ISD::SETULE ? ISD::SETULT
17530 : CC == ISD::SETUGT ? ISD::SETUGE
17531 : CC == ISD::SETLE ? ISD::SETLT
17532 : ISD::SETGE;
17533 }
17534 }
17535 break;
17536 }
17537 }
17538
17539 if (NewCC != ISD::SETCC_INVALID)
17540 return DAG.getSetCC(DL: SL, VT: N->getValueType(ResNo: 0), LHS: getHiHalf64(Op: LHS, DAG),
17541 RHS: getHiHalf64(Op: RHS, DAG), Cond: NewCC);
17542 }
17543
17544 // Eliminate setcc by using carryout from add/sub instruction
17545
17546 // LHS = ADD i64 RHS, Z LHSlo = UADDO i32 RHSlo, Zlo
17547 // setcc LHS ult RHS -> LHSHi = UADDO_CARRY i32 RHShi, Zhi
17548 // similarly for subtraction
17549
17550 // LHS = ADD i64 Y, 1 LHSlo = UADDO i32 Ylo, 1
17551 // setcc LHS eq 0 -> LHSHi = UADDO_CARRY i32 Yhi, 0
17552
17553 if (VT == MVT::i64 && ((CC == ISD::SETULT &&
17554 sd_match(N: LHS, P: m_Add(L: m_Specific(N: RHS), R: m_Value()))) ||
17555 (CC == ISD::SETUGT &&
17556 sd_match(N: LHS, P: m_Sub(L: m_Specific(N: RHS), R: m_Value()))) ||
17557 (CC == ISD::SETEQ && CRHS && CRHS->isZero() &&
17558 sd_match(N: LHS, P: m_Add(L: m_Value(), R: m_One()))))) {
17559 bool IsAdd = LHS.getOpcode() == ISD::ADD;
17560
17561 SDValue Op0 = LHS.getOperand(i: 0);
17562 SDValue Op1 = LHS.getOperand(i: 1);
17563
17564 SDValue Op0Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Op0);
17565 SDValue Op1Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Op1);
17566
17567 SDValue Op0Hi = getHiHalf64(Op: Op0, DAG);
17568 SDValue Op1Hi = getHiHalf64(Op: Op1, DAG);
17569
17570 SDValue NodeLo =
17571 DAG.getNode(Opcode: IsAdd ? ISD::UADDO : ISD::USUBO, DL: SL,
17572 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1), Ops: {Op0Lo, Op1Lo});
17573
17574 SDValue CarryInHi = NodeLo.getValue(R: 1);
17575 SDValue NodeHi = DAG.getNode(Opcode: IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY,
17576 DL: SL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1),
17577 Ops: {Op0Hi, Op1Hi, CarryInHi});
17578
17579 SDValue ResultLo = NodeLo.getValue(R: 0);
17580 SDValue ResultHi = NodeHi.getValue(R: 0);
17581
17582 SDValue JoinedResult =
17583 DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {ResultLo, ResultHi});
17584
17585 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: JoinedResult);
17586 SDValue Overflow = NodeHi.getValue(R: 1);
17587 DCI.CombineTo(N: LHS.getNode(), Res: Result);
17588 return Overflow;
17589 }
17590
17591 if (VT != MVT::f32 && VT != MVT::f64 &&
17592 (!Subtarget->has16BitInsts() || VT != MVT::f16))
17593 return SDValue();
17594
17595 // Match isinf/isfinite pattern
17596 // (fcmp oeq (fabs x), inf) -> (fp_class x, (p_infinity | n_infinity))
17597 // (fcmp one (fabs x), inf) -> (fp_class x,
17598 // (p_normal | n_normal | p_subnormal | n_subnormal | p_zero | n_zero)
17599 if ((CC == ISD::SETOEQ || CC == ISD::SETONE) &&
17600 LHS.getOpcode() == ISD::FABS) {
17601 const ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
17602 if (!CRHS)
17603 return SDValue();
17604
17605 const APFloat &APF = CRHS->getValueAPF();
17606 if (APF.isInfinity() && !APF.isNegative()) {
17607 const unsigned IsInfMask =
17608 SIInstrFlags::P_INFINITY | SIInstrFlags::N_INFINITY;
17609 const unsigned IsFiniteMask =
17610 SIInstrFlags::N_ZERO | SIInstrFlags::P_ZERO | SIInstrFlags::N_NORMAL |
17611 SIInstrFlags::P_NORMAL | SIInstrFlags::N_SUBNORMAL |
17612 SIInstrFlags::P_SUBNORMAL;
17613 unsigned Mask = CC == ISD::SETOEQ ? IsInfMask : IsFiniteMask;
17614 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL: SL, VT: MVT::i1, N1: LHS.getOperand(i: 0),
17615 N2: DAG.getConstant(Val: Mask, DL: SL, VT: MVT::i32));
17616 }
17617 }
17618
17619 return SDValue();
17620}
17621
17622SDValue
17623SITargetLowering::performCvtF32UByteNCombine(SDNode *N,
17624 DAGCombinerInfo &DCI) const {
17625 SelectionDAG &DAG = DCI.DAG;
17626 SDLoc SL(N);
17627 unsigned Offset = N->getOpcode() - AMDGPUISD::CVT_F32_UBYTE0;
17628
17629 SDValue Src = N->getOperand(Num: 0);
17630 SDValue Shift = N->getOperand(Num: 0);
17631
17632 // TODO: Extend type shouldn't matter (assuming legal types).
17633 if (Shift.getOpcode() == ISD::ZERO_EXTEND)
17634 Shift = Shift.getOperand(i: 0);
17635
17636 if (Shift.getOpcode() == ISD::SRL || Shift.getOpcode() == ISD::SHL) {
17637 // cvt_f32_ubyte1 (shl x, 8) -> cvt_f32_ubyte0 x
17638 // cvt_f32_ubyte3 (shl x, 16) -> cvt_f32_ubyte1 x
17639 // cvt_f32_ubyte0 (srl x, 16) -> cvt_f32_ubyte2 x
17640 // cvt_f32_ubyte1 (srl x, 16) -> cvt_f32_ubyte3 x
17641 // cvt_f32_ubyte0 (srl x, 8) -> cvt_f32_ubyte1 x
17642 if (auto *C = dyn_cast<ConstantSDNode>(Val: Shift.getOperand(i: 1))) {
17643 SDValue Shifted = DAG.getZExtOrTrunc(
17644 Op: Shift.getOperand(i: 0), DL: SDLoc(Shift.getOperand(i: 0)), VT: MVT::i32);
17645
17646 unsigned ShiftOffset = 8 * Offset;
17647 if (Shift.getOpcode() == ISD::SHL)
17648 ShiftOffset -= C->getZExtValue();
17649 else
17650 ShiftOffset += C->getZExtValue();
17651
17652 if (ShiftOffset < 32 && (ShiftOffset % 8) == 0) {
17653 return DAG.getNode(Opcode: AMDGPUISD::CVT_F32_UBYTE0 + ShiftOffset / 8, DL: SL,
17654 VT: MVT::f32, Operand: Shifted);
17655 }
17656 }
17657 }
17658
17659 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17660 APInt DemandedBits = APInt::getBitsSet(numBits: 32, loBit: 8 * Offset, hiBit: 8 * Offset + 8);
17661 if (TLI.SimplifyDemandedBits(Op: Src, DemandedBits, DCI)) {
17662 // We simplified Src. If this node is not dead, visit it again so it is
17663 // folded properly.
17664 if (N->getOpcode() != ISD::DELETED_NODE)
17665 DCI.AddToWorklist(N);
17666 return SDValue(N, 0);
17667 }
17668
17669 // Handle (or x, (srl y, 8)) pattern when known bits are zero.
17670 if (SDValue DemandedSrc =
17671 TLI.SimplifyMultipleUseDemandedBits(Op: Src, DemandedBits, DAG))
17672 return DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: MVT::f32, Operand: DemandedSrc);
17673
17674 return SDValue();
17675}
17676
17677SDValue SITargetLowering::performClampCombine(SDNode *N,
17678 DAGCombinerInfo &DCI) const {
17679 ConstantFPSDNode *CSrc = dyn_cast<ConstantFPSDNode>(Val: N->getOperand(Num: 0));
17680 if (!CSrc)
17681 return SDValue();
17682
17683 const MachineFunction &MF = DCI.DAG.getMachineFunction();
17684 const APFloat &F = CSrc->getValueAPF();
17685 APFloat Zero = APFloat::getZero(Sem: F.getSemantics());
17686 if (F < Zero ||
17687 (F.isNaN() && MF.getInfo<SIMachineFunctionInfo>()->getMode().DX10Clamp)) {
17688 return DCI.DAG.getConstantFP(Val: Zero, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
17689 }
17690
17691 APFloat One(F.getSemantics(), "1.0");
17692 if (F > One)
17693 return DCI.DAG.getConstantFP(Val: One, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
17694
17695 return SDValue(CSrc, 0);
17696}
17697
17698SDValue SITargetLowering::performSelectCombine(SDNode *N,
17699 DAGCombinerInfo &DCI) const {
17700
17701 // Try to fold CMP + SELECT patterns with shared constants (both FP and
17702 // integer).
17703 // Detect when CMP and SELECT use the same constant and fold them to avoid
17704 // loading the constant twice. Specifically handles patterns like:
17705 // %cmp = icmp eq i32 %val, 4242
17706 // %sel = select i1 %cmp, i32 4242, i32 %other
17707 // It can be optimized to reuse %val instead of 4242 in select.
17708 SDValue Cond = N->getOperand(Num: 0);
17709 SDValue TrueVal = N->getOperand(Num: 1);
17710 SDValue FalseVal = N->getOperand(Num: 2);
17711
17712 // Check if condition is a comparison.
17713 if (Cond.getOpcode() != ISD::SETCC)
17714 return SDValue();
17715
17716 SDValue LHS = Cond.getOperand(i: 0);
17717 SDValue RHS = Cond.getOperand(i: 1);
17718 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
17719
17720 bool isFloatingPoint = LHS.getValueType().isFloatingPoint();
17721 bool isInteger = LHS.getValueType().isInteger();
17722
17723 // Handle simple floating-point and integer types only.
17724 if (!isFloatingPoint && !isInteger)
17725 return SDValue();
17726
17727 bool isEquality = CC == (isFloatingPoint ? ISD::SETOEQ : ISD::SETEQ);
17728 bool isNonEquality = CC == (isFloatingPoint ? ISD::SETONE : ISD::SETNE);
17729 if (!isEquality && !isNonEquality)
17730 return SDValue();
17731
17732 SDValue ArgVal, ConstVal;
17733 if ((isFloatingPoint && isa<ConstantFPSDNode>(Val: RHS)) ||
17734 (isInteger && isa<ConstantSDNode>(Val: RHS))) {
17735 ConstVal = RHS;
17736 ArgVal = LHS;
17737 } else if ((isFloatingPoint && isa<ConstantFPSDNode>(Val: LHS)) ||
17738 (isInteger && isa<ConstantSDNode>(Val: LHS))) {
17739 ConstVal = LHS;
17740 ArgVal = RHS;
17741 } else {
17742 return SDValue();
17743 }
17744
17745 // Skip optimization for inlinable immediates.
17746 if (isFloatingPoint) {
17747 const APFloat &Val = cast<ConstantFPSDNode>(Val&: ConstVal)->getValueAPF();
17748 if (!Val.isNormal() || Subtarget->getInstrInfo()->isInlineConstant(Imm: Val))
17749 return SDValue();
17750 } else {
17751 if (AMDGPU::isInlinableIntLiteral(
17752 Literal: cast<ConstantSDNode>(Val&: ConstVal)->getSExtValue()))
17753 return SDValue();
17754 }
17755
17756 // For equality and non-equality comparisons, patterns:
17757 // select (setcc x, const), const, y -> select (setcc x, const), x, y
17758 // select (setccinv x, const), y, const -> select (setccinv x, const), y, x
17759 if (!(isEquality && TrueVal == ConstVal) &&
17760 !(isNonEquality && FalseVal == ConstVal))
17761 return SDValue();
17762
17763 SDValue SelectLHS = (isEquality && TrueVal == ConstVal) ? ArgVal : TrueVal;
17764 SDValue SelectRHS =
17765 (isNonEquality && FalseVal == ConstVal) ? ArgVal : FalseVal;
17766 return DCI.DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Cond,
17767 N2: SelectLHS, N3: SelectRHS);
17768}
17769
17770SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
17771 DAGCombinerInfo &DCI) const {
17772 switch (N->getOpcode()) {
17773 case ISD::ADD:
17774 case ISD::SUB:
17775 case ISD::SHL:
17776 case ISD::SRL:
17777 case ISD::SRA:
17778 case ISD::AND:
17779 case ISD::OR:
17780 case ISD::XOR:
17781 case ISD::MUL:
17782 case ISD::SETCC:
17783 case ISD::SELECT:
17784 case ISD::SMIN:
17785 case ISD::SMAX:
17786 case ISD::UMIN:
17787 case ISD::UMAX:
17788 if (auto Res = promoteUniformOpToI32(Op: SDValue(N, 0), DCI))
17789 return Res;
17790 break;
17791 default:
17792 break;
17793 }
17794
17795 if (getTargetMachine().getOptLevel() == CodeGenOptLevel::None)
17796 return SDValue();
17797
17798 switch (N->getOpcode()) {
17799 case ISD::ADD:
17800 return performAddCombine(N, DCI);
17801 case ISD::PTRADD:
17802 return performPtrAddCombine(N, DCI);
17803 case ISD::SUB:
17804 return performSubCombine(N, DCI);
17805 case ISD::UADDO_CARRY:
17806 case ISD::USUBO_CARRY:
17807 return performAddCarrySubCarryCombine(N, DCI);
17808 case ISD::FADD:
17809 return performFAddCombine(N, DCI);
17810 case ISD::FSUB:
17811 return performFSubCombine(N, DCI);
17812 case ISD::FDIV:
17813 return performFDivCombine(N, DCI);
17814 case ISD::FMUL:
17815 return performFMulCombine(N, DCI);
17816 case ISD::SETCC:
17817 return performSetCCCombine(N, DCI);
17818 case ISD::SELECT:
17819 if (auto Res = performSelectCombine(N, DCI))
17820 return Res;
17821 break;
17822 case ISD::FMAXNUM:
17823 case ISD::FMINNUM:
17824 case ISD::FMAXNUM_IEEE:
17825 case ISD::FMINNUM_IEEE:
17826 case ISD::FMAXIMUM:
17827 case ISD::FMINIMUM:
17828 case ISD::FMAXIMUMNUM:
17829 case ISD::FMINIMUMNUM:
17830 case ISD::SMAX:
17831 case ISD::SMIN:
17832 case ISD::UMAX:
17833 case ISD::UMIN:
17834 case AMDGPUISD::FMIN_LEGACY:
17835 case AMDGPUISD::FMAX_LEGACY:
17836 return performMinMaxCombine(N, DCI);
17837 case ISD::FMA:
17838 return performFMACombine(N, DCI);
17839 case ISD::AND:
17840 return performAndCombine(N, DCI);
17841 case ISD::OR:
17842 return performOrCombine(N, DCI);
17843 case ISD::FSHR: {
17844 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
17845 if (N->getValueType(ResNo: 0) == MVT::i32 && N->isDivergent() &&
17846 TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
17847 return matchPERM(N, DCI);
17848 }
17849 break;
17850 }
17851 case ISD::XOR:
17852 return performXorCombine(N, DCI);
17853 case ISD::ANY_EXTEND:
17854 case ISD::ZERO_EXTEND:
17855 return performZeroOrAnyExtendCombine(N, DCI);
17856 case ISD::SIGN_EXTEND_INREG:
17857 return performSignExtendInRegCombine(N, DCI);
17858 case AMDGPUISD::FP_CLASS:
17859 return performClassCombine(N, DCI);
17860 case ISD::FCANONICALIZE:
17861 return performFCanonicalizeCombine(N, DCI);
17862 case AMDGPUISD::RCP:
17863 return performRcpCombine(N, DCI);
17864 case ISD::FLDEXP:
17865 case AMDGPUISD::FRACT:
17866 case AMDGPUISD::RSQ:
17867 case AMDGPUISD::RCP_LEGACY:
17868 case AMDGPUISD::RCP_IFLAG:
17869 case AMDGPUISD::RSQ_CLAMP: {
17870 // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
17871 SDValue Src = N->getOperand(Num: 0);
17872 if (Src.isUndef())
17873 return Src;
17874 break;
17875 }
17876 case ISD::SINT_TO_FP:
17877 case ISD::UINT_TO_FP:
17878 return performUCharToFloatCombine(N, DCI);
17879 case ISD::FCOPYSIGN:
17880 return performFCopySignCombine(N, DCI);
17881 case AMDGPUISD::CVT_F32_UBYTE0:
17882 case AMDGPUISD::CVT_F32_UBYTE1:
17883 case AMDGPUISD::CVT_F32_UBYTE2:
17884 case AMDGPUISD::CVT_F32_UBYTE3:
17885 return performCvtF32UByteNCombine(N, DCI);
17886 case AMDGPUISD::FMED3:
17887 return performFMed3Combine(N, DCI);
17888 case AMDGPUISD::CVT_PKRTZ_F16_F32:
17889 return performCvtPkRTZCombine(N, DCI);
17890 case AMDGPUISD::CLAMP:
17891 return performClampCombine(N, DCI);
17892 case ISD::SCALAR_TO_VECTOR: {
17893 SelectionDAG &DAG = DCI.DAG;
17894 EVT VT = N->getValueType(ResNo: 0);
17895
17896 // v2i16 (scalar_to_vector i16:x) -> v2i16 (bitcast (any_extend i16:x))
17897 if (VT == MVT::v2i16 || VT == MVT::v2f16 || VT == MVT::v2bf16) {
17898 SDLoc SL(N);
17899 SDValue Src = N->getOperand(Num: 0);
17900 EVT EltVT = Src.getValueType();
17901 if (EltVT != MVT::i16)
17902 Src = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Src);
17903
17904 SDValue Ext = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Src);
17905 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Ext);
17906 }
17907
17908 break;
17909 }
17910 case ISD::EXTRACT_VECTOR_ELT:
17911 return performExtractVectorEltCombine(N, DCI);
17912 case ISD::INSERT_VECTOR_ELT:
17913 return performInsertVectorEltCombine(N, DCI);
17914 case ISD::FP_ROUND:
17915 return performFPRoundCombine(N, DCI);
17916 case ISD::LOAD: {
17917 if (SDValue Widened = widenLoad(Ld: cast<LoadSDNode>(Val: N), DCI))
17918 return Widened;
17919 [[fallthrough]];
17920 }
17921 default: {
17922 if (!DCI.isBeforeLegalize()) {
17923 if (MemSDNode *MemNode = dyn_cast<MemSDNode>(Val: N))
17924 return performMemSDNodeCombine(N: MemNode, DCI);
17925 }
17926
17927 break;
17928 }
17929 }
17930
17931 return AMDGPUTargetLowering::PerformDAGCombine(N, DCI);
17932}
17933
17934/// Helper function for adjustWritemask
17935static unsigned SubIdx2Lane(unsigned Idx) {
17936 switch (Idx) {
17937 default:
17938 return ~0u;
17939 case AMDGPU::sub0:
17940 return 0;
17941 case AMDGPU::sub1:
17942 return 1;
17943 case AMDGPU::sub2:
17944 return 2;
17945 case AMDGPU::sub3:
17946 return 3;
17947 case AMDGPU::sub4:
17948 return 4; // Possible with TFE/LWE
17949 }
17950}
17951
17952/// Adjust the writemask of MIMG, VIMAGE or VSAMPLE instructions
17953SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
17954 SelectionDAG &DAG) const {
17955 unsigned Opcode = Node->getMachineOpcode();
17956
17957 // Subtract 1 because the vdata output is not a MachineSDNode operand.
17958 int D16Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::d16) - 1;
17959 if (D16Idx >= 0 && Node->getConstantOperandVal(Num: D16Idx))
17960 return Node; // not implemented for D16
17961
17962 SDNode *Users[5] = {nullptr};
17963 unsigned Lane = 0;
17964 unsigned DmaskIdx =
17965 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::dmask) - 1;
17966 unsigned OldDmask = Node->getConstantOperandVal(Num: DmaskIdx);
17967 unsigned NewDmask = 0;
17968 unsigned TFEIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::tfe) - 1;
17969 unsigned LWEIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::lwe) - 1;
17970 bool UsesTFC = (int(TFEIdx) >= 0 && Node->getConstantOperandVal(Num: TFEIdx)) ||
17971 (int(LWEIdx) >= 0 && Node->getConstantOperandVal(Num: LWEIdx));
17972 unsigned TFCLane = 0;
17973 bool HasChain = Node->getNumValues() > 1;
17974
17975 if (OldDmask == 0) {
17976 // These are folded out, but on the chance it happens don't assert.
17977 return Node;
17978 }
17979
17980 unsigned OldBitsSet = llvm::popcount(Value: OldDmask);
17981 // Work out which is the TFE/LWE lane if that is enabled.
17982 if (UsesTFC) {
17983 TFCLane = OldBitsSet;
17984 }
17985
17986 // Try to figure out the used register components
17987 for (SDUse &Use : Node->uses()) {
17988
17989 // Don't look at users of the chain.
17990 if (Use.getResNo() != 0)
17991 continue;
17992
17993 SDNode *User = Use.getUser();
17994
17995 // Abort if we can't understand the usage
17996 if (!User->isMachineOpcode() ||
17997 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
17998 return Node;
17999
18000 // Lane means which subreg of %vgpra_vgprb_vgprc_vgprd is used.
18001 // Note that subregs are packed, i.e. Lane==0 is the first bit set
18002 // in OldDmask, so it can be any of X,Y,Z,W; Lane==1 is the second bit
18003 // set, etc.
18004 Lane = SubIdx2Lane(Idx: User->getConstantOperandVal(Num: 1));
18005 if (Lane == ~0u)
18006 return Node;
18007
18008 // Check if the use is for the TFE/LWE generated result at VGPRn+1.
18009 if (UsesTFC && Lane == TFCLane) {
18010 Users[Lane] = User;
18011 } else {
18012 // Set which texture component corresponds to the lane.
18013 unsigned Comp;
18014 for (unsigned i = 0, Dmask = OldDmask; (i <= Lane) && (Dmask != 0); i++) {
18015 Comp = llvm::countr_zero(Val: Dmask);
18016 Dmask &= ~(1 << Comp);
18017 }
18018
18019 // Abort if we have more than one user per component.
18020 if (Users[Lane])
18021 return Node;
18022
18023 Users[Lane] = User;
18024 NewDmask |= 1 << Comp;
18025 }
18026 }
18027
18028 // Don't allow 0 dmask, as hardware assumes one channel enabled.
18029 bool NoChannels = !NewDmask;
18030 if (NoChannels) {
18031 if (!UsesTFC) {
18032 // No uses of the result and not using TFC. Then do nothing.
18033 return Node;
18034 }
18035 // If the original dmask has one channel - then nothing to do
18036 if (OldBitsSet == 1)
18037 return Node;
18038 // Use an arbitrary dmask - required for the instruction to work
18039 NewDmask = 1;
18040 }
18041 // Abort if there's no change
18042 if (NewDmask == OldDmask)
18043 return Node;
18044
18045 unsigned BitsSet = llvm::popcount(Value: NewDmask);
18046
18047 // Check for TFE or LWE - increase the number of channels by one to account
18048 // for the extra return value
18049 // This will need adjustment for D16 if this is also included in
18050 // adjustWriteMask (this function) but at present D16 are excluded.
18051 unsigned NewChannels = BitsSet + UsesTFC;
18052
18053 int NewOpcode =
18054 AMDGPU::getMaskedMIMGOp(Opc: Node->getMachineOpcode(), NewChannels);
18055 assert(NewOpcode != -1 &&
18056 NewOpcode != static_cast<int>(Node->getMachineOpcode()) &&
18057 "failed to find equivalent MIMG op");
18058
18059 // Adjust the writemask in the node
18060 SmallVector<SDValue, 12> Ops;
18061 llvm::append_range(C&: Ops, R: Node->ops().take_front(N: DmaskIdx));
18062 Ops.push_back(Elt: DAG.getTargetConstant(Val: NewDmask, DL: SDLoc(Node), VT: MVT::i32));
18063 llvm::append_range(C&: Ops, R: Node->ops().drop_front(N: DmaskIdx + 1));
18064
18065 MVT SVT = Node->getValueType(ResNo: 0).getVectorElementType().getSimpleVT();
18066
18067 MVT ResultVT = NewChannels == 1
18068 ? SVT
18069 : MVT::getVectorVT(VT: SVT, NumElements: NewChannels == 3 ? 4
18070 : NewChannels == 5 ? 8
18071 : NewChannels);
18072 SDVTList NewVTList =
18073 HasChain ? DAG.getVTList(VT1: ResultVT, VT2: MVT::Other) : DAG.getVTList(VT: ResultVT);
18074
18075 MachineSDNode *NewNode =
18076 DAG.getMachineNode(Opcode: NewOpcode, dl: SDLoc(Node), VTs: NewVTList, Ops);
18077
18078 if (HasChain) {
18079 // Update chain.
18080 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: Node->memoperands());
18081 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: SDValue(NewNode, 1));
18082 }
18083
18084 if (NewChannels == 1) {
18085 assert(Node->hasNUsesOfValue(1, 0));
18086 SDNode *Copy =
18087 DAG.getMachineNode(Opcode: TargetOpcode::COPY, dl: SDLoc(Node),
18088 VT: Users[Lane]->getValueType(ResNo: 0), Op1: SDValue(NewNode, 0));
18089 DAG.ReplaceAllUsesWith(From: Users[Lane], To: Copy);
18090 return nullptr;
18091 }
18092
18093 // Update the users of the node with the new indices
18094 for (unsigned i = 0, Idx = AMDGPU::sub0; i < 5; ++i) {
18095 SDNode *User = Users[i];
18096 if (!User) {
18097 // Handle the special case of NoChannels. We set NewDmask to 1 above, but
18098 // Users[0] is still nullptr because channel 0 doesn't really have a use.
18099 if (i || !NoChannels)
18100 continue;
18101 } else {
18102 SDValue Op = DAG.getTargetConstant(Val: Idx, DL: SDLoc(User), VT: MVT::i32);
18103 SDNode *NewUser = DAG.UpdateNodeOperands(N: User, Op1: SDValue(NewNode, 0), Op2: Op);
18104 if (NewUser != User) {
18105 DAG.ReplaceAllUsesWith(From: SDValue(User, 0), To: SDValue(NewUser, 0));
18106 DAG.RemoveDeadNode(N: User);
18107 }
18108 }
18109
18110 switch (Idx) {
18111 default:
18112 break;
18113 case AMDGPU::sub0:
18114 Idx = AMDGPU::sub1;
18115 break;
18116 case AMDGPU::sub1:
18117 Idx = AMDGPU::sub2;
18118 break;
18119 case AMDGPU::sub2:
18120 Idx = AMDGPU::sub3;
18121 break;
18122 case AMDGPU::sub3:
18123 Idx = AMDGPU::sub4;
18124 break;
18125 }
18126 }
18127
18128 DAG.RemoveDeadNode(N: Node);
18129 return nullptr;
18130}
18131
18132static bool isFrameIndexOp(SDValue Op) {
18133 if (Op.getOpcode() == ISD::AssertZext)
18134 Op = Op.getOperand(i: 0);
18135
18136 return isa<FrameIndexSDNode>(Val: Op);
18137}
18138
18139/// Legalize target independent instructions (e.g. INSERT_SUBREG)
18140/// with frame index operands.
18141/// LLVM assumes that inputs are to these instructions are registers.
18142SDNode *
18143SITargetLowering::legalizeTargetIndependentNode(SDNode *Node,
18144 SelectionDAG &DAG) const {
18145 if (Node->getOpcode() == ISD::CopyToReg) {
18146 RegisterSDNode *DestReg = cast<RegisterSDNode>(Val: Node->getOperand(Num: 1));
18147 SDValue SrcVal = Node->getOperand(Num: 2);
18148
18149 // Insert a copy to a VReg_1 virtual register so LowerI1Copies doesn't have
18150 // to try understanding copies to physical registers.
18151 if (SrcVal.getValueType() == MVT::i1 && DestReg->getReg().isPhysical()) {
18152 SDLoc SL(Node);
18153 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
18154 SDValue VReg = DAG.getRegister(
18155 Reg: MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_1RegClass), VT: MVT::i1);
18156
18157 SDNode *Glued = Node->getGluedNode();
18158 SDValue ToVReg = DAG.getCopyToReg(
18159 Chain: Node->getOperand(Num: 0), dl: SL, Reg: VReg, N: SrcVal,
18160 Glue: SDValue(Glued, Glued ? Glued->getNumValues() - 1 : 0));
18161 SDValue ToResultReg = DAG.getCopyToReg(Chain: ToVReg, dl: SL, Reg: SDValue(DestReg, 0),
18162 N: VReg, Glue: ToVReg.getValue(R: 1));
18163 DAG.ReplaceAllUsesWith(From: Node, To: ToResultReg.getNode());
18164 DAG.RemoveDeadNode(N: Node);
18165 return ToResultReg.getNode();
18166 }
18167 }
18168
18169 SmallVector<SDValue, 8> Ops;
18170 for (unsigned i = 0; i < Node->getNumOperands(); ++i) {
18171 if (!isFrameIndexOp(Op: Node->getOperand(Num: i))) {
18172 Ops.push_back(Elt: Node->getOperand(Num: i));
18173 continue;
18174 }
18175
18176 SDLoc DL(Node);
18177 Ops.push_back(Elt: SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL,
18178 VT: Node->getOperand(Num: i).getValueType(),
18179 Op1: Node->getOperand(Num: i)),
18180 0));
18181 }
18182
18183 return DAG.UpdateNodeOperands(N: Node, Ops);
18184}
18185
18186/// Fold the instructions after selecting them.
18187/// Returns null if users were already updated.
18188SDNode *SITargetLowering::PostISelFolding(MachineSDNode *Node,
18189 SelectionDAG &DAG) const {
18190 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
18191 unsigned Opcode = Node->getMachineOpcode();
18192
18193 if (TII->isImage(Opcode) && !TII->get(Opcode).mayStore() &&
18194 !TII->isGather4(Opcode) &&
18195 AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::dmask)) {
18196 return adjustWritemask(Node, DAG);
18197 }
18198
18199 if (Opcode == AMDGPU::INSERT_SUBREG || Opcode == AMDGPU::REG_SEQUENCE) {
18200 legalizeTargetIndependentNode(Node, DAG);
18201 return Node;
18202 }
18203
18204 switch (Opcode) {
18205 case AMDGPU::V_DIV_SCALE_F32_e64:
18206 case AMDGPU::V_DIV_SCALE_F64_e64: {
18207 // Satisfy the operand register constraint when one of the inputs is
18208 // undefined. Ordinarily each undef value will have its own implicit_def of
18209 // a vreg, so force these to use a single register.
18210 SDValue Src0 = Node->getOperand(Num: 1);
18211 SDValue Src1 = Node->getOperand(Num: 3);
18212 SDValue Src2 = Node->getOperand(Num: 5);
18213
18214 if ((Src0.isMachineOpcode() &&
18215 Src0.getMachineOpcode() != AMDGPU::IMPLICIT_DEF) &&
18216 (Src0 == Src1 || Src0 == Src2))
18217 break;
18218
18219 MVT VT = Src0.getValueType().getSimpleVT();
18220 const TargetRegisterClass *RC =
18221 getRegClassFor(VT, isDivergent: Src0.getNode()->isDivergent());
18222
18223 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
18224 SDValue UndefReg = DAG.getRegister(Reg: MRI.createVirtualRegister(RegClass: RC), VT);
18225
18226 SDValue ImpDef = DAG.getCopyToReg(Chain: DAG.getEntryNode(), dl: SDLoc(Node), Reg: UndefReg,
18227 N: Src0, Glue: SDValue());
18228
18229 // src0 must be the same register as src1 or src2, even if the value is
18230 // undefined, so make sure we don't violate this constraint.
18231 if (Src0.isMachineOpcode() &&
18232 Src0.getMachineOpcode() == AMDGPU::IMPLICIT_DEF) {
18233 if (Src1.isMachineOpcode() &&
18234 Src1.getMachineOpcode() != AMDGPU::IMPLICIT_DEF)
18235 Src0 = Src1;
18236 else if (Src2.isMachineOpcode() &&
18237 Src2.getMachineOpcode() != AMDGPU::IMPLICIT_DEF)
18238 Src0 = Src2;
18239 else {
18240 assert(Src1.getMachineOpcode() == AMDGPU::IMPLICIT_DEF);
18241 Src0 = UndefReg;
18242 Src1 = UndefReg;
18243 }
18244 } else
18245 break;
18246
18247 SmallVector<SDValue, 9> Ops(Node->ops());
18248 Ops[1] = Src0;
18249 Ops[3] = Src1;
18250 Ops[5] = Src2;
18251 Ops.push_back(Elt: ImpDef.getValue(R: 1));
18252 return DAG.getMachineNode(Opcode, dl: SDLoc(Node), VTs: Node->getVTList(), Ops);
18253 }
18254 default:
18255 break;
18256 }
18257
18258 return Node;
18259}
18260
18261// Any MIMG instructions that use tfe or lwe require an initialization of the
18262// result register that will be written in the case of a memory access failure.
18263// The required code is also added to tie this init code to the result of the
18264// img instruction.
18265void SITargetLowering::AddMemOpInit(MachineInstr &MI) const {
18266 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
18267 const SIRegisterInfo &TRI = TII->getRegisterInfo();
18268 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
18269 MachineBasicBlock &MBB = *MI.getParent();
18270
18271 int DstIdx =
18272 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdata);
18273 unsigned InitIdx = 0;
18274
18275 if (TII->isImage(MI)) {
18276 MachineOperand *TFE = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe);
18277 MachineOperand *LWE = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::lwe);
18278 MachineOperand *D16 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::d16);
18279
18280 if (!TFE && !LWE) // intersect_ray
18281 return;
18282
18283 unsigned TFEVal = TFE ? TFE->getImm() : 0;
18284 unsigned LWEVal = LWE ? LWE->getImm() : 0;
18285 unsigned D16Val = D16 ? D16->getImm() : 0;
18286
18287 if (!TFEVal && !LWEVal)
18288 return;
18289
18290 // At least one of TFE or LWE are non-zero
18291 // We have to insert a suitable initialization of the result value and
18292 // tie this to the dest of the image instruction.
18293
18294 // Calculate which dword we have to initialize to 0.
18295 MachineOperand *MO_Dmask = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dmask);
18296
18297 // check that dmask operand is found.
18298 assert(MO_Dmask && "Expected dmask operand in instruction");
18299
18300 unsigned dmask = MO_Dmask->getImm();
18301 // Determine the number of active lanes taking into account the
18302 // Gather4 special case
18303 unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(Value: dmask);
18304
18305 bool Packed = !Subtarget->hasUnpackedD16VMem();
18306
18307 InitIdx = D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1;
18308
18309 // Abandon attempt if the dst size isn't large enough
18310 // - this is in fact an error but this is picked up elsewhere and
18311 // reported correctly.
18312 const TargetRegisterClass *DstRC = TII->getRegClass(MCID: MI.getDesc(), OpNum: DstIdx);
18313
18314 uint32_t DstSize = TRI.getRegSizeInBits(RC: *DstRC) / 32;
18315 if (DstSize < InitIdx)
18316 return;
18317 } else if (TII->isMUBUF(MI) && AMDGPU::getMUBUFTfe(Opc: MI.getOpcode())) {
18318 const TargetRegisterClass *DstRC = TII->getRegClass(MCID: MI.getDesc(), OpNum: DstIdx);
18319 InitIdx = TRI.getRegSizeInBits(RC: *DstRC) / 32;
18320 } else {
18321 return;
18322 }
18323
18324 const DebugLoc &DL = MI.getDebugLoc();
18325
18326 // Create a register for the initialization value.
18327 Register PrevDst = MRI.cloneVirtualRegister(VReg: MI.getOperand(i: DstIdx).getReg());
18328 unsigned NewDst = 0; // Final initialized value will be in here
18329
18330 // If PRTStrictNull feature is enabled (the default) then initialize
18331 // all the result registers to 0, otherwise just the error indication
18332 // register (VGPRn+1)
18333 unsigned SizeLeft = Subtarget->usePRTStrictNull() ? InitIdx : 1;
18334 unsigned CurrIdx = Subtarget->usePRTStrictNull() ? 0 : (InitIdx - 1);
18335
18336 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: PrevDst);
18337 for (; SizeLeft; SizeLeft--, CurrIdx++) {
18338 NewDst = MRI.createVirtualRegister(RegClass: TII->getOpRegClass(MI, OpNo: DstIdx));
18339 // Initialize dword
18340 Register SubReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
18341 // clang-format off
18342 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: SubReg)
18343 .addImm(Val: 0);
18344 // clang-format on
18345 // Insert into the super-reg
18346 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: NewDst)
18347 .addReg(RegNo: PrevDst)
18348 .addReg(RegNo: SubReg)
18349 .addImm(Val: SIRegisterInfo::getSubRegFromChannel(Channel: CurrIdx));
18350
18351 PrevDst = NewDst;
18352 }
18353
18354 // Add as an implicit operand
18355 MI.addOperand(Op: MachineOperand::CreateReg(Reg: NewDst, isDef: false, isImp: true));
18356
18357 // Tie the just added implicit operand to the dst
18358 MI.tieOperands(DefIdx: DstIdx, UseIdx: MI.getNumOperands() - 1);
18359}
18360
18361/// Assign the register class depending on the number of
18362/// bits set in the writemask
18363void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
18364 SDNode *Node) const {
18365 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
18366
18367 MachineFunction *MF = MI.getMF();
18368 MachineRegisterInfo &MRI = MF->getRegInfo();
18369
18370 if (TII->isVOP3(Opcode: MI.getOpcode())) {
18371 // Make sure constant bus requirements are respected.
18372 TII->legalizeOperandsVOP3(MRI, MI);
18373
18374 if (TII->isMAI(MI)) {
18375 // The ordinary src0, src1, src2 were legalized above.
18376 //
18377 // We have to also legalize the appended v_mfma_ld_scale_b32 operands,
18378 // as a separate instruction.
18379 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
18380 Name: AMDGPU::OpName::scale_src0);
18381 if (Src0Idx != -1) {
18382 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
18383 Name: AMDGPU::OpName::scale_src1);
18384 if (TII->usesConstantBus(MRI, MI, OpIdx: Src0Idx) &&
18385 TII->usesConstantBus(MRI, MI, OpIdx: Src1Idx))
18386 TII->legalizeOpWithMove(MI, OpIdx: Src1Idx);
18387 }
18388 }
18389
18390 return;
18391 }
18392
18393 if (TII->isImage(MI))
18394 TII->enforceOperandRCAlignment(MI, OpName: AMDGPU::OpName::vaddr);
18395}
18396
18397static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL,
18398 uint64_t Val) {
18399 SDValue K = DAG.getTargetConstant(Val, DL, VT: MVT::i32);
18400 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: K), 0);
18401}
18402
18403MachineSDNode *SITargetLowering::wrapAddr64Rsrc(SelectionDAG &DAG,
18404 const SDLoc &DL,
18405 SDValue Ptr) const {
18406 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
18407
18408 // Build the half of the subregister with the constants before building the
18409 // full 128-bit register. If we are building multiple resource descriptors,
18410 // this will allow CSEing of the 2-component register.
18411 const SDValue Ops0[] = {
18412 DAG.getTargetConstant(Val: AMDGPU::SGPR_64RegClassID, DL, VT: MVT::i32),
18413 buildSMovImm32(DAG, DL, Val: 0),
18414 DAG.getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32),
18415 buildSMovImm32(DAG, DL, Val: TII->getDefaultRsrcDataFormat() >> 32),
18416 DAG.getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32)};
18417
18418 SDValue SubRegHi = SDValue(
18419 DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v2i32, Ops: Ops0), 0);
18420
18421 // Combine the constants and the pointer.
18422 const SDValue Ops1[] = {
18423 DAG.getTargetConstant(Val: AMDGPU::SGPR_128RegClassID, DL, VT: MVT::i32), Ptr,
18424 DAG.getTargetConstant(Val: AMDGPU::sub0_sub1, DL, VT: MVT::i32), SubRegHi,
18425 DAG.getTargetConstant(Val: AMDGPU::sub2_sub3, DL, VT: MVT::i32)};
18426
18427 return DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v4i32, Ops: Ops1);
18428}
18429
18430/// Return a resource descriptor with the 'Add TID' bit enabled
18431/// The TID (Thread ID) is multiplied by the stride value (bits [61:48]
18432/// of the resource descriptor) to create an offset, which is added to
18433/// the resource pointer.
18434MachineSDNode *SITargetLowering::buildRSRC(SelectionDAG &DAG, const SDLoc &DL,
18435 SDValue Ptr, uint32_t RsrcDword1,
18436 uint64_t RsrcDword2And3) const {
18437 SDValue PtrLo = DAG.getTargetExtractSubreg(SRIdx: AMDGPU::sub0, DL, VT: MVT::i32, Operand: Ptr);
18438 SDValue PtrHi = DAG.getTargetExtractSubreg(SRIdx: AMDGPU::sub1, DL, VT: MVT::i32, Operand: Ptr);
18439 if (RsrcDword1) {
18440 PtrHi =
18441 SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_OR_B32, dl: DL, VT: MVT::i32, Op1: PtrHi,
18442 Op2: DAG.getConstant(Val: RsrcDword1, DL, VT: MVT::i32)),
18443 0);
18444 }
18445
18446 SDValue DataLo =
18447 buildSMovImm32(DAG, DL, Val: RsrcDword2And3 & UINT64_C(0xFFFFFFFF));
18448 SDValue DataHi = buildSMovImm32(DAG, DL, Val: RsrcDword2And3 >> 32);
18449
18450 const SDValue Ops[] = {
18451 DAG.getTargetConstant(Val: AMDGPU::SGPR_128RegClassID, DL, VT: MVT::i32),
18452 PtrLo,
18453 DAG.getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32),
18454 PtrHi,
18455 DAG.getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32),
18456 DataLo,
18457 DAG.getTargetConstant(Val: AMDGPU::sub2, DL, VT: MVT::i32),
18458 DataHi,
18459 DAG.getTargetConstant(Val: AMDGPU::sub3, DL, VT: MVT::i32)};
18460
18461 return DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v4i32, Ops);
18462}
18463
18464//===----------------------------------------------------------------------===//
18465// SI Inline Assembly Support
18466//===----------------------------------------------------------------------===//
18467
18468std::pair<unsigned, const TargetRegisterClass *>
18469SITargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI_,
18470 StringRef Constraint,
18471 MVT VT) const {
18472 const SIRegisterInfo *TRI = static_cast<const SIRegisterInfo *>(TRI_);
18473
18474 const TargetRegisterClass *RC = nullptr;
18475 if (Constraint.size() == 1) {
18476 // Check if we cannot determine the bit size of the given value type. This
18477 // can happen, for example, in this situation where we have an empty struct
18478 // (size 0): `call void asm "", "v"({} poison)`-
18479 if (VT == MVT::Other)
18480 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
18481 const unsigned BitWidth = VT.getSizeInBits();
18482 switch (Constraint[0]) {
18483 default:
18484 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
18485 case 's':
18486 case 'r':
18487 switch (BitWidth) {
18488 case 16:
18489 RC = &AMDGPU::SReg_32RegClass;
18490 break;
18491 case 64:
18492 RC = &AMDGPU::SGPR_64RegClass;
18493 break;
18494 default:
18495 RC = SIRegisterInfo::getSGPRClassForBitWidth(BitWidth);
18496 if (!RC)
18497 return std::pair(0U, nullptr);
18498 break;
18499 }
18500 break;
18501 case 'v':
18502 switch (BitWidth) {
18503 case 1:
18504 return std::pair(0U, nullptr);
18505 case 16:
18506 RC = Subtarget->useRealTrue16Insts() ? &AMDGPU::VGPR_16RegClass
18507 : &AMDGPU::VGPR_32_Lo256RegClass;
18508 break;
18509 default:
18510 RC = Subtarget->has1024AddressableVGPRs()
18511 ? TRI->getAlignedLo256VGPRClassForBitWidth(BitWidth)
18512 : TRI->getVGPRClassForBitWidth(BitWidth);
18513 if (!RC)
18514 return std::pair(0U, nullptr);
18515 break;
18516 }
18517 break;
18518 case 'a':
18519 if (!Subtarget->hasMAIInsts())
18520 break;
18521 switch (BitWidth) {
18522 case 1:
18523 return std::pair(0U, nullptr);
18524 case 16:
18525 RC = &AMDGPU::AGPR_32RegClass;
18526 break;
18527 default:
18528 RC = TRI->getAGPRClassForBitWidth(BitWidth);
18529 if (!RC)
18530 return std::pair(0U, nullptr);
18531 break;
18532 }
18533 break;
18534 }
18535 } else if (Constraint == "VA" && Subtarget->hasGFX90AInsts()) {
18536 const unsigned BitWidth = VT.getSizeInBits();
18537 switch (BitWidth) {
18538 case 16:
18539 RC = &AMDGPU::AV_32RegClass;
18540 break;
18541 default:
18542 RC = TRI->getVectorSuperClassForBitWidth(BitWidth);
18543 if (!RC)
18544 return std::pair(0U, nullptr);
18545 break;
18546 }
18547 }
18548
18549 // We actually support i128, i16 and f16 as inline parameters
18550 // even if they are not reported as legal
18551 if (RC && (isTypeLegal(VT) || VT.SimpleTy == MVT::i128 ||
18552 VT.SimpleTy == MVT::i16 || VT.SimpleTy == MVT::f16))
18553 return std::pair(0U, RC);
18554
18555 auto [Kind, Idx, NumRegs] = AMDGPU::parseAsmConstraintPhysReg(Constraint);
18556 if (Kind != '\0') {
18557 if (Kind == 'v') {
18558 RC = &AMDGPU::VGPR_32_Lo256RegClass;
18559 } else if (Kind == 's') {
18560 RC = &AMDGPU::SGPR_32RegClass;
18561 } else if (Kind == 'a') {
18562 RC = &AMDGPU::AGPR_32RegClass;
18563 }
18564
18565 if (RC) {
18566 if (NumRegs > 1) {
18567 if (Idx >= RC->getNumRegs() || Idx + NumRegs - 1 >= RC->getNumRegs())
18568 return std::pair(0U, nullptr);
18569
18570 uint32_t Width = NumRegs * 32;
18571 // Prohibit constraints for register ranges with a width that does not
18572 // match the required type.
18573 if (VT.SimpleTy != MVT::Other && Width != VT.getSizeInBits())
18574 return std::pair(0U, nullptr);
18575
18576 MCRegister Reg = RC->getRegister(i: Idx);
18577 if (SIRegisterInfo::isVGPRClass(RC))
18578 RC = TRI->getVGPRClassForBitWidth(BitWidth: Width);
18579 else if (SIRegisterInfo::isSGPRClass(RC))
18580 RC = TRI->getSGPRClassForBitWidth(BitWidth: Width);
18581 else if (SIRegisterInfo::isAGPRClass(RC))
18582 RC = TRI->getAGPRClassForBitWidth(BitWidth: Width);
18583 if (RC) {
18584 Reg = TRI->getMatchingSuperReg(Reg, SubIdx: AMDGPU::sub0, RC);
18585 if (!Reg) {
18586 // The register class does not contain the requested register,
18587 // e.g., because it is an SGPR pair that would violate alignment
18588 // requirements.
18589 return std::pair(0U, nullptr);
18590 }
18591 return std::pair(Reg, RC);
18592 }
18593 }
18594
18595 // Check for lossy scalar/vector conversions.
18596 if (VT.isVector() && VT.getSizeInBits() != 32)
18597 return std::pair(0U, nullptr);
18598 if (Idx < RC->getNumRegs())
18599 return std::pair(RC->getRegister(i: Idx), RC);
18600 return std::pair(0U, nullptr);
18601 }
18602 }
18603
18604 auto Ret = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
18605 if (Ret.first)
18606 Ret.second = TRI->getPhysRegBaseClass(Reg: Ret.first);
18607
18608 return Ret;
18609}
18610
18611static bool isImmConstraint(StringRef Constraint) {
18612 if (Constraint.size() == 1) {
18613 switch (Constraint[0]) {
18614 default:
18615 break;
18616 case 'I':
18617 case 'J':
18618 case 'A':
18619 case 'B':
18620 case 'C':
18621 return true;
18622 }
18623 } else if (Constraint == "DA" || Constraint == "DB") {
18624 return true;
18625 }
18626 return false;
18627}
18628
18629SITargetLowering::ConstraintType
18630SITargetLowering::getConstraintType(StringRef Constraint) const {
18631 if (Constraint.size() == 1) {
18632 switch (Constraint[0]) {
18633 default:
18634 break;
18635 case 's':
18636 case 'v':
18637 case 'a':
18638 return C_RegisterClass;
18639 }
18640 } else if (Constraint.size() == 2) {
18641 if (Constraint == "VA")
18642 return C_RegisterClass;
18643 }
18644 if (isImmConstraint(Constraint)) {
18645 return C_Other;
18646 }
18647 return TargetLowering::getConstraintType(Constraint);
18648}
18649
18650static uint64_t clearUnusedBits(uint64_t Val, unsigned Size) {
18651 if (!AMDGPU::isInlinableIntLiteral(Literal: Val)) {
18652 Val = Val & maskTrailingOnes<uint64_t>(N: Size);
18653 }
18654 return Val;
18655}
18656
18657void SITargetLowering::LowerAsmOperandForConstraint(SDValue Op,
18658 StringRef Constraint,
18659 std::vector<SDValue> &Ops,
18660 SelectionDAG &DAG) const {
18661 if (isImmConstraint(Constraint)) {
18662 uint64_t Val;
18663 if (getAsmOperandConstVal(Op, Val) &&
18664 checkAsmConstraintVal(Op, Constraint, Val)) {
18665 Val = clearUnusedBits(Val, Size: Op.getScalarValueSizeInBits());
18666 Ops.push_back(x: DAG.getTargetConstant(Val, DL: SDLoc(Op), VT: MVT::i64));
18667 }
18668 } else {
18669 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
18670 }
18671}
18672
18673bool SITargetLowering::getAsmOperandConstVal(SDValue Op, uint64_t &Val) const {
18674 unsigned Size = Op.getScalarValueSizeInBits();
18675 if (Size > 64)
18676 return false;
18677
18678 if (Size == 16 && !Subtarget->has16BitInsts())
18679 return false;
18680
18681 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
18682 Val = C->getSExtValue();
18683 return true;
18684 }
18685 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
18686 Val = C->getValueAPF().bitcastToAPInt().getSExtValue();
18687 return true;
18688 }
18689 if (BuildVectorSDNode *V = dyn_cast<BuildVectorSDNode>(Val&: Op)) {
18690 if (Size != 16 || Op.getNumOperands() != 2)
18691 return false;
18692 if (Op.getOperand(i: 0).isUndef() || Op.getOperand(i: 1).isUndef())
18693 return false;
18694 if (ConstantSDNode *C = V->getConstantSplatNode()) {
18695 Val = C->getSExtValue();
18696 return true;
18697 }
18698 if (ConstantFPSDNode *C = V->getConstantFPSplatNode()) {
18699 Val = C->getValueAPF().bitcastToAPInt().getSExtValue();
18700 return true;
18701 }
18702 }
18703
18704 return false;
18705}
18706
18707bool SITargetLowering::checkAsmConstraintVal(SDValue Op, StringRef Constraint,
18708 uint64_t Val) const {
18709 if (Constraint.size() == 1) {
18710 switch (Constraint[0]) {
18711 case 'I':
18712 return AMDGPU::isInlinableIntLiteral(Literal: Val);
18713 case 'J':
18714 return isInt<16>(x: Val);
18715 case 'A':
18716 return checkAsmConstraintValA(Op, Val);
18717 case 'B':
18718 return isInt<32>(x: Val);
18719 case 'C':
18720 return isUInt<32>(x: clearUnusedBits(Val, Size: Op.getScalarValueSizeInBits())) ||
18721 AMDGPU::isInlinableIntLiteral(Literal: Val);
18722 default:
18723 break;
18724 }
18725 } else if (Constraint.size() == 2) {
18726 if (Constraint == "DA") {
18727 int64_t HiBits = static_cast<int32_t>(Val >> 32);
18728 int64_t LoBits = static_cast<int32_t>(Val);
18729 return checkAsmConstraintValA(Op, Val: HiBits, MaxSize: 32) &&
18730 checkAsmConstraintValA(Op, Val: LoBits, MaxSize: 32);
18731 }
18732 if (Constraint == "DB") {
18733 return true;
18734 }
18735 }
18736 llvm_unreachable("Invalid asm constraint");
18737}
18738
18739bool SITargetLowering::checkAsmConstraintValA(SDValue Op, uint64_t Val,
18740 unsigned MaxSize) const {
18741 unsigned Size = std::min<unsigned>(a: Op.getScalarValueSizeInBits(), b: MaxSize);
18742 bool HasInv2Pi = Subtarget->hasInv2PiInlineImm();
18743 if (Size == 16) {
18744 MVT VT = Op.getSimpleValueType();
18745 switch (VT.SimpleTy) {
18746 default:
18747 return false;
18748 case MVT::i16:
18749 return AMDGPU::isInlinableLiteralI16(Literal: Val, HasInv2Pi);
18750 case MVT::f16:
18751 return AMDGPU::isInlinableLiteralFP16(Literal: Val, HasInv2Pi);
18752 case MVT::bf16:
18753 return AMDGPU::isInlinableLiteralBF16(Literal: Val, HasInv2Pi);
18754 case MVT::v2i16:
18755 return AMDGPU::getInlineEncodingV2I16(Literal: Val).has_value();
18756 case MVT::v2f16:
18757 return AMDGPU::getInlineEncodingV2F16(Literal: Val).has_value();
18758 case MVT::v2bf16:
18759 return AMDGPU::getInlineEncodingV2BF16(Literal: Val).has_value();
18760 }
18761 }
18762 if ((Size == 32 && AMDGPU::isInlinableLiteral32(Literal: Val, HasInv2Pi)) ||
18763 (Size == 64 && AMDGPU::isInlinableLiteral64(Literal: Val, HasInv2Pi)))
18764 return true;
18765 return false;
18766}
18767
18768static int getAlignedAGPRClassID(unsigned UnalignedClassID) {
18769 switch (UnalignedClassID) {
18770 case AMDGPU::VReg_64RegClassID:
18771 return AMDGPU::VReg_64_Align2RegClassID;
18772 case AMDGPU::VReg_96RegClassID:
18773 return AMDGPU::VReg_96_Align2RegClassID;
18774 case AMDGPU::VReg_128RegClassID:
18775 return AMDGPU::VReg_128_Align2RegClassID;
18776 case AMDGPU::VReg_160RegClassID:
18777 return AMDGPU::VReg_160_Align2RegClassID;
18778 case AMDGPU::VReg_192RegClassID:
18779 return AMDGPU::VReg_192_Align2RegClassID;
18780 case AMDGPU::VReg_224RegClassID:
18781 return AMDGPU::VReg_224_Align2RegClassID;
18782 case AMDGPU::VReg_256RegClassID:
18783 return AMDGPU::VReg_256_Align2RegClassID;
18784 case AMDGPU::VReg_288RegClassID:
18785 return AMDGPU::VReg_288_Align2RegClassID;
18786 case AMDGPU::VReg_320RegClassID:
18787 return AMDGPU::VReg_320_Align2RegClassID;
18788 case AMDGPU::VReg_352RegClassID:
18789 return AMDGPU::VReg_352_Align2RegClassID;
18790 case AMDGPU::VReg_384RegClassID:
18791 return AMDGPU::VReg_384_Align2RegClassID;
18792 case AMDGPU::VReg_512RegClassID:
18793 return AMDGPU::VReg_512_Align2RegClassID;
18794 case AMDGPU::VReg_1024RegClassID:
18795 return AMDGPU::VReg_1024_Align2RegClassID;
18796 case AMDGPU::AReg_64RegClassID:
18797 return AMDGPU::AReg_64_Align2RegClassID;
18798 case AMDGPU::AReg_96RegClassID:
18799 return AMDGPU::AReg_96_Align2RegClassID;
18800 case AMDGPU::AReg_128RegClassID:
18801 return AMDGPU::AReg_128_Align2RegClassID;
18802 case AMDGPU::AReg_160RegClassID:
18803 return AMDGPU::AReg_160_Align2RegClassID;
18804 case AMDGPU::AReg_192RegClassID:
18805 return AMDGPU::AReg_192_Align2RegClassID;
18806 case AMDGPU::AReg_256RegClassID:
18807 return AMDGPU::AReg_256_Align2RegClassID;
18808 case AMDGPU::AReg_512RegClassID:
18809 return AMDGPU::AReg_512_Align2RegClassID;
18810 case AMDGPU::AReg_1024RegClassID:
18811 return AMDGPU::AReg_1024_Align2RegClassID;
18812 default:
18813 return -1;
18814 }
18815}
18816
18817// Figure out which registers should be reserved for stack access. Only after
18818// the function is legalized do we know all of the non-spill stack objects or if
18819// calls are present.
18820void SITargetLowering::finalizeLowering(MachineFunction &MF) const {
18821 MachineRegisterInfo &MRI = MF.getRegInfo();
18822 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
18823 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
18824 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
18825 const SIInstrInfo *TII = ST.getInstrInfo();
18826
18827 if (Info->isEntryFunction()) {
18828 // Callable functions have fixed registers used for stack access.
18829 reservePrivateMemoryRegs(TM: getTargetMachine(), MF, TRI: *TRI, Info&: *Info);
18830 }
18831
18832 // TODO: Move this logic to getReservedRegs()
18833 // Reserve the SGPR(s) to save/restore EXEC for WWM spill/copy handling.
18834 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF);
18835 Register SReg = ST.isWave32()
18836 ? AMDGPU::SGPR_32RegClass.getRegister(i: MaxNumSGPRs - 1)
18837 : TRI->getAlignedHighSGPRForRC(MF, /*Align=*/2,
18838 RC: &AMDGPU::SGPR_64RegClass);
18839 Info->setSGPRForEXECCopy(SReg);
18840
18841 assert(!TRI->isSubRegister(Info->getScratchRSrcReg(),
18842 Info->getStackPtrOffsetReg()));
18843 if (Info->getStackPtrOffsetReg() != AMDGPU::SP_REG)
18844 MRI.replaceRegWith(FromReg: AMDGPU::SP_REG, ToReg: Info->getStackPtrOffsetReg());
18845
18846 // We need to worry about replacing the default register with itself in case
18847 // of MIR testcases missing the MFI.
18848 if (Info->getScratchRSrcReg() != AMDGPU::PRIVATE_RSRC_REG)
18849 MRI.replaceRegWith(FromReg: AMDGPU::PRIVATE_RSRC_REG, ToReg: Info->getScratchRSrcReg());
18850
18851 if (Info->getFrameOffsetReg() != AMDGPU::FP_REG)
18852 MRI.replaceRegWith(FromReg: AMDGPU::FP_REG, ToReg: Info->getFrameOffsetReg());
18853
18854 Info->limitOccupancy(MF);
18855
18856 if (ST.isWave32() && !MF.empty()) {
18857 for (auto &MBB : MF) {
18858 for (auto &MI : MBB) {
18859 TII->fixImplicitOperands(MI);
18860 }
18861 }
18862 }
18863
18864 // FIXME: This is a hack to fixup AGPR classes to use the properly aligned
18865 // classes if required. Ideally the register class constraints would differ
18866 // per-subtarget, but there's no easy way to achieve that right now. This is
18867 // not a problem for VGPRs because the correctly aligned VGPR class is implied
18868 // from using them as the register class for legal types.
18869 if (ST.needsAlignedVGPRs()) {
18870 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
18871 const Register Reg = Register::index2VirtReg(Index: I);
18872 const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg);
18873 if (!RC)
18874 continue;
18875 int NewClassID = getAlignedAGPRClassID(UnalignedClassID: RC->getID());
18876 if (NewClassID != -1)
18877 MRI.setRegClass(Reg, RC: TRI->getRegClass(i: NewClassID));
18878 }
18879 }
18880
18881 TargetLoweringBase::finalizeLowering(MF);
18882}
18883
18884void SITargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
18885 KnownBits &Known,
18886 const APInt &DemandedElts,
18887 const SelectionDAG &DAG,
18888 unsigned Depth) const {
18889 Known.resetAll();
18890 unsigned Opc = Op.getOpcode();
18891 switch (Opc) {
18892 case ISD::INTRINSIC_WO_CHAIN: {
18893 unsigned IID = Op.getConstantOperandVal(i: 0);
18894 switch (IID) {
18895 case Intrinsic::amdgcn_mbcnt_lo:
18896 case Intrinsic::amdgcn_mbcnt_hi: {
18897 const GCNSubtarget &ST =
18898 DAG.getMachineFunction().getSubtarget<GCNSubtarget>();
18899 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
18900 // most 31 + src1.
18901 Known.Zero.setBitsFrom(
18902 IID == Intrinsic::amdgcn_mbcnt_lo ? ST.getWavefrontSizeLog2() : 5);
18903 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
18904 Known = KnownBits::add(LHS: Known, RHS: Known2);
18905 return;
18906 }
18907 }
18908 break;
18909 }
18910 }
18911 return AMDGPUTargetLowering::computeKnownBitsForTargetNode(
18912 Op, Known, DemandedElts, DAG, Depth);
18913}
18914
18915void SITargetLowering::computeKnownBitsForFrameIndex(
18916 const int FI, KnownBits &Known, const MachineFunction &MF) const {
18917 TargetLowering::computeKnownBitsForFrameIndex(FIOp: FI, Known, MF);
18918
18919 // Set the high bits to zero based on the maximum allowed scratch size per
18920 // wave. We can't use vaddr in MUBUF instructions if we don't know the address
18921 // calculation won't overflow, so assume the sign bit is never set.
18922 Known.Zero.setHighBits(getSubtarget()->getKnownHighZeroBitsForFrameIndex());
18923}
18924
18925static void knownBitsForWorkitemID(const GCNSubtarget &ST,
18926 GISelValueTracking &VT, KnownBits &Known,
18927 unsigned Dim) {
18928 unsigned MaxValue =
18929 ST.getMaxWorkitemID(Kernel: VT.getMachineFunction().getFunction(), Dimension: Dim);
18930 Known.Zero.setHighBits(llvm::countl_zero(Val: MaxValue));
18931}
18932
18933static void knownBitsForSBFE(const MachineInstr &MI, GISelValueTracking &VT,
18934 KnownBits &Known, const APInt &DemandedElts,
18935 unsigned BFEWidth, bool SExt, unsigned Depth) {
18936 const MachineRegisterInfo &MRI = VT.getMachineFunction().getRegInfo();
18937 const MachineOperand &Src1 = MI.getOperand(i: 2);
18938
18939 unsigned Src1Cst = 0;
18940 if (Src1.isImm()) {
18941 Src1Cst = Src1.getImm();
18942 } else if (Src1.isReg()) {
18943 auto Cst = getIConstantVRegValWithLookThrough(VReg: Src1.getReg(), MRI);
18944 if (!Cst)
18945 return;
18946 Src1Cst = Cst->Value.getZExtValue();
18947 } else {
18948 return;
18949 }
18950
18951 // Offset is at bits [4:0] for 32 bit, [5:0] for 64 bit.
18952 // Width is always [22:16].
18953 const unsigned Offset =
18954 Src1Cst & maskTrailingOnes<unsigned>(N: (BFEWidth == 32) ? 5 : 6);
18955 const unsigned Width = (Src1Cst >> 16) & maskTrailingOnes<unsigned>(N: 6);
18956
18957 if (Width >= BFEWidth) // Ill-formed.
18958 return;
18959
18960 VT.computeKnownBitsImpl(R: MI.getOperand(i: 1).getReg(), Known, DemandedElts,
18961 Depth: Depth + 1);
18962
18963 Known = Known.extractBits(NumBits: Width, BitPosition: Offset);
18964
18965 if (SExt)
18966 Known = Known.sext(BitWidth: BFEWidth);
18967 else
18968 Known = Known.zext(BitWidth: BFEWidth);
18969}
18970
18971void SITargetLowering::computeKnownBitsForTargetInstr(
18972 GISelValueTracking &VT, Register R, KnownBits &Known,
18973 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
18974 unsigned Depth) const {
18975 Known.resetAll();
18976 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
18977 switch (MI->getOpcode()) {
18978 case AMDGPU::S_BFE_I32:
18979 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 32,
18980 /*SExt=*/true, Depth);
18981 case AMDGPU::S_BFE_U32:
18982 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 32,
18983 /*SExt=*/false, Depth);
18984 case AMDGPU::S_BFE_I64:
18985 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 64,
18986 /*SExt=*/true, Depth);
18987 case AMDGPU::S_BFE_U64:
18988 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 64,
18989 /*SExt=*/false, Depth);
18990 case AMDGPU::G_INTRINSIC:
18991 case AMDGPU::G_INTRINSIC_CONVERGENT: {
18992 Intrinsic::ID IID = cast<GIntrinsic>(Val: MI)->getIntrinsicID();
18993 switch (IID) {
18994 case Intrinsic::amdgcn_workitem_id_x:
18995 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 0);
18996 break;
18997 case Intrinsic::amdgcn_workitem_id_y:
18998 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 1);
18999 break;
19000 case Intrinsic::amdgcn_workitem_id_z:
19001 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 2);
19002 break;
19003 case Intrinsic::amdgcn_mbcnt_lo:
19004 case Intrinsic::amdgcn_mbcnt_hi: {
19005 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
19006 // most 31 + src1.
19007 Known.Zero.setBitsFrom(IID == Intrinsic::amdgcn_mbcnt_lo
19008 ? getSubtarget()->getWavefrontSizeLog2()
19009 : 5);
19010 KnownBits Known2;
19011 VT.computeKnownBitsImpl(R: MI->getOperand(i: 3).getReg(), Known&: Known2, DemandedElts,
19012 Depth: Depth + 1);
19013 Known = KnownBits::add(LHS: Known, RHS: Known2);
19014 break;
19015 }
19016 case Intrinsic::amdgcn_groupstaticsize: {
19017 // We can report everything over the maximum size as 0. We can't report
19018 // based on the actual size because we don't know if it's accurate or not
19019 // at any given point.
19020 Known.Zero.setHighBits(
19021 llvm::countl_zero(Val: getSubtarget()->getAddressableLocalMemorySize()));
19022 break;
19023 }
19024 }
19025 break;
19026 }
19027 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
19028 Known.Zero.setHighBits(24);
19029 break;
19030 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
19031 Known.Zero.setHighBits(16);
19032 break;
19033 case AMDGPU::G_AMDGPU_COPY_SCC_VCC:
19034 // G_AMDGPU_COPY_SCC_VCC converts a uniform boolean in VCC to SGPR s32,
19035 // producing exactly 0 or 1.
19036 Known.Zero.setHighBits(Known.getBitWidth() - 1);
19037 break;
19038 case AMDGPU::G_AMDGPU_SMED3:
19039 case AMDGPU::G_AMDGPU_UMED3: {
19040 auto [Dst, Src0, Src1, Src2] = MI->getFirst4Regs();
19041
19042 KnownBits Known2;
19043 VT.computeKnownBitsImpl(R: Src2, Known&: Known2, DemandedElts, Depth: Depth + 1);
19044 if (Known2.isUnknown())
19045 break;
19046
19047 KnownBits Known1;
19048 VT.computeKnownBitsImpl(R: Src1, Known&: Known1, DemandedElts, Depth: Depth + 1);
19049 if (Known1.isUnknown())
19050 break;
19051
19052 KnownBits Known0;
19053 VT.computeKnownBitsImpl(R: Src0, Known&: Known0, DemandedElts, Depth: Depth + 1);
19054 if (Known0.isUnknown())
19055 break;
19056
19057 // TODO: Handle LeadZero/LeadOne from UMIN/UMAX handling.
19058 Known.Zero = Known0.Zero & Known1.Zero & Known2.Zero;
19059 Known.One = Known0.One & Known1.One & Known2.One;
19060 break;
19061 }
19062 }
19063}
19064
19065Align SITargetLowering::computeKnownAlignForTargetInstr(
19066 GISelValueTracking &VT, Register R, const MachineRegisterInfo &MRI,
19067 unsigned Depth) const {
19068 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
19069 if (auto *GI = dyn_cast<GIntrinsic>(Val: MI)) {
19070 // FIXME: Can this move to generic code? What about the case where the call
19071 // site specifies a lower alignment?
19072 Intrinsic::ID IID = GI->getIntrinsicID();
19073 LLVMContext &Ctx = VT.getMachineFunction().getFunction().getContext();
19074 AttributeList Attrs =
19075 Intrinsic::getAttributes(C&: Ctx, id: IID, FT: Intrinsic::getType(Context&: Ctx, id: IID));
19076 if (MaybeAlign RetAlign = Attrs.getRetAlignment())
19077 return *RetAlign;
19078 }
19079 return Align(1);
19080}
19081
19082Align SITargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
19083 const Align PrefAlign = TargetLowering::getPrefLoopAlignment(ML);
19084 const Align CacheLineAlign = Align(64);
19085
19086 // GFX950: Prevent an 8-byte instruction at loop header from being split by
19087 // the 32-byte instruction fetch window boundary. This avoids a significant
19088 // fetch delay after backward branch. We use 32-byte alignment with max
19089 // padding of 4 bytes (one s_nop), see getMaxPermittedBytesForAlignment().
19090 if (ML && !DisableLoopAlignment &&
19091 getSubtarget()->hasLoopHeadInstSplitSensitivity()) {
19092 const MachineBasicBlock *Header = ML->getHeader();
19093 // Respect user-specified or previously set alignment.
19094 if (Header->getAlignment() != PrefAlign)
19095 return Header->getAlignment();
19096 if (needsFetchWindowAlignment(MBB: *Header))
19097 return Align(32);
19098 }
19099
19100 // Pre-GFX10 target did not benefit from loop alignment
19101 if (!ML || DisableLoopAlignment || !getSubtarget()->hasInstPrefetch() ||
19102 getSubtarget()->hasInstFwdPrefetchBug())
19103 return PrefAlign;
19104
19105 // On GFX10 I$ is 4 x 64 bytes cache lines.
19106 // By default prefetcher keeps one cache line behind and reads two ahead.
19107 // We can modify it with S_INST_PREFETCH for larger loops to have two lines
19108 // behind and one ahead.
19109 // Therefor we can benefit from aligning loop headers if loop fits 192 bytes.
19110 // If loop fits 64 bytes it always spans no more than two cache lines and
19111 // does not need an alignment.
19112 // Else if loop is less or equal 128 bytes we do not need to modify prefetch,
19113 // Else if loop is less or equal 192 bytes we need two lines behind.
19114
19115 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19116 const MachineBasicBlock *Header = ML->getHeader();
19117 if (Header->getAlignment() != PrefAlign)
19118 return Header->getAlignment(); // Already processed.
19119
19120 unsigned LoopSize = 0;
19121 for (const MachineBasicBlock *MBB : ML->blocks()) {
19122 // If inner loop block is aligned assume in average half of the alignment
19123 // size to be added as nops.
19124 if (MBB != Header)
19125 LoopSize += MBB->getAlignment().value() / 2;
19126
19127 for (const MachineInstr &MI : *MBB) {
19128 LoopSize += TII->getInstSizeInBytes(MI);
19129 if (LoopSize > 192)
19130 return PrefAlign;
19131 }
19132 }
19133
19134 if (LoopSize <= 64)
19135 return PrefAlign;
19136
19137 if (LoopSize <= 128)
19138 return CacheLineAlign;
19139
19140 // If any of parent loops is surrounded by prefetch instructions do not
19141 // insert new for inner loop, which would reset parent's settings.
19142 for (MachineLoop *P = ML->getParentLoop(); P; P = P->getParentLoop()) {
19143 if (MachineBasicBlock *Exit = P->getExitBlock()) {
19144 auto I = Exit->getFirstNonDebugInstr();
19145 if (I != Exit->end() && I->getOpcode() == AMDGPU::S_INST_PREFETCH)
19146 return CacheLineAlign;
19147 }
19148 }
19149
19150 MachineBasicBlock *Pre = ML->getLoopPreheader();
19151 MachineBasicBlock *Exit = ML->getExitBlock();
19152
19153 if (Pre && Exit) {
19154 auto PreTerm = Pre->getFirstTerminator();
19155 if (PreTerm == Pre->begin() ||
19156 std::prev(x: PreTerm)->getOpcode() != AMDGPU::S_INST_PREFETCH)
19157 BuildMI(BB&: *Pre, I: PreTerm, MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_INST_PREFETCH))
19158 .addImm(Val: 1); // prefetch 2 lines behind PC
19159
19160 auto ExitHead = Exit->getFirstNonDebugInstr();
19161 if (ExitHead == Exit->end() ||
19162 ExitHead->getOpcode() != AMDGPU::S_INST_PREFETCH)
19163 BuildMI(BB&: *Exit, I: ExitHead, MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_INST_PREFETCH))
19164 .addImm(Val: 2); // prefetch 1 line behind PC
19165 }
19166
19167 return CacheLineAlign;
19168}
19169
19170unsigned SITargetLowering::getMaxPermittedBytesForAlignment(
19171 MachineBasicBlock *MBB) const {
19172 // GFX950: Limit padding to 4 bytes (one s_nop) for blocks where an 8-byte
19173 // instruction could be split by the 32-byte fetch window boundary.
19174 // See getPrefLoopAlignment() for context.
19175 if (needsFetchWindowAlignment(MBB: *MBB))
19176 return 4;
19177 return TargetLowering::getMaxPermittedBytesForAlignment(MBB);
19178}
19179
19180bool SITargetLowering::needsFetchWindowAlignment(
19181 const MachineBasicBlock &MBB) const {
19182 if (!getSubtarget()->hasLoopHeadInstSplitSensitivity())
19183 return false;
19184 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19185 for (const MachineInstr &MI : MBB) {
19186 if (MI.isMetaInstruction())
19187 continue;
19188 // Instructions larger than 4 bytes can be split by a 32-byte boundary.
19189 return TII->getInstSizeInBytes(MI) > 4;
19190 }
19191 return false;
19192}
19193
19194[[maybe_unused]]
19195static bool isCopyFromRegOfInlineAsm(const SDNode *N) {
19196 assert(N->getOpcode() == ISD::CopyFromReg);
19197 do {
19198 // Follow the chain until we find an INLINEASM node.
19199 N = N->getOperand(Num: 0).getNode();
19200 if (N->getOpcode() == ISD::INLINEASM || N->getOpcode() == ISD::INLINEASM_BR)
19201 return true;
19202 } while (N->getOpcode() == ISD::CopyFromReg);
19203 return false;
19204}
19205
19206bool SITargetLowering::isSDNodeSourceOfDivergence(const SDNode *N,
19207 FunctionLoweringInfo *FLI,
19208 UniformityInfo *UA) const {
19209 switch (N->getOpcode()) {
19210 case ISD::CopyFromReg: {
19211 const RegisterSDNode *R = cast<RegisterSDNode>(Val: N->getOperand(Num: 1));
19212 const MachineRegisterInfo &MRI = FLI->MF->getRegInfo();
19213 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
19214 Register Reg = R->getReg();
19215
19216 // FIXME: Why does this need to consider isLiveIn?
19217 if (Reg.isPhysical() || MRI.isLiveIn(Reg))
19218 return !TRI->isSGPRReg(MRI, Reg);
19219
19220 if (const Value *V = FLI->getValueFromVirtualReg(Vreg: R->getReg()))
19221 return UA->isDivergent(V);
19222
19223 assert(Reg == FLI->DemoteRegister || isCopyFromRegOfInlineAsm(N));
19224 return !TRI->isSGPRReg(MRI, Reg);
19225 }
19226 case ISD::LOAD: {
19227 const LoadSDNode *L = cast<LoadSDNode>(Val: N);
19228 unsigned AS = L->getAddressSpace();
19229 // A flat load may access private memory.
19230 return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS;
19231 }
19232 case ISD::CALLSEQ_END:
19233 return true;
19234 case ISD::INTRINSIC_WO_CHAIN:
19235 return AMDGPU::isIntrinsicSourceOfDivergence(IntrID: N->getConstantOperandVal(Num: 0));
19236 case ISD::INTRINSIC_W_CHAIN:
19237 return AMDGPU::isIntrinsicSourceOfDivergence(IntrID: N->getConstantOperandVal(Num: 1));
19238 case AMDGPUISD::ATOMIC_CMP_SWAP:
19239 case AMDGPUISD::BUFFER_ATOMIC_SWAP:
19240 case AMDGPUISD::BUFFER_ATOMIC_ADD:
19241 case AMDGPUISD::BUFFER_ATOMIC_SUB:
19242 case AMDGPUISD::BUFFER_ATOMIC_SMIN:
19243 case AMDGPUISD::BUFFER_ATOMIC_UMIN:
19244 case AMDGPUISD::BUFFER_ATOMIC_SMAX:
19245 case AMDGPUISD::BUFFER_ATOMIC_UMAX:
19246 case AMDGPUISD::BUFFER_ATOMIC_AND:
19247 case AMDGPUISD::BUFFER_ATOMIC_OR:
19248 case AMDGPUISD::BUFFER_ATOMIC_XOR:
19249 case AMDGPUISD::BUFFER_ATOMIC_INC:
19250 case AMDGPUISD::BUFFER_ATOMIC_DEC:
19251 case AMDGPUISD::BUFFER_ATOMIC_CMPSWAP:
19252 case AMDGPUISD::BUFFER_ATOMIC_FADD:
19253 case AMDGPUISD::BUFFER_ATOMIC_FMIN:
19254 case AMDGPUISD::BUFFER_ATOMIC_FMAX:
19255 // Target-specific read-modify-write atomics are sources of divergence.
19256 return true;
19257 default:
19258 if (auto *A = dyn_cast<AtomicSDNode>(Val: N)) {
19259 // Generic read-modify-write atomics are sources of divergence.
19260 return A->readMem() && A->writeMem();
19261 }
19262 return false;
19263 }
19264}
19265
19266bool SITargetLowering::denormalsEnabledForType(const SelectionDAG &DAG,
19267 EVT VT) const {
19268 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
19269 case MVT::f32:
19270 return !denormalModeIsFlushAllF32(MF: DAG.getMachineFunction());
19271 case MVT::f64:
19272 case MVT::f16:
19273 return !denormalModeIsFlushAllF64F16(MF: DAG.getMachineFunction());
19274 default:
19275 return false;
19276 }
19277}
19278
19279bool SITargetLowering::denormalsEnabledForType(
19280 LLT Ty, const MachineFunction &MF) const {
19281 switch (Ty.getScalarSizeInBits()) {
19282 case 32:
19283 return !denormalModeIsFlushAllF32(MF);
19284 case 64:
19285 case 16:
19286 return !denormalModeIsFlushAllF64F16(MF);
19287 default:
19288 return false;
19289 }
19290}
19291
19292bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
19293 const APInt &DemandedElts,
19294 const SelectionDAG &DAG,
19295 bool SNaN,
19296 unsigned Depth) const {
19297 if (Op.getOpcode() == AMDGPUISD::CLAMP) {
19298 const MachineFunction &MF = DAG.getMachineFunction();
19299 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
19300
19301 if (Info->getMode().DX10Clamp)
19302 return true; // Clamped to 0.
19303 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
19304 }
19305
19306 return AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(Op, DemandedElts,
19307 DAG, SNaN, Depth);
19308}
19309
19310// On older subtargets, global FP atomic instructions have a hardcoded FP mode
19311// and do not support FP32 denormals, and only support v2f16/f64 denormals.
19312static bool atomicIgnoresDenormalModeOrFPModeIsFTZ(const AtomicRMWInst *RMW) {
19313 if (RMW->hasMetadata(Kind: "amdgpu.ignore.denormal.mode"))
19314 return true;
19315
19316 const fltSemantics &Flt = RMW->getType()->getScalarType()->getFltSemantics();
19317 auto DenormMode = RMW->getFunction()->getDenormalMode(FPType: Flt);
19318 if (DenormMode == DenormalMode::getPreserveSign())
19319 return true;
19320
19321 // TODO: Remove this.
19322 return RMW->getFunction()
19323 ->getFnAttribute(Kind: "amdgpu-unsafe-fp-atomics")
19324 .getValueAsBool();
19325}
19326
19327static OptimizationRemark emitAtomicRMWLegalRemark(const AtomicRMWInst *RMW) {
19328 LLVMContext &Ctx = RMW->getContext();
19329 StringRef MemScope =
19330 Ctx.getSyncScopeName(Id: RMW->getSyncScopeID()).value_or(u: "system");
19331
19332 return OptimizationRemark(DEBUG_TYPE, "Passed", RMW)
19333 << "Hardware instruction generated for atomic "
19334 << RMW->getOperationName(Op: RMW->getOperation())
19335 << " operation at memory scope " << MemScope;
19336}
19337
19338static bool isV2F16OrV2BF16(Type *Ty) {
19339 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
19340 Type *EltTy = VT->getElementType();
19341 return VT->getNumElements() == 2 &&
19342 (EltTy->isHalfTy() || EltTy->isBFloatTy());
19343 }
19344
19345 return false;
19346}
19347
19348static bool isV2F16(Type *Ty) {
19349 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
19350 return VT && VT->getNumElements() == 2 && VT->getElementType()->isHalfTy();
19351}
19352
19353static bool isV2BF16(Type *Ty) {
19354 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
19355 return VT && VT->getNumElements() == 2 && VT->getElementType()->isBFloatTy();
19356}
19357
19358/// \return true if atomicrmw integer ops work for the type.
19359static bool isAtomicRMWLegalIntTy(Type *Ty) {
19360 if (auto *IT = dyn_cast<IntegerType>(Val: Ty)) {
19361 unsigned BW = IT->getBitWidth();
19362 return BW == 32 || BW == 64;
19363 }
19364
19365 return false;
19366}
19367
19368/// \return true if this atomicrmw xchg type can be selected.
19369static bool isAtomicRMWLegalXChgTy(const AtomicRMWInst *RMW) {
19370 Type *Ty = RMW->getType();
19371 if (isAtomicRMWLegalIntTy(Ty))
19372 return true;
19373
19374 if (PointerType *PT = dyn_cast<PointerType>(Val: Ty)) {
19375 const DataLayout &DL = RMW->getFunction()->getParent()->getDataLayout();
19376 unsigned BW = DL.getPointerSizeInBits(AS: PT->getAddressSpace());
19377 return BW == 32 || BW == 64;
19378 }
19379
19380 if (Ty->isFloatTy() || Ty->isDoubleTy())
19381 return true;
19382
19383 if (FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
19384 return VT->getNumElements() == 2 &&
19385 VT->getElementType()->getPrimitiveSizeInBits() == 16;
19386 }
19387
19388 return false;
19389}
19390
19391/// \returns true if it's valid to emit a native instruction for \p RMW, based
19392/// on the properties of the target memory.
19393static bool globalMemoryFPAtomicIsLegal(const GCNSubtarget &Subtarget,
19394 const AtomicRMWInst *RMW,
19395 bool HasSystemScope) {
19396 // The remote/fine-grained access logic is different from the integer
19397 // atomics. Without AgentScopeFineGrainedRemoteMemoryAtomics support,
19398 // fine-grained access does not work, even for a device local allocation.
19399 //
19400 // With AgentScopeFineGrainedRemoteMemoryAtomics, system scoped device local
19401 // allocations work.
19402 if (HasSystemScope) {
19403 if (Subtarget.hasAgentScopeFineGrainedRemoteMemoryAtomics() &&
19404 RMW->hasMetadata(Kind: "amdgpu.no.remote.memory"))
19405 return true;
19406 if (Subtarget.hasEmulatedSystemScopeAtomics())
19407 return true;
19408 } else if (Subtarget.hasAgentScopeFineGrainedRemoteMemoryAtomics())
19409 return true;
19410
19411 return RMW->hasMetadata(Kind: "amdgpu.no.fine.grained.memory");
19412}
19413
19414/// \return Action to perform on AtomicRMWInsts for integer operations.
19415static TargetLowering::AtomicExpansionKind
19416atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW) {
19417 return isAtomicRMWLegalIntTy(Ty: RMW->getType())
19418 ? TargetLowering::AtomicExpansionKind::None
19419 : TargetLowering::AtomicExpansionKind::CmpXChg;
19420}
19421
19422/// Return if a flat address space atomicrmw can access private memory.
19423static bool flatInstrMayAccessPrivate(const Instruction *I) {
19424 const MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_noalias_addrspace);
19425 return !MD ||
19426 !AMDGPU::hasValueInRangeLikeMetadata(MD: *MD, Val: AMDGPUAS::PRIVATE_ADDRESS);
19427}
19428
19429static TargetLowering::AtomicExpansionKind
19430getPrivateAtomicExpansionKind(const GCNSubtarget &STI) {
19431 // For GAS, lower to flat atomic.
19432 return STI.hasGloballyAddressableScratch()
19433 ? TargetLowering::AtomicExpansionKind::CustomExpand
19434 : TargetLowering::AtomicExpansionKind::NotAtomic;
19435}
19436
19437TargetLowering::AtomicExpansionKind
19438SITargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
19439 unsigned AS = RMW->getPointerAddressSpace();
19440 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
19441 return getPrivateAtomicExpansionKind(STI: *getSubtarget());
19442
19443 // 64-bit flat atomics that dynamically reside in private memory will silently
19444 // be dropped.
19445 //
19446 // Note that we will emit a new copy of the original atomic in the expansion,
19447 // which will be incrementally relegalized.
19448 const DataLayout &DL = RMW->getFunction()->getDataLayout();
19449 if (AS == AMDGPUAS::FLAT_ADDRESS &&
19450 DL.getTypeSizeInBits(Ty: RMW->getType()) == 64 &&
19451 flatInstrMayAccessPrivate(I: RMW))
19452 return AtomicExpansionKind::CustomExpand;
19453
19454 auto ReportUnsafeHWInst = [=](TargetLowering::AtomicExpansionKind Kind) {
19455 OptimizationRemarkEmitter ORE(RMW->getFunction());
19456 ORE.emit(RemarkBuilder: [=]() {
19457 return emitAtomicRMWLegalRemark(RMW) << " due to an unsafe request.";
19458 });
19459 return Kind;
19460 };
19461
19462 auto SSID = RMW->getSyncScopeID();
19463 bool HasSystemScope =
19464 SSID == SyncScope::System ||
19465 SSID == RMW->getContext().getOrInsertSyncScopeID(SSN: "one-as");
19466
19467 auto Op = RMW->getOperation();
19468 switch (Op) {
19469 case AtomicRMWInst::Xchg:
19470 // PCIe supports add and xchg for system atomics.
19471 return isAtomicRMWLegalXChgTy(RMW)
19472 ? TargetLowering::AtomicExpansionKind::None
19473 : TargetLowering::AtomicExpansionKind::CmpXChg;
19474 case AtomicRMWInst::Add:
19475 // PCIe supports add and xchg for system atomics.
19476 return atomicSupportedIfLegalIntType(RMW);
19477 case AtomicRMWInst::Sub:
19478 case AtomicRMWInst::And:
19479 case AtomicRMWInst::Or:
19480 case AtomicRMWInst::Xor:
19481 case AtomicRMWInst::Max:
19482 case AtomicRMWInst::Min:
19483 case AtomicRMWInst::UMax:
19484 case AtomicRMWInst::UMin:
19485 case AtomicRMWInst::UIncWrap:
19486 case AtomicRMWInst::UDecWrap:
19487 case AtomicRMWInst::USubCond:
19488 case AtomicRMWInst::USubSat: {
19489 if (Op == AtomicRMWInst::USubCond && !Subtarget->hasCondSubInsts())
19490 return AtomicExpansionKind::CmpXChg;
19491 if (Op == AtomicRMWInst::USubSat && !Subtarget->hasSubClampInsts())
19492 return AtomicExpansionKind::CmpXChg;
19493 if (Op == AtomicRMWInst::USubCond || Op == AtomicRMWInst::USubSat) {
19494 auto *IT = dyn_cast<IntegerType>(Val: RMW->getType());
19495 if (!IT || IT->getBitWidth() != 32)
19496 return AtomicExpansionKind::CmpXChg;
19497 }
19498
19499 if (AMDGPU::isFlatGlobalAddrSpace(AS) ||
19500 AS == AMDGPUAS::BUFFER_FAT_POINTER) {
19501 if (Subtarget->hasEmulatedSystemScopeAtomics())
19502 return atomicSupportedIfLegalIntType(RMW);
19503
19504 // On most subtargets, for atomicrmw operations other than add/xchg,
19505 // whether or not the instructions will behave correctly depends on where
19506 // the address physically resides and what interconnect is used in the
19507 // system configuration. On some some targets the instruction will nop,
19508 // and in others synchronization will only occur at degraded device scope.
19509 //
19510 // If the allocation is known local to the device, the instructions should
19511 // work correctly.
19512 if (RMW->hasMetadata(Kind: "amdgpu.no.remote.memory"))
19513 return atomicSupportedIfLegalIntType(RMW);
19514
19515 // If fine-grained remote memory works at device scope, we don't need to
19516 // do anything.
19517 if (!HasSystemScope &&
19518 Subtarget->hasAgentScopeFineGrainedRemoteMemoryAtomics())
19519 return atomicSupportedIfLegalIntType(RMW);
19520
19521 // If we are targeting a remote allocated address, it depends what kind of
19522 // allocation the address belongs to.
19523 //
19524 // If the allocation is fine-grained (in host memory, or in PCIe peer
19525 // device memory), the operation will fail depending on the target.
19526 //
19527 // Note fine-grained host memory access does work on APUs or if XGMI is
19528 // used, but we do not know if we are targeting an APU or the system
19529 // configuration from the ISA version/target-cpu.
19530 if (RMW->hasMetadata(Kind: "amdgpu.no.fine.grained.memory"))
19531 return atomicSupportedIfLegalIntType(RMW);
19532
19533 if (Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Or ||
19534 Op == AtomicRMWInst::Xor) {
19535 // Atomic sub/or/xor do not work over PCI express, but atomic add
19536 // does. InstCombine transforms these with 0 to or, so undo that.
19537 if (const Constant *ConstVal = dyn_cast<Constant>(Val: RMW->getValOperand());
19538 ConstVal && ConstVal->isNullValue())
19539 return AtomicExpansionKind::CustomExpand;
19540 }
19541
19542 // If the allocation could be in remote, fine-grained memory, the rmw
19543 // instructions may fail. cmpxchg should work, so emit that. On some
19544 // system configurations, PCIe atomics aren't supported so cmpxchg won't
19545 // even work, so you're out of luck anyway.
19546
19547 // In summary:
19548 //
19549 // Cases that may fail:
19550 // - fine-grained pinned host memory
19551 // - fine-grained migratable host memory
19552 // - fine-grained PCIe peer device
19553 //
19554 // Cases that should work, but may be treated overly conservatively.
19555 // - fine-grained host memory on an APU
19556 // - fine-grained XGMI peer device
19557 return AtomicExpansionKind::CmpXChg;
19558 }
19559
19560 return atomicSupportedIfLegalIntType(RMW);
19561 }
19562 case AtomicRMWInst::FAdd: {
19563 Type *Ty = RMW->getType();
19564
19565 // TODO: Handle REGION_ADDRESS
19566 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
19567 // DS F32 FP atomics do respect the denormal mode, but the rounding mode
19568 // is fixed to round-to-nearest-even.
19569 //
19570 // F64 / PK_F16 / PK_BF16 never flush and are also fixed to
19571 // round-to-nearest-even.
19572 //
19573 // We ignore the rounding mode problem, even in strictfp. The C++ standard
19574 // suggests it is OK if the floating-point mode may not match the calling
19575 // thread.
19576 if (Ty->isFloatTy()) {
19577 return Subtarget->hasLDSFPAtomicAddF32() ? AtomicExpansionKind::None
19578 : AtomicExpansionKind::CmpXChg;
19579 }
19580
19581 if (Ty->isDoubleTy()) {
19582 // Ignores denormal mode, but we don't consider flushing mandatory.
19583 return Subtarget->hasLDSFPAtomicAddF64() ? AtomicExpansionKind::None
19584 : AtomicExpansionKind::CmpXChg;
19585 }
19586
19587 if (Subtarget->hasAtomicDsPkAdd16Insts() && isV2F16OrV2BF16(Ty))
19588 return AtomicExpansionKind::None;
19589
19590 return AtomicExpansionKind::CmpXChg;
19591 }
19592
19593 // LDS atomics respect the denormal mode from the mode register.
19594 //
19595 // Traditionally f32 global/buffer memory atomics would unconditionally
19596 // flush denormals, but newer targets do not flush. f64/f16/bf16 cases never
19597 // flush.
19598 //
19599 // On targets with flat atomic fadd, denormals would flush depending on
19600 // whether the target address resides in LDS or global memory. We consider
19601 // this flat-maybe-flush as will-flush.
19602 if (Ty->isFloatTy() &&
19603 !Subtarget->hasMemoryAtomicFaddF32DenormalSupport() &&
19604 !atomicIgnoresDenormalModeOrFPModeIsFTZ(RMW))
19605 return AtomicExpansionKind::CmpXChg;
19606
19607 // FIXME: These ReportUnsafeHWInsts are imprecise. Some of these cases are
19608 // safe. The message phrasing also should be better.
19609 if (globalMemoryFPAtomicIsLegal(Subtarget: *Subtarget, RMW, HasSystemScope)) {
19610 if (AS == AMDGPUAS::FLAT_ADDRESS) {
19611 // gfx942, gfx12
19612 if (Subtarget->hasAtomicFlatPkAdd16Insts() && isV2F16OrV2BF16(Ty))
19613 return ReportUnsafeHWInst(AtomicExpansionKind::None);
19614 } else if (AMDGPU::isExtendedGlobalAddrSpace(AS)) {
19615 // gfx90a, gfx942, gfx12
19616 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
19617 return ReportUnsafeHWInst(AtomicExpansionKind::None);
19618
19619 // gfx942, gfx12
19620 if (Subtarget->hasAtomicGlobalPkAddBF16Inst() && isV2BF16(Ty))
19621 return ReportUnsafeHWInst(AtomicExpansionKind::None);
19622 } else if (AS == AMDGPUAS::BUFFER_FAT_POINTER) {
19623 // gfx90a, gfx942, gfx12
19624 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
19625 return ReportUnsafeHWInst(AtomicExpansionKind::None);
19626
19627 // While gfx90a/gfx942 supports v2bf16 for global/flat, it does not for
19628 // buffer. gfx12 does have the buffer version.
19629 if (Subtarget->hasAtomicBufferPkAddBF16Inst() && isV2BF16(Ty))
19630 return ReportUnsafeHWInst(AtomicExpansionKind::None);
19631 }
19632
19633 // global and flat atomic fadd f64: gfx90a, gfx942.
19634 if (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() && Ty->isDoubleTy())
19635 return ReportUnsafeHWInst(AtomicExpansionKind::None);
19636
19637 if (AS != AMDGPUAS::FLAT_ADDRESS) {
19638 if (Ty->isFloatTy()) {
19639 // global/buffer atomic fadd f32 no-rtn: gfx908, gfx90a, gfx942,
19640 // gfx11+.
19641 if (RMW->use_empty() && Subtarget->hasAtomicFaddNoRtnInsts())
19642 return ReportUnsafeHWInst(AtomicExpansionKind::None);
19643 // global/buffer atomic fadd f32 rtn: gfx90a, gfx942, gfx11+.
19644 if (!RMW->use_empty() && Subtarget->hasAtomicFaddRtnInsts())
19645 return ReportUnsafeHWInst(AtomicExpansionKind::None);
19646 } else {
19647 // gfx908
19648 if (RMW->use_empty() &&
19649 Subtarget->hasAtomicBufferGlobalPkAddF16NoRtnInsts() &&
19650 isV2F16(Ty))
19651 return ReportUnsafeHWInst(AtomicExpansionKind::None);
19652 }
19653 }
19654
19655 // flat atomic fadd f32: gfx942, gfx11+.
19656 if (AS == AMDGPUAS::FLAT_ADDRESS && Ty->isFloatTy()) {
19657 if (Subtarget->hasFlatAtomicFaddF32Inst())
19658 return ReportUnsafeHWInst(AtomicExpansionKind::None);
19659
19660 // If it is in flat address space, and the type is float, we will try to
19661 // expand it, if the target supports global and lds atomic fadd. The
19662 // reason we need that is, in the expansion, we emit the check of
19663 // address space. If it is in global address space, we emit the global
19664 // atomic fadd; if it is in shared address space, we emit the LDS atomic
19665 // fadd.
19666 if (Subtarget->hasLDSFPAtomicAddF32()) {
19667 if (RMW->use_empty() && Subtarget->hasAtomicFaddNoRtnInsts())
19668 return AtomicExpansionKind::CustomExpand;
19669 if (!RMW->use_empty() && Subtarget->hasAtomicFaddRtnInsts())
19670 return AtomicExpansionKind::CustomExpand;
19671 }
19672 }
19673 }
19674
19675 return AtomicExpansionKind::CmpXChg;
19676 }
19677 case AtomicRMWInst::FMin:
19678 case AtomicRMWInst::FMax: {
19679 Type *Ty = RMW->getType();
19680
19681 // LDS float and double fmin/fmax were always supported.
19682 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
19683 return Ty->isFloatTy() || Ty->isDoubleTy() ? AtomicExpansionKind::None
19684 : AtomicExpansionKind::CmpXChg;
19685 }
19686
19687 if (globalMemoryFPAtomicIsLegal(Subtarget: *Subtarget, RMW, HasSystemScope)) {
19688 // For flat and global cases:
19689 // float, double in gfx7. Manual claims denormal support.
19690 // Removed in gfx8.
19691 // float, double restored in gfx10.
19692 // double removed again in gfx11, so only f32 for gfx11/gfx12.
19693 //
19694 // For gfx9, gfx90a and gfx942 support f64 for global (same as fadd), but
19695 // no f32.
19696 if (AS == AMDGPUAS::FLAT_ADDRESS) {
19697 if (Subtarget->hasAtomicFMinFMaxF32FlatInsts() && Ty->isFloatTy())
19698 return ReportUnsafeHWInst(AtomicExpansionKind::None);
19699 if (Subtarget->hasAtomicFMinFMaxF64FlatInsts() && Ty->isDoubleTy())
19700 return ReportUnsafeHWInst(AtomicExpansionKind::None);
19701 } else if (AMDGPU::isExtendedGlobalAddrSpace(AS) ||
19702 AS == AMDGPUAS::BUFFER_FAT_POINTER) {
19703 if (Subtarget->hasAtomicFMinFMaxF32GlobalInsts() && Ty->isFloatTy())
19704 return ReportUnsafeHWInst(AtomicExpansionKind::None);
19705 if (Subtarget->hasAtomicFMinFMaxF64GlobalInsts() && Ty->isDoubleTy())
19706 return ReportUnsafeHWInst(AtomicExpansionKind::None);
19707 }
19708 }
19709
19710 return AtomicExpansionKind::CmpXChg;
19711 }
19712 case AtomicRMWInst::Nand:
19713 case AtomicRMWInst::FSub:
19714 default:
19715 return AtomicExpansionKind::CmpXChg;
19716 }
19717
19718 llvm_unreachable("covered atomicrmw op switch");
19719}
19720
19721TargetLowering::AtomicExpansionKind
19722SITargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
19723 return LI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS
19724 ? getPrivateAtomicExpansionKind(STI: *getSubtarget())
19725 : AtomicExpansionKind::None;
19726}
19727
19728TargetLowering::AtomicExpansionKind
19729SITargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
19730 return SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS
19731 ? getPrivateAtomicExpansionKind(STI: *getSubtarget())
19732 : AtomicExpansionKind::None;
19733}
19734
19735TargetLowering::AtomicExpansionKind
19736SITargetLowering::shouldExpandAtomicCmpXchgInIR(
19737 const AtomicCmpXchgInst *CmpX) const {
19738 unsigned AddrSpace = CmpX->getPointerAddressSpace();
19739 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)
19740 return getPrivateAtomicExpansionKind(STI: *getSubtarget());
19741
19742 if (AddrSpace != AMDGPUAS::FLAT_ADDRESS || !flatInstrMayAccessPrivate(I: CmpX))
19743 return AtomicExpansionKind::None;
19744
19745 const DataLayout &DL = CmpX->getDataLayout();
19746
19747 Type *ValTy = CmpX->getNewValOperand()->getType();
19748
19749 // If a 64-bit flat atomic may alias private, we need to avoid using the
19750 // atomic in the private case.
19751 return DL.getTypeSizeInBits(Ty: ValTy) == 64 ? AtomicExpansionKind::CustomExpand
19752 : AtomicExpansionKind::None;
19753}
19754
19755const TargetRegisterClass *
19756SITargetLowering::getRegClassFor(MVT VT, bool isDivergent) const {
19757 const TargetRegisterClass *RC = TargetLoweringBase::getRegClassFor(VT, isDivergent: false);
19758 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
19759 if (RC == &AMDGPU::VReg_1RegClass && !isDivergent)
19760 return Subtarget->isWave64() ? &AMDGPU::SReg_64RegClass
19761 : &AMDGPU::SReg_32RegClass;
19762 if (!TRI->isSGPRClass(RC) && !isDivergent)
19763 return TRI->getEquivalentSGPRClass(VRC: RC);
19764 if (TRI->isSGPRClass(RC) && isDivergent) {
19765 if (Subtarget->hasGFX90AInsts())
19766 return TRI->getEquivalentAVClass(SRC: RC);
19767 return TRI->getEquivalentVGPRClass(SRC: RC);
19768 }
19769
19770 return RC;
19771}
19772
19773// FIXME: This is a workaround for DivergenceAnalysis not understanding always
19774// uniform values (as produced by the mask results of control flow intrinsics)
19775// used outside of divergent blocks. The phi users need to also be treated as
19776// always uniform.
19777//
19778// FIXME: DA is no longer in-use. Does this still apply to UniformityAnalysis?
19779static bool hasCFUser(const Value *V, SmallPtrSet<const Value *, 16> &Visited,
19780 unsigned WaveSize) {
19781 // FIXME: We assume we never cast the mask results of a control flow
19782 // intrinsic.
19783 // Early exit if the type won't be consistent as a compile time hack.
19784 IntegerType *IT = dyn_cast<IntegerType>(Val: V->getType());
19785 if (!IT || IT->getBitWidth() != WaveSize)
19786 return false;
19787
19788 if (!isa<Instruction>(Val: V))
19789 return false;
19790 if (!Visited.insert(Ptr: V).second)
19791 return false;
19792 bool Result = false;
19793 for (const auto *U : V->users()) {
19794 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: U)) {
19795 if (V == U->getOperand(i: 1)) {
19796 switch (Intrinsic->getIntrinsicID()) {
19797 default:
19798 Result = false;
19799 break;
19800 case Intrinsic::amdgcn_if_break:
19801 case Intrinsic::amdgcn_if:
19802 case Intrinsic::amdgcn_else:
19803 Result = true;
19804 break;
19805 }
19806 }
19807 if (V == U->getOperand(i: 0)) {
19808 switch (Intrinsic->getIntrinsicID()) {
19809 default:
19810 Result = false;
19811 break;
19812 case Intrinsic::amdgcn_end_cf:
19813 case Intrinsic::amdgcn_loop:
19814 Result = true;
19815 break;
19816 }
19817 }
19818 } else {
19819 Result = hasCFUser(V: U, Visited, WaveSize);
19820 }
19821 if (Result)
19822 break;
19823 }
19824 return Result;
19825}
19826
19827bool SITargetLowering::requiresUniformRegister(MachineFunction &MF,
19828 const Value *V) const {
19829 if (const CallInst *CI = dyn_cast<CallInst>(Val: V)) {
19830 if (CI->isInlineAsm()) {
19831 // FIXME: This cannot give a correct answer. This should only trigger in
19832 // the case where inline asm returns mixed SGPR and VGPR results, used
19833 // outside the defining block. We don't have a specific result to
19834 // consider, so this assumes if any value is SGPR, the overall register
19835 // also needs to be SGPR.
19836 const SIRegisterInfo *SIRI = Subtarget->getRegisterInfo();
19837 TargetLowering::AsmOperandInfoVector TargetConstraints = ParseConstraints(
19838 DL: MF.getDataLayout(), TRI: Subtarget->getRegisterInfo(), Call: *CI);
19839 for (auto &TC : TargetConstraints) {
19840 if (TC.Type == InlineAsm::isOutput) {
19841 ComputeConstraintToUse(OpInfo&: TC, Op: SDValue());
19842 const TargetRegisterClass *RC =
19843 getRegForInlineAsmConstraint(TRI_: SIRI, Constraint: TC.ConstraintCode,
19844 VT: TC.ConstraintVT)
19845 .second;
19846 if (RC && SIRI->isSGPRClass(RC))
19847 return true;
19848 }
19849 }
19850 }
19851 }
19852 SmallPtrSet<const Value *, 16> Visited;
19853 return hasCFUser(V, Visited, WaveSize: Subtarget->getWavefrontSize());
19854}
19855
19856bool SITargetLowering::hasMemSDNodeUser(SDNode *N) const {
19857 for (SDUse &Use : N->uses()) {
19858 if (MemSDNode *M = dyn_cast<MemSDNode>(Val: Use.getUser())) {
19859 if (getBasePtrIndex(N: M) == Use.getOperandNo())
19860 return true;
19861 }
19862 }
19863 return false;
19864}
19865
19866bool SITargetLowering::isReassocProfitable(SelectionDAG &DAG, SDValue N0,
19867 SDValue N1) const {
19868 if (!N0.hasOneUse())
19869 return false;
19870 // Take care of the opportunity to keep N0 uniform
19871 if (N0->isDivergent() || !N1->isDivergent())
19872 return true;
19873 // Check if we have a good chance to form the memory access pattern with the
19874 // base and offset
19875 return (DAG.isBaseWithConstantOffset(Op: N0) &&
19876 hasMemSDNodeUser(N: *N0->user_begin()));
19877}
19878
19879bool SITargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
19880 Register N0, Register N1) const {
19881 return MRI.hasOneNonDBGUse(RegNo: N0); // FIXME: handle regbanks
19882}
19883
19884MachineMemOperand::Flags
19885SITargetLowering::getTargetMMOFlags(const Instruction &I) const {
19886 // Propagate metadata set by AMDGPUAnnotateUniformValues to the MMO of a load.
19887 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
19888 if (I.getMetadata(Kind: "amdgpu.noclobber"))
19889 Flags |= MONoClobber;
19890 if (I.getMetadata(Kind: "amdgpu.last.use"))
19891 Flags |= MOLastUse;
19892 return Flags;
19893}
19894
19895void SITargetLowering::emitExpandAtomicAddrSpacePredicate(
19896 Instruction *AI) const {
19897 // Given: atomicrmw fadd ptr %addr, float %val ordering
19898 //
19899 // With this expansion we produce the following code:
19900 // [...]
19901 // %is.shared = call i1 @llvm.amdgcn.is.shared(ptr %addr)
19902 // br i1 %is.shared, label %atomicrmw.shared, label %atomicrmw.check.private
19903 //
19904 // atomicrmw.shared:
19905 // %cast.shared = addrspacecast ptr %addr to ptr addrspace(3)
19906 // %loaded.shared = atomicrmw fadd ptr addrspace(3) %cast.shared,
19907 // float %val ordering
19908 // br label %atomicrmw.phi
19909 //
19910 // atomicrmw.check.private:
19911 // %is.private = call i1 @llvm.amdgcn.is.private(ptr %int8ptr)
19912 // br i1 %is.private, label %atomicrmw.private, label %atomicrmw.global
19913 //
19914 // atomicrmw.private:
19915 // %cast.private = addrspacecast ptr %addr to ptr addrspace(5)
19916 // %loaded.private = load float, ptr addrspace(5) %cast.private
19917 // %val.new = fadd float %loaded.private, %val
19918 // store float %val.new, ptr addrspace(5) %cast.private
19919 // br label %atomicrmw.phi
19920 //
19921 // atomicrmw.global:
19922 // %cast.global = addrspacecast ptr %addr to ptr addrspace(1)
19923 // %loaded.global = atomicrmw fadd ptr addrspace(1) %cast.global,
19924 // float %val ordering
19925 // br label %atomicrmw.phi
19926 //
19927 // atomicrmw.phi:
19928 // %loaded.phi = phi float [ %loaded.shared, %atomicrmw.shared ],
19929 // [ %loaded.private, %atomicrmw.private ],
19930 // [ %loaded.global, %atomicrmw.global ]
19931 // br label %atomicrmw.end
19932 //
19933 // atomicrmw.end:
19934 // [...]
19935 //
19936 //
19937 // For 64-bit atomics which may reside in private memory, we perform a simpler
19938 // version that only inserts the private check, and uses the flat operation.
19939
19940 IRBuilder<> Builder(AI);
19941 LLVMContext &Ctx = Builder.getContext();
19942
19943 auto *RMW = dyn_cast<AtomicRMWInst>(Val: AI);
19944 const unsigned PtrOpIdx = RMW ? AtomicRMWInst::getPointerOperandIndex()
19945 : AtomicCmpXchgInst::getPointerOperandIndex();
19946 Value *Addr = AI->getOperand(i: PtrOpIdx);
19947
19948 /// TODO: Only need to check private, then emit flat-known-not private (no
19949 /// need for shared block, or cast to global).
19950 AtomicCmpXchgInst *CX = dyn_cast<AtomicCmpXchgInst>(Val: AI);
19951
19952 Align Alignment;
19953 if (RMW)
19954 Alignment = RMW->getAlign();
19955 else if (CX)
19956 Alignment = CX->getAlign();
19957 else
19958 llvm_unreachable("unhandled atomic operation");
19959
19960 // FullFlatEmulation is true if we need to issue the private, shared, and
19961 // global cases.
19962 //
19963 // If this is false, we are only dealing with the flat-targeting-private case,
19964 // where we only insert a check for private and still use the flat instruction
19965 // for global and shared.
19966
19967 bool FullFlatEmulation =
19968 RMW && RMW->getOperation() == AtomicRMWInst::FAdd &&
19969 ((Subtarget->hasAtomicFaddInsts() && RMW->getType()->isFloatTy()) ||
19970 (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() &&
19971 RMW->getType()->isDoubleTy()));
19972
19973 // If the return value isn't used, do not introduce a false use in the phi.
19974 bool ReturnValueIsUsed = !AI->use_empty();
19975
19976 BasicBlock *BB = Builder.GetInsertBlock();
19977 Function *F = BB->getParent();
19978 BasicBlock *ExitBB =
19979 BB->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "atomicrmw.end");
19980 BasicBlock *SharedBB = nullptr;
19981
19982 BasicBlock *CheckPrivateBB = BB;
19983 if (FullFlatEmulation) {
19984 SharedBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.shared", Parent: F, InsertBefore: ExitBB);
19985 CheckPrivateBB =
19986 BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.check.private", Parent: F, InsertBefore: ExitBB);
19987 }
19988
19989 BasicBlock *PrivateBB =
19990 BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.private", Parent: F, InsertBefore: ExitBB);
19991 BasicBlock *GlobalBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.global", Parent: F, InsertBefore: ExitBB);
19992 BasicBlock *PhiBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.phi", Parent: F, InsertBefore: ExitBB);
19993
19994 std::prev(x: BB->end())->eraseFromParent();
19995 Builder.SetInsertPoint(BB);
19996
19997 Value *LoadedShared = nullptr;
19998 if (FullFlatEmulation) {
19999 CallInst *IsShared = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_is_shared,
20000 Args: {Addr}, FMFSource: nullptr, Name: "is.shared");
20001 Builder.CreateCondBr(Cond: IsShared, True: SharedBB, False: CheckPrivateBB);
20002 Builder.SetInsertPoint(SharedBB);
20003 Value *CastToLocal = Builder.CreateAddrSpaceCast(
20004 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::LOCAL_ADDRESS));
20005
20006 Instruction *Clone = AI->clone();
20007 Clone->insertInto(ParentBB: SharedBB, It: SharedBB->end());
20008 Clone->getOperandUse(i: PtrOpIdx).set(CastToLocal);
20009 LoadedShared = Clone;
20010
20011 Builder.CreateBr(Dest: PhiBB);
20012 Builder.SetInsertPoint(CheckPrivateBB);
20013 }
20014
20015 CallInst *IsPrivate = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_is_private,
20016 Args: {Addr}, FMFSource: nullptr, Name: "is.private");
20017 Builder.CreateCondBr(Cond: IsPrivate, True: PrivateBB, False: GlobalBB);
20018
20019 Builder.SetInsertPoint(PrivateBB);
20020
20021 Value *CastToPrivate = Builder.CreateAddrSpaceCast(
20022 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::PRIVATE_ADDRESS));
20023
20024 Value *LoadedPrivate;
20025 if (RMW) {
20026 LoadedPrivate = Builder.CreateAlignedLoad(
20027 Ty: RMW->getType(), Ptr: CastToPrivate, Align: RMW->getAlign(), Name: "loaded.private");
20028
20029 Value *NewVal = buildAtomicRMWValue(Op: RMW->getOperation(), Builder,
20030 Loaded: LoadedPrivate, Val: RMW->getValOperand());
20031
20032 Builder.CreateAlignedStore(Val: NewVal, Ptr: CastToPrivate, Align: RMW->getAlign());
20033 } else {
20034 auto [ResultLoad, Equal] =
20035 buildCmpXchgValue(Builder, Ptr: CastToPrivate, Cmp: CX->getCompareOperand(),
20036 Val: CX->getNewValOperand(), Alignment: CX->getAlign());
20037
20038 Value *Insert = Builder.CreateInsertValue(Agg: PoisonValue::get(T: CX->getType()),
20039 Val: ResultLoad, Idxs: 0);
20040 LoadedPrivate = Builder.CreateInsertValue(Agg: Insert, Val: Equal, Idxs: 1);
20041 }
20042
20043 Builder.CreateBr(Dest: PhiBB);
20044
20045 Builder.SetInsertPoint(GlobalBB);
20046
20047 // Continue using a flat instruction if we only emitted the check for private.
20048 Instruction *LoadedGlobal = AI;
20049 if (FullFlatEmulation) {
20050 Value *CastToGlobal = Builder.CreateAddrSpaceCast(
20051 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::GLOBAL_ADDRESS));
20052 AI->getOperandUse(i: PtrOpIdx).set(CastToGlobal);
20053 }
20054
20055 AI->removeFromParent();
20056 AI->insertInto(ParentBB: GlobalBB, It: GlobalBB->end());
20057
20058 // The new atomicrmw may go through another round of legalization later.
20059 if (!FullFlatEmulation) {
20060 // We inserted the runtime check already, make sure we do not try to
20061 // re-expand this.
20062 // TODO: Should union with any existing metadata.
20063 MDBuilder MDB(F->getContext());
20064 MDNode *RangeNotPrivate =
20065 MDB.createRange(Lo: APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
20066 Hi: APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
20067 LoadedGlobal->setMetadata(KindID: LLVMContext::MD_noalias_addrspace,
20068 Node: RangeNotPrivate);
20069 }
20070
20071 Builder.CreateBr(Dest: PhiBB);
20072
20073 Builder.SetInsertPoint(PhiBB);
20074
20075 if (ReturnValueIsUsed) {
20076 PHINode *Loaded = Builder.CreatePHI(Ty: AI->getType(), NumReservedValues: 3);
20077 AI->replaceAllUsesWith(V: Loaded);
20078 if (FullFlatEmulation)
20079 Loaded->addIncoming(V: LoadedShared, BB: SharedBB);
20080 Loaded->addIncoming(V: LoadedPrivate, BB: PrivateBB);
20081 Loaded->addIncoming(V: LoadedGlobal, BB: GlobalBB);
20082 Loaded->takeName(V: AI);
20083 }
20084
20085 Builder.CreateBr(Dest: ExitBB);
20086}
20087
20088static void convertScratchAtomicToFlatAtomic(Instruction *I,
20089 unsigned PtrOpIdx) {
20090 Value *PtrOp = I->getOperand(i: PtrOpIdx);
20091 assert(PtrOp->getType()->getPointerAddressSpace() ==
20092 AMDGPUAS::PRIVATE_ADDRESS);
20093
20094 Type *FlatPtr = PointerType::get(C&: I->getContext(), AddressSpace: AMDGPUAS::FLAT_ADDRESS);
20095 Value *ASCast = CastInst::CreatePointerCast(S: PtrOp, Ty: FlatPtr, Name: "scratch.ascast",
20096 InsertBefore: I->getIterator());
20097 I->setOperand(i: PtrOpIdx, Val: ASCast);
20098}
20099
20100void SITargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
20101 AtomicRMWInst::BinOp Op = AI->getOperation();
20102
20103 if (AI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
20104 return convertScratchAtomicToFlatAtomic(I: AI, PtrOpIdx: AI->getPointerOperandIndex());
20105
20106 if (Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Or ||
20107 Op == AtomicRMWInst::Xor) {
20108 if (const auto *ConstVal = dyn_cast<Constant>(Val: AI->getValOperand());
20109 ConstVal && ConstVal->isNullValue()) {
20110 // atomicrmw or %ptr, 0 -> atomicrmw add %ptr, 0
20111 AI->setOperation(AtomicRMWInst::Add);
20112
20113 // We may still need the private-alias-flat handling below.
20114
20115 // TODO: Skip this for cases where we cannot access remote memory.
20116 }
20117 }
20118
20119 // The non-flat expansions should only perform the de-canonicalization of
20120 // identity values.
20121 if (AI->getPointerAddressSpace() != AMDGPUAS::FLAT_ADDRESS)
20122 return;
20123
20124 emitExpandAtomicAddrSpacePredicate(AI);
20125}
20126
20127void SITargetLowering::emitExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) const {
20128 if (CI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
20129 return convertScratchAtomicToFlatAtomic(I: CI, PtrOpIdx: CI->getPointerOperandIndex());
20130
20131 emitExpandAtomicAddrSpacePredicate(AI: CI);
20132}
20133
20134void SITargetLowering::emitExpandAtomicLoad(LoadInst *LI) const {
20135 if (LI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
20136 return convertScratchAtomicToFlatAtomic(I: LI, PtrOpIdx: LI->getPointerOperandIndex());
20137
20138 llvm_unreachable(
20139 "Expand Atomic Load only handles SCRATCH -> FLAT conversion");
20140}
20141
20142void SITargetLowering::emitExpandAtomicStore(StoreInst *SI) const {
20143 if (SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
20144 return convertScratchAtomicToFlatAtomic(I: SI, PtrOpIdx: SI->getPointerOperandIndex());
20145
20146 llvm_unreachable(
20147 "Expand Atomic Store only handles SCRATCH -> FLAT conversion");
20148}
20149
20150LoadInst *
20151SITargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
20152 IRBuilder<> Builder(AI);
20153 auto Order = AI->getOrdering();
20154
20155 // The optimization removes store aspect of the atomicrmw. Therefore, cache
20156 // must be flushed if the atomic ordering had a release semantics. This is
20157 // not necessary a fence, a release fence just coincides to do that flush.
20158 // Avoid replacing of an atomicrmw with a release semantics.
20159 if (isReleaseOrStronger(AO: Order))
20160 return nullptr;
20161
20162 LoadInst *LI = Builder.CreateAlignedLoad(
20163 Ty: AI->getType(), Ptr: AI->getPointerOperand(), Align: AI->getAlign());
20164 LI->setAtomic(Ordering: Order, SSID: AI->getSyncScopeID());
20165 LI->copyMetadata(SrcInst: *AI);
20166 LI->takeName(V: AI);
20167 AI->replaceAllUsesWith(V: LI);
20168 AI->eraseFromParent();
20169 return LI;
20170}
20171