1//===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Custom DAG lowering for SI
11//
12//===----------------------------------------------------------------------===//
13
14#include "SIISelLowering.h"
15#include "AMDGPU.h"
16#include "AMDGPUInstrInfo.h"
17#include "AMDGPULaneMaskUtils.h"
18#include "AMDGPUMemoryUtils.h"
19#include "AMDGPUSelectionDAGInfo.h"
20#include "AMDGPUTargetMachine.h"
21#include "GCNSubtarget.h"
22#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
23#include "SIMachineFunctionInfo.h"
24#include "SIRegisterInfo.h"
25#include "llvm/ADT/APFloat.h"
26#include "llvm/ADT/APInt.h"
27#include "llvm/ADT/FloatingPointMode.h"
28#include "llvm/ADT/Statistic.h"
29#include "llvm/Analysis/OptimizationRemarkEmitter.h"
30#include "llvm/Analysis/UniformityAnalysis.h"
31#include "llvm/CodeGen/Analysis.h"
32#include "llvm/CodeGen/ByteProvider.h"
33#include "llvm/CodeGen/FunctionLoweringInfo.h"
34#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
35#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
36#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
37#include "llvm/CodeGen/MachineFrameInfo.h"
38#include "llvm/CodeGen/MachineFunction.h"
39#include "llvm/CodeGen/MachineLoopInfo.h"
40#include "llvm/CodeGen/PseudoSourceValueManager.h"
41#include "llvm/CodeGen/SDPatternMatch.h"
42#include "llvm/IR/DiagnosticInfo.h"
43#include "llvm/IR/IRBuilder.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/IntrinsicsAMDGPU.h"
46#include "llvm/IR/IntrinsicsR600.h"
47#include "llvm/IR/MDBuilder.h"
48#include "llvm/Support/CommandLine.h"
49#include "llvm/Support/KnownBits.h"
50#include "llvm/Support/ModRef.h"
51#include "llvm/TargetParser/AtomicScope.h"
52#include "llvm/Transforms/Utils/LowerAtomic.h"
53#include <optional>
54
55using namespace llvm;
56using namespace llvm::SDPatternMatch;
57
58#define DEBUG_TYPE "si-lower"
59
60STATISTIC(NumTailCalls, "Number of tail calls");
61
62static cl::opt<bool>
63 DisableLoopAlignment("amdgpu-disable-loop-alignment",
64 cl::desc("Do not align and prefetch loops"),
65 cl::init(Val: false));
66
67static cl::opt<bool> UseDivergentRegisterIndexing(
68 "amdgpu-use-divergent-register-indexing", cl::Hidden,
69 cl::desc("Use indirect register addressing for divergent indexes"),
70 cl::init(Val: false));
71
72static DenormalFPEnv getDenormalFPEnv(const MachineFunction &MF) {
73 return MF.getInfo<SIMachineFunctionInfo>()->getMode().getDenormalFPEnv();
74}
75
76static bool denormalModeIsFlushAllF32(const MachineFunction &MF) {
77 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
78 return Info->getMode().FP32Denormals == DenormalMode::getPreserveSign();
79}
80
81static bool denormalModeIsFlushAllF64F16(const MachineFunction &MF) {
82 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
83 return Info->getMode().FP64FP16Denormals == DenormalMode::getPreserveSign();
84}
85
86static unsigned findFirstFreeSGPR(CCState &CCInfo) {
87 unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
88 for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) {
89 if (!CCInfo.isAllocated(Reg: AMDGPU::SGPR0 + Reg)) {
90 return AMDGPU::SGPR0 + Reg;
91 }
92 }
93 llvm_unreachable("Cannot allocate sgpr");
94}
95
96SITargetLowering::SITargetLowering(const TargetMachine &TM,
97 const GCNSubtarget &STI)
98 : AMDGPUTargetLowering(TM, STI, STI), Subtarget(&STI) {
99 addRegisterClass(VT: MVT::i1, RC: &AMDGPU::VReg_1RegClass);
100 addRegisterClass(VT: MVT::i64, RC: &AMDGPU::SReg_64RegClass);
101
102 addRegisterClass(VT: MVT::i32, RC: &AMDGPU::SReg_32RegClass);
103
104 const SIRegisterInfo *TRI = STI.getRegisterInfo();
105 const TargetRegisterClass *V32RegClass =
106 TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 32);
107 addRegisterClass(VT: MVT::f32, RC: V32RegClass);
108
109 addRegisterClass(VT: MVT::v2i32, RC: &AMDGPU::SReg_64RegClass);
110
111 const TargetRegisterClass *V64RegClass =
112 TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 64);
113
114 addRegisterClass(VT: MVT::f64, RC: V64RegClass);
115 addRegisterClass(VT: MVT::v2f32, RC: V64RegClass);
116 addRegisterClass(VT: MVT::Untyped, RC: V64RegClass);
117
118 addRegisterClass(VT: MVT::v3i32, RC: &AMDGPU::SGPR_96RegClass);
119 addRegisterClass(VT: MVT::v3f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 96));
120
121 addRegisterClass(VT: MVT::v2i64, RC: &AMDGPU::SGPR_128RegClass);
122 addRegisterClass(VT: MVT::v2f64, RC: &AMDGPU::SGPR_128RegClass);
123
124 addRegisterClass(VT: MVT::v4i32, RC: &AMDGPU::SGPR_128RegClass);
125 addRegisterClass(VT: MVT::v4f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 128));
126
127 addRegisterClass(VT: MVT::v5i32, RC: &AMDGPU::SGPR_160RegClass);
128 addRegisterClass(VT: MVT::v5f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 160));
129
130 addRegisterClass(VT: MVT::v6i32, RC: &AMDGPU::SGPR_192RegClass);
131 addRegisterClass(VT: MVT::v6f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 192));
132
133 addRegisterClass(VT: MVT::v3i64, RC: &AMDGPU::SGPR_192RegClass);
134 addRegisterClass(VT: MVT::v3f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 192));
135
136 addRegisterClass(VT: MVT::v7i32, RC: &AMDGPU::SGPR_224RegClass);
137 addRegisterClass(VT: MVT::v7f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 224));
138
139 addRegisterClass(VT: MVT::v8i32, RC: &AMDGPU::SGPR_256RegClass);
140 addRegisterClass(VT: MVT::v8f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 256));
141
142 addRegisterClass(VT: MVT::v4i64, RC: &AMDGPU::SGPR_256RegClass);
143 addRegisterClass(VT: MVT::v4f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 256));
144
145 addRegisterClass(VT: MVT::v9i32, RC: &AMDGPU::SGPR_288RegClass);
146 addRegisterClass(VT: MVT::v9f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 288));
147
148 addRegisterClass(VT: MVT::v10i32, RC: &AMDGPU::SGPR_320RegClass);
149 addRegisterClass(VT: MVT::v10f32,
150 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 320));
151
152 addRegisterClass(VT: MVT::v11i32, RC: &AMDGPU::SGPR_352RegClass);
153 addRegisterClass(VT: MVT::v11f32,
154 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 352));
155
156 addRegisterClass(VT: MVT::v12i32, RC: &AMDGPU::SGPR_384RegClass);
157 addRegisterClass(VT: MVT::v12f32,
158 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 384));
159
160 addRegisterClass(VT: MVT::v16i32, RC: &AMDGPU::SGPR_512RegClass);
161 addRegisterClass(VT: MVT::v16f32,
162 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 512));
163
164 addRegisterClass(VT: MVT::v8i64, RC: &AMDGPU::SGPR_512RegClass);
165 addRegisterClass(VT: MVT::v8f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 512));
166
167 addRegisterClass(VT: MVT::v16i64, RC: &AMDGPU::SGPR_1024RegClass);
168 addRegisterClass(VT: MVT::v16f64,
169 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 1024));
170
171 if (Subtarget->has16BitInsts()) {
172 if (Subtarget->useRealTrue16Insts()) {
173 addRegisterClass(VT: MVT::i16, RC: &AMDGPU::VGPR_16RegClass);
174 addRegisterClass(VT: MVT::f16, RC: &AMDGPU::VGPR_16RegClass);
175 addRegisterClass(VT: MVT::bf16, RC: &AMDGPU::VGPR_16RegClass);
176 } else {
177 addRegisterClass(VT: MVT::i16, RC: &AMDGPU::SReg_32RegClass);
178 addRegisterClass(VT: MVT::f16, RC: &AMDGPU::SReg_32RegClass);
179 addRegisterClass(VT: MVT::bf16, RC: &AMDGPU::SReg_32RegClass);
180 }
181
182 // Unless there are also VOP3P operations, not operations are really legal.
183 addRegisterClass(VT: MVT::v2i16, RC: &AMDGPU::SReg_32RegClass);
184 addRegisterClass(VT: MVT::v2f16, RC: &AMDGPU::SReg_32RegClass);
185 addRegisterClass(VT: MVT::v2bf16, RC: &AMDGPU::SReg_32RegClass);
186 addRegisterClass(VT: MVT::v4i16, RC: &AMDGPU::SReg_64RegClass);
187 addRegisterClass(VT: MVT::v4f16, RC: &AMDGPU::SReg_64RegClass);
188 addRegisterClass(VT: MVT::v4bf16, RC: &AMDGPU::SReg_64RegClass);
189 addRegisterClass(VT: MVT::v8i16, RC: &AMDGPU::SGPR_128RegClass);
190 addRegisterClass(VT: MVT::v8f16, RC: &AMDGPU::SGPR_128RegClass);
191 addRegisterClass(VT: MVT::v8bf16, RC: &AMDGPU::SGPR_128RegClass);
192 addRegisterClass(VT: MVT::v16i16, RC: &AMDGPU::SGPR_256RegClass);
193 addRegisterClass(VT: MVT::v16f16, RC: &AMDGPU::SGPR_256RegClass);
194 addRegisterClass(VT: MVT::v16bf16, RC: &AMDGPU::SGPR_256RegClass);
195 addRegisterClass(VT: MVT::v32i16, RC: &AMDGPU::SGPR_512RegClass);
196 addRegisterClass(VT: MVT::v32f16, RC: &AMDGPU::SGPR_512RegClass);
197 addRegisterClass(VT: MVT::v32bf16, RC: &AMDGPU::SGPR_512RegClass);
198 }
199
200 addRegisterClass(VT: MVT::v32i32, RC: &AMDGPU::VReg_1024RegClass);
201 addRegisterClass(VT: MVT::v32f32,
202 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 1024));
203
204 computeRegisterProperties(TRI: Subtarget->getRegisterInfo());
205
206 setMinFunctionAlignment(Align(4));
207 setPrefFunctionAlignment(Align(STI.getInstCacheLineSize()));
208
209 // The boolean content concept here is too inflexible. Compares only ever
210 // really produce a 1-bit result. Any copy/extend from these will turn into a
211 // select, and zext/1 or sext/-1 are equally cheap. Arbitrarily choose 0/1, as
212 // it's what most targets use.
213 setBooleanContents(ZeroOrOneBooleanContent);
214 setBooleanVectorContents(ZeroOrOneBooleanContent);
215
216 // We need to custom lower vector stores from local memory
217 setOperationAction(Ops: ISD::LOAD,
218 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
219 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
220 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
221 MVT::i1, MVT::v32i32},
222 Action: Custom);
223
224 setOperationAction(Ops: ISD::STORE,
225 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
226 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
227 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
228 MVT::i1, MVT::v32i32},
229 Action: Custom);
230
231 if (isTypeLegal(VT: MVT::bf16)) {
232 for (unsigned Opc :
233 {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
234 ISD::FREM, ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
235 ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FSQRT, ISD::FCBRT,
236 ISD::FSIN, ISD::FCOS, ISD::FPOW, ISD::FPOWI,
237 ISD::FLDEXP, ISD::FFREXP, ISD::FLOG, ISD::FLOG2,
238 ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FEXP10,
239 ISD::FCEIL, ISD::FTRUNC, ISD::FRINT, ISD::FNEARBYINT,
240 ISD::FROUND, ISD::FROUNDEVEN, ISD::FFLOOR, ISD::FCANONICALIZE,
241 ISD::SETCC}) {
242 setOperationAction(Op: Opc, VT: MVT::bf16, Action: Promote);
243 }
244
245 // Only targets with packed bf16 instructions, e.g. gfx13.
246 if (Subtarget->hasBF16PackedInsts()) {
247 // Don't use Expand for fsub - the DAG combiner will undo fadd+fneg back
248 // to fsub, causing a libcall (which doesn't exist for bf16). Instead,
249 // directly expand to widened v2bf16 operations.
250 setOperationAction(Op: ISD::FSUB, VT: MVT::bf16, Action: Custom);
251 // Promote scalar operations to a v2bf16 operation with an unused high
252 // lane.
253 for (unsigned Opc : {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FMAXNUM,
254 ISD::FMINNUM, ISD::FCANONICALIZE})
255 AddPromotedToType(Opc, OrigVT: MVT::bf16, DestVT: MVT::v2bf16);
256 }
257
258 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::bf16, Action: Expand);
259
260 setOperationAction(Op: ISD::SELECT, VT: MVT::bf16, Action: Promote);
261 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::bf16, DestVT: MVT::i16);
262
263 setOperationAction(Op: ISD::FABS, VT: MVT::bf16, Action: Legal);
264 setOperationAction(Op: ISD::FNEG, VT: MVT::bf16, Action: Legal);
265 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::bf16, Action: Legal);
266
267 // We only need to custom lower because we can't specify an action for bf16
268 // sources.
269 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
270 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
271 }
272
273 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i16, Action: Expand);
274 setTruncStoreAction(ValVT: MVT::v3i32, MemVT: MVT::v3i16, Action: Expand);
275 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i16, Action: Expand);
276 setTruncStoreAction(ValVT: MVT::v8i32, MemVT: MVT::v8i16, Action: Expand);
277 setTruncStoreAction(ValVT: MVT::v16i32, MemVT: MVT::v16i16, Action: Expand);
278 setTruncStoreAction(ValVT: MVT::v32i32, MemVT: MVT::v32i16, Action: Expand);
279 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i8, Action: Expand);
280 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i8, Action: Expand);
281 setTruncStoreAction(ValVT: MVT::v8i32, MemVT: MVT::v8i8, Action: Expand);
282 setTruncStoreAction(ValVT: MVT::v16i32, MemVT: MVT::v16i8, Action: Expand);
283 setTruncStoreAction(ValVT: MVT::v32i32, MemVT: MVT::v32i8, Action: Expand);
284 setTruncStoreAction(ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Expand);
285 setTruncStoreAction(ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Expand);
286 setTruncStoreAction(ValVT: MVT::v8i16, MemVT: MVT::v8i8, Action: Expand);
287 setTruncStoreAction(ValVT: MVT::v16i16, MemVT: MVT::v16i8, Action: Expand);
288 setTruncStoreAction(ValVT: MVT::v32i16, MemVT: MVT::v32i8, Action: Expand);
289
290 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i16, Action: Expand);
291 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i32, Action: Expand);
292 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i8, Action: Expand);
293 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i8, Action: Expand);
294 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i16, Action: Expand);
295 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i32, Action: Expand);
296 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i32, Action: Expand);
297
298 setOperationAction(Ops: ISD::GlobalAddress, VTs: {MVT::i32, MVT::i64}, Action: Custom);
299 setOperationAction(Ops: ISD::BlockAddress, VTs: {MVT::i32, MVT::i64}, Action: Custom);
300 setOperationAction(Ops: ISD::ExternalSymbol, VTs: {MVT::i32, MVT::i64}, Action: Custom);
301
302 setOperationAction(Op: ISD::SELECT, VT: MVT::i1, Action: Promote);
303 setOperationAction(Op: ISD::SELECT, VT: MVT::i64, Action: Custom);
304 setOperationAction(Op: ISD::SELECT, VT: MVT::f64, Action: Promote);
305 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::f64, DestVT: MVT::i64);
306
307 setOperationAction(Ops: ISD::FSQRT, VTs: {MVT::f32, MVT::f64}, Action: Custom);
308
309 setOperationAction(Ops: ISD::SELECT_CC,
310 VTs: {MVT::f32, MVT::i32, MVT::i64, MVT::f64, MVT::i1}, Action: Expand);
311
312 setOperationAction(Op: ISD::SETCC, VT: MVT::i1, Action: Promote);
313 setOperationAction(Ops: ISD::SETCC, VTs: {MVT::v2i1, MVT::v4i1}, Action: Expand);
314 AddPromotedToType(Opc: ISD::SETCC, OrigVT: MVT::i1, DestVT: MVT::i32);
315
316 setOperationAction(Ops: ISD::TRUNCATE,
317 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
318 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
319 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32},
320 Action: Expand);
321 setOperationAction(Ops: ISD::FP_ROUND,
322 VTs: {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
323 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v9f32,
324 MVT::v10f32, MVT::v11f32, MVT::v12f32, MVT::v16f32},
325 Action: Expand);
326
327 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG,
328 VTs: {MVT::v2i1, MVT::v4i1, MVT::v2i8, MVT::v4i8, MVT::v2i16,
329 MVT::v3i16, MVT::v4i16, MVT::Other},
330 Action: Custom);
331
332 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
333 setOperationAction(Ops: ISD::BR_CC,
334 VTs: {MVT::i1, MVT::i32, MVT::i64, MVT::f32, MVT::f64}, Action: Expand);
335
336 setOperationAction(Ops: {ISD::ABS, ISD::UADDO, ISD::USUBO}, VT: MVT::i32, Action: Legal);
337 setOperationAction(Ops: {ISD::UADDO, ISD::USUBO}, VT: MVT::i64, Action: Legal);
338
339 setOperationAction(Ops: {ISD::UADDO_CARRY, ISD::USUBO_CARRY}, VT: MVT::i32, Action: Legal);
340 setOperationAction(Ops: {ISD::UADDO_CARRY, ISD::USUBO_CARRY}, VT: MVT::i64, Action: Legal);
341
342 setOperationAction(Ops: {ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS}, VT: MVT::i64,
343 Action: Expand);
344
345 setOperationAction(Op: ISD::INLINEASM, VT: MVT::Other, Action: Custom);
346
347 // We only support LOAD/STORE and vector manipulation ops for vectors
348 // with > 4 elements.
349 for (MVT VT :
350 {MVT::v8i32, MVT::v8f32, MVT::v9i32, MVT::v9f32, MVT::v10i32,
351 MVT::v10f32, MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32,
352 MVT::v16i32, MVT::v16f32, MVT::v2i64, MVT::v2f64, MVT::v4i16,
353 MVT::v4f16, MVT::v4bf16, MVT::v3i64, MVT::v3f64, MVT::v6i32,
354 MVT::v6f32, MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64,
355 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
356 MVT::v16bf16, MVT::v16i64, MVT::v16f64, MVT::v32i32, MVT::v32f32,
357 MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
358 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
359 switch (Op) {
360 case ISD::LOAD:
361 case ISD::STORE:
362 case ISD::BUILD_VECTOR:
363 case ISD::BITCAST:
364 case ISD::UNDEF:
365 case ISD::POISON:
366 case ISD::EXTRACT_VECTOR_ELT:
367 case ISD::INSERT_VECTOR_ELT:
368 case ISD::SCALAR_TO_VECTOR:
369 case ISD::IS_FPCLASS:
370 break;
371 case ISD::EXTRACT_SUBVECTOR:
372 case ISD::INSERT_SUBVECTOR:
373 case ISD::CONCAT_VECTORS:
374 setOperationAction(Op, VT, Action: Custom);
375 break;
376 default:
377 setOperationAction(Op, VT, Action: Expand);
378 break;
379 }
380 }
381 }
382
383 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v4f32, Action: Expand);
384
385 // TODO: For dynamic 64-bit vector inserts/extracts, should emit a pseudo that
386 // is expanded to avoid having two separate loops in case the index is a VGPR.
387
388 // Most operations are naturally 32-bit vector operations. We only support
389 // load and store of i64 vectors, so promote v2i64 vector operations to v4i32.
390 for (MVT Vec64 : {MVT::v2i64, MVT::v2f64}) {
391 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
392 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v4i32);
393
394 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
395 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v4i32);
396
397 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
398 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v4i32);
399
400 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
401 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v4i32);
402 }
403
404 for (MVT Vec64 : {MVT::v3i64, MVT::v3f64}) {
405 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
406 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v6i32);
407
408 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
409 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v6i32);
410
411 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
412 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v6i32);
413
414 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
415 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v6i32);
416 }
417
418 for (MVT Vec64 : {MVT::v4i64, MVT::v4f64}) {
419 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
420 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v8i32);
421
422 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
423 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v8i32);
424
425 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
426 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v8i32);
427
428 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
429 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v8i32);
430 }
431
432 for (MVT Vec64 : {MVT::v8i64, MVT::v8f64}) {
433 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
434 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v16i32);
435
436 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
437 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v16i32);
438
439 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
440 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v16i32);
441
442 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
443 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v16i32);
444 }
445
446 for (MVT Vec64 : {MVT::v16i64, MVT::v16f64}) {
447 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
448 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v32i32);
449
450 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
451 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v32i32);
452
453 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
454 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v32i32);
455
456 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
457 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v32i32);
458 }
459
460 setOperationAction(Ops: ISD::VECTOR_SHUFFLE,
461 VTs: {MVT::v4i32, MVT::v4f32, MVT::v8i32, MVT::v8f32,
462 MVT::v16i32, MVT::v16f32, MVT::v32i32, MVT::v32f32},
463 Action: Custom);
464
465 if (Subtarget->hasPkMovB32()) {
466 // TODO: 16-bit element vectors should be legal with even aligned elements.
467 // TODO: Can be legal with wider source types than the result with
468 // subregister extracts.
469 setOperationAction(Ops: ISD::VECTOR_SHUFFLE, VTs: {MVT::v2i32, MVT::v2f32}, Action: Legal);
470 }
471
472 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VT: MVT::v2i32, Action: Legal);
473 // Prevent SELECT v2i32 from being implemented with the above bitwise ops and
474 // instead lower to cndmask in SITargetLowering::LowerSELECT().
475 setOperationAction(Op: ISD::SELECT, VT: MVT::v2i32, Action: Custom);
476 // Enable MatchRotate to produce ISD::ROTR, which is later transformed to
477 // alignbit.
478 setOperationAction(Op: ISD::ROTR, VT: MVT::v2i32, Action: Custom);
479
480 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs: {MVT::v4f16, MVT::v4i16, MVT::v4bf16},
481 Action: Custom);
482
483 // Avoid stack access for these.
484 // TODO: Generalize to more vector types.
485 setOperationAction(Ops: {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
486 VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v2i8, MVT::v4i8,
487 MVT::v8i8, MVT::v4i16, MVT::v4f16, MVT::v4bf16},
488 Action: Custom);
489
490 // Deal with vec3 vector operations when widened to vec4.
491 setOperationAction(Ops: ISD::INSERT_SUBVECTOR,
492 VTs: {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32}, Action: Custom);
493
494 // Deal with vec5/6/7 vector operations when widened to vec8.
495 setOperationAction(Ops: ISD::INSERT_SUBVECTOR,
496 VTs: {MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
497 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
498 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
499 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
500 Action: Custom);
501
502 // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling,
503 // and output demarshalling
504 setOperationAction(Ops: ISD::ATOMIC_CMP_SWAP, VTs: {MVT::i32, MVT::i64}, Action: Custom);
505
506 // We can't return success/failure, only the old value,
507 // let LLVM add the comparison
508 setOperationAction(Ops: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VTs: {MVT::i32, MVT::i64},
509 Action: Expand);
510
511 setOperationAction(Ops: ISD::ADDRSPACECAST, VTs: {MVT::i32, MVT::i64}, Action: Custom);
512
513 setOperationAction(Ops: ISD::BITREVERSE, VTs: {MVT::i32, MVT::i64}, Action: Legal);
514
515 // FIXME: This should be narrowed to i32, but that only happens if i64 is
516 // illegal.
517 // FIXME: Should lower sub-i32 bswaps to bit-ops without v_perm_b32.
518 setOperationAction(Ops: ISD::BSWAP, VTs: {MVT::i64, MVT::i32}, Action: Legal);
519
520 // On SI this is s_memtime and s_memrealtime on VI.
521 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Legal);
522
523 if (Subtarget->hasSMemRealTime() ||
524 Subtarget->getGeneration() >= AMDGPUSubtarget::GFX11)
525 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64, Action: Legal);
526 setOperationAction(Ops: {ISD::TRAP, ISD::DEBUGTRAP}, VT: MVT::Other, Action: Custom);
527
528 if (Subtarget->has16BitInsts()) {
529 setOperationAction(Ops: {ISD::FPOW, ISD::FPOWI}, VT: MVT::f16, Action: Promote);
530 setOperationAction(Ops: {ISD::FLOG, ISD::FEXP, ISD::FLOG10}, VT: MVT::f16, Action: Custom);
531 setOperationAction(Ops: ISD::IS_FPCLASS, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Legal);
532 setOperationAction(Ops: {ISD::FLOG2, ISD::FEXP2}, VT: MVT::f16, Action: Legal);
533 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f16, Action: Legal);
534 } else {
535 setOperationAction(Op: ISD::FSQRT, VT: MVT::f16, Action: Custom);
536 }
537
538 if (Subtarget->hasMadMacF32Insts())
539 setOperationAction(Op: ISD::FMAD, VT: MVT::f32, Action: Legal);
540
541 setOperationAction(Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT: MVT::i32, Action: Custom);
542 setOperationAction(Ops: {ISD::CTTZ, ISD::CTTZ_ZERO_POISON}, VT: MVT::i32, Action: Custom);
543 setOperationAction(Op: ISD::CTLS, VT: MVT::i32, Action: Custom);
544
545 // We only really have 32-bit BFE instructions (and 16-bit on VI).
546 //
547 // On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any
548 // effort to match them now. We want this to be false for i64 cases when the
549 // extraction isn't restricted to the upper or lower half. Ideally we would
550 // have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that
551 // span the midpoint are probably relatively rare, so don't worry about them
552 // for now.
553 setHasExtractBitsInsn(true);
554
555 // Clamp modifier on add/sub
556 if (Subtarget->hasIntClamp())
557 setOperationAction(Ops: {ISD::UADDSAT, ISD::USUBSAT}, VT: MVT::i32, Action: Legal);
558
559 if (Subtarget->hasAddNoCarryInsts())
560 setOperationAction(Ops: {ISD::SADDSAT, ISD::SSUBSAT}, VTs: {MVT::i16, MVT::i32},
561 Action: Legal);
562
563 setOperationAction(
564 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
565 VTs: {MVT::f32, MVT::f64}, Action: Custom);
566
567 // These are really only legal for ieee_mode functions. We should be avoiding
568 // them for functions that don't have ieee_mode enabled, so just say they are
569 // legal.
570 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE},
571 VTs: {MVT::f32, MVT::f64}, Action: Legal);
572
573 if (Subtarget->haveRoundOpsF64())
574 setOperationAction(Ops: {ISD::FTRUNC, ISD::FCEIL, ISD::FROUNDEVEN}, VT: MVT::f64,
575 Action: Legal);
576 else
577 setOperationAction(Ops: {ISD::FCEIL, ISD::FTRUNC, ISD::FROUNDEVEN, ISD::FFLOOR},
578 VT: MVT::f64, Action: Custom);
579
580 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f64, Action: Legal);
581 setOperationAction(Ops: {ISD::FLDEXP, ISD::STRICT_FLDEXP}, VTs: {MVT::f32, MVT::f64},
582 Action: Legal);
583 setOperationAction(Ops: ISD::FFREXP, VTs: {MVT::f32, MVT::f64}, Action: Custom);
584
585 setOperationAction(Ops: {ISD::FSIN, ISD::FCOS, ISD::FDIV}, VT: MVT::f32, Action: Custom);
586 setOperationAction(Op: ISD::FDIV, VT: MVT::f64, Action: Custom);
587
588 setOperationAction(Ops: ISD::BF16_TO_FP, VTs: {MVT::i16, MVT::f32, MVT::f64}, Action: Expand);
589 setOperationAction(Ops: ISD::FP_TO_BF16, VTs: {MVT::i16, MVT::f32, MVT::f64}, Action: Expand);
590
591 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i32,
592 Action: Custom);
593 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i16,
594 Action: Custom);
595 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i1,
596 Action: Custom);
597
598 // Custom lower these because we can't specify a rule based on an illegal
599 // source bf16.
600 setOperationAction(Ops: {ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, VT: MVT::f32, Action: Custom);
601 setOperationAction(Ops: {ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, VT: MVT::f64, Action: Custom);
602
603 if (Subtarget->has16BitInsts()) {
604 setOperationAction(Ops: {ISD::Constant, ISD::SMIN, ISD::SMAX, ISD::UMIN,
605 ISD::UMAX, ISD::UADDSAT, ISD::USUBSAT},
606 VT: MVT::i16, Action: Legal);
607
608 AddPromotedToType(Opc: ISD::SIGN_EXTEND, OrigVT: MVT::i16, DestVT: MVT::i32);
609
610 setOperationAction(Ops: {ISD::ROTR, ISD::ROTL, ISD::SELECT_CC, ISD::BR_CC},
611 VT: MVT::i16, Action: Expand);
612
613 setOperationAction(Ops: {ISD::SIGN_EXTEND, ISD::SDIV, ISD::UDIV, ISD::SREM,
614 ISD::UREM, ISD::BITREVERSE, ISD::CTTZ,
615 ISD::CTTZ_ZERO_POISON, ISD::CTLZ, ISD::CTLZ_ZERO_POISON,
616 ISD::CTPOP},
617 VT: MVT::i16, Action: Promote);
618
619 setOperationAction(Op: ISD::LOAD, VT: MVT::i16, Action: Custom);
620
621 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i16, Action: Expand);
622
623 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::i16, Action: Promote);
624 AddPromotedToType(Opc: ISD::FP16_TO_FP, OrigVT: MVT::i16, DestVT: MVT::i32);
625 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::i16, Action: Promote);
626 AddPromotedToType(Opc: ISD::FP_TO_FP16, OrigVT: MVT::i16, DestVT: MVT::i32);
627
628 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: MVT::i16, Action: Custom);
629 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: MVT::i32, Action: Custom);
630 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i16, Action: Custom);
631 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i1, Action: Custom);
632
633 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i32, Action: Custom);
634
635 // F16 - Constant Actions.
636 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Legal);
637 setOperationAction(Op: ISD::ConstantFP, VT: MVT::bf16, Action: Legal);
638
639 // F16 - Load/Store Actions.
640 setOperationAction(Op: ISD::LOAD, VT: MVT::f16, Action: Promote);
641 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f16, DestVT: MVT::i16);
642 setOperationAction(Op: ISD::STORE, VT: MVT::f16, Action: Promote);
643 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f16, DestVT: MVT::i16);
644
645 // BF16 - Load/Store Actions.
646 setOperationAction(Op: ISD::LOAD, VT: MVT::bf16, Action: Promote);
647 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::bf16, DestVT: MVT::i16);
648 setOperationAction(Op: ISD::STORE, VT: MVT::bf16, Action: Promote);
649 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::bf16, DestVT: MVT::i16);
650
651 // F16 - VOP1 Actions.
652 setOperationAction(Ops: {ISD::FP_ROUND, ISD::STRICT_FP_ROUND, ISD::FCOS,
653 ISD::FSIN, ISD::FROUND},
654 VT: MVT::f16, Action: Custom);
655
656 // BF16 - VOP1 Actions.
657 if (Subtarget->hasBF16TransInsts())
658 setOperationAction(Ops: {ISD::FCOS, ISD::FSIN, ISD::FDIV}, VT: MVT::bf16, Action: Custom);
659
660 // F16 - VOP2 Actions.
661 setOperationAction(Ops: {ISD::BR_CC, ISD::SELECT_CC}, VTs: {MVT::f16, MVT::bf16},
662 Action: Expand);
663 setOperationAction(Ops: {ISD::FLDEXP, ISD::STRICT_FLDEXP}, VT: MVT::f16, Action: Custom);
664 setOperationAction(Op: ISD::FFREXP, VT: MVT::f16, Action: Custom);
665 setOperationAction(Op: ISD::FDIV, VT: MVT::f16, Action: Custom);
666
667 // F16 - VOP3 Actions.
668 setOperationAction(Op: ISD::FMA, VT: MVT::f16, Action: Legal);
669 if (STI.hasMadF16())
670 setOperationAction(Op: ISD::FMAD, VT: MVT::f16, Action: Legal);
671
672 for (MVT VT :
673 {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v4i16, MVT::v4f16,
674 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16,
675 MVT::v16f16, MVT::v16bf16, MVT::v32i16, MVT::v32f16}) {
676 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
677 switch (Op) {
678 case ISD::LOAD:
679 case ISD::STORE:
680 case ISD::BUILD_VECTOR:
681 case ISD::BITCAST:
682 case ISD::UNDEF:
683 case ISD::POISON:
684 case ISD::EXTRACT_VECTOR_ELT:
685 case ISD::INSERT_VECTOR_ELT:
686 case ISD::INSERT_SUBVECTOR:
687 case ISD::SCALAR_TO_VECTOR:
688 case ISD::IS_FPCLASS:
689 break;
690 case ISD::EXTRACT_SUBVECTOR:
691 case ISD::CONCAT_VECTORS:
692 case ISD::FSIN:
693 case ISD::FCOS:
694 setOperationAction(Op, VT, Action: Custom);
695 break;
696 default:
697 setOperationAction(Op, VT, Action: Expand);
698 break;
699 }
700 }
701 }
702
703 // v_perm_b32 can handle either of these.
704 setOperationAction(Ops: ISD::BSWAP, VTs: {MVT::i16, MVT::v2i16}, Action: Legal);
705 setOperationAction(Op: ISD::BSWAP, VT: MVT::v4i16, Action: Custom);
706
707 // Legalize vector types for sat conversions to select v_cvt_pk_[iu]16_f32.
708 if (Subtarget->hasVCvtPkIU16F32())
709 setOperationAction(
710 Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT},
711 VTs: {MVT::v2i16, MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16},
712 Action: Custom);
713
714 // XXX - Do these do anything? Vector constants turn into build_vector.
715 setOperationAction(Ops: ISD::Constant, VTs: {MVT::v2i16, MVT::v2f16}, Action: Legal);
716
717 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON},
718 VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Action: Legal);
719
720 setOperationAction(Op: ISD::STORE, VT: MVT::v2i16, Action: Promote);
721 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2i16, DestVT: MVT::i32);
722 setOperationAction(Op: ISD::STORE, VT: MVT::v2f16, Action: Promote);
723 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f16, DestVT: MVT::i32);
724
725 setOperationAction(Op: ISD::LOAD, VT: MVT::v2i16, Action: Promote);
726 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2i16, DestVT: MVT::i32);
727 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f16, Action: Promote);
728 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f16, DestVT: MVT::i32);
729
730 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::v2i16, Action: Promote);
731 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::v2i16, DestVT: MVT::i32);
732 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::v2f16, Action: Promote);
733 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::v2f16, DestVT: MVT::i32);
734
735 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::v2i16, Action: Promote);
736 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::v2i16, DestVT: MVT::i32);
737 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::v2f16, Action: Promote);
738 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::v2f16, DestVT: MVT::i32);
739
740 setOperationAction(Op: ISD::AND, VT: MVT::v2i16, Action: Promote);
741 AddPromotedToType(Opc: ISD::AND, OrigVT: MVT::v2i16, DestVT: MVT::i32);
742 setOperationAction(Op: ISD::OR, VT: MVT::v2i16, Action: Promote);
743 AddPromotedToType(Opc: ISD::OR, OrigVT: MVT::v2i16, DestVT: MVT::i32);
744 setOperationAction(Op: ISD::XOR, VT: MVT::v2i16, Action: Promote);
745 AddPromotedToType(Opc: ISD::XOR, OrigVT: MVT::v2i16, DestVT: MVT::i32);
746
747 setOperationAction(Op: ISD::LOAD, VT: MVT::v4i16, Action: Promote);
748 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
749 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f16, Action: Promote);
750 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
751 setOperationAction(Op: ISD::LOAD, VT: MVT::v4bf16, Action: Promote);
752 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4bf16, DestVT: MVT::v2i32);
753
754 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::v4i16, Action: Promote);
755 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::v4i16, DestVT: MVT::i64);
756 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::v4f16, Action: Promote);
757 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::v4f16, DestVT: MVT::i64);
758
759 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::v4i16, Action: Promote);
760 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::v4i16, DestVT: MVT::i64);
761 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::v4f16, Action: Promote);
762 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::v4f16, DestVT: MVT::i64);
763
764 setOperationAction(Op: ISD::STORE, VT: MVT::v4i16, Action: Promote);
765 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
766 setOperationAction(Op: ISD::STORE, VT: MVT::v4f16, Action: Promote);
767 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
768 setOperationAction(Op: ISD::STORE, VT: MVT::v4bf16, Action: Promote);
769 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4bf16, DestVT: MVT::v2i32);
770
771 setOperationAction(Op: ISD::LOAD, VT: MVT::v8i16, Action: Promote);
772 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8i16, DestVT: MVT::v4i32);
773 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f16, Action: Promote);
774 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f16, DestVT: MVT::v4i32);
775 setOperationAction(Op: ISD::LOAD, VT: MVT::v8bf16, Action: Promote);
776 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8bf16, DestVT: MVT::v4i32);
777
778 setOperationAction(Op: ISD::STORE, VT: MVT::v4i16, Action: Promote);
779 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
780 setOperationAction(Op: ISD::STORE, VT: MVT::v4f16, Action: Promote);
781 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
782
783 setOperationAction(Op: ISD::STORE, VT: MVT::v8i16, Action: Promote);
784 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8i16, DestVT: MVT::v4i32);
785 setOperationAction(Op: ISD::STORE, VT: MVT::v8f16, Action: Promote);
786 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f16, DestVT: MVT::v4i32);
787 setOperationAction(Op: ISD::STORE, VT: MVT::v8bf16, Action: Promote);
788 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8bf16, DestVT: MVT::v4i32);
789
790 setOperationAction(Op: ISD::LOAD, VT: MVT::v16i16, Action: Promote);
791 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16i16, DestVT: MVT::v8i32);
792 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f16, Action: Promote);
793 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f16, DestVT: MVT::v8i32);
794 setOperationAction(Op: ISD::LOAD, VT: MVT::v16bf16, Action: Promote);
795 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16bf16, DestVT: MVT::v8i32);
796
797 setOperationAction(Op: ISD::STORE, VT: MVT::v16i16, Action: Promote);
798 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16i16, DestVT: MVT::v8i32);
799 setOperationAction(Op: ISD::STORE, VT: MVT::v16f16, Action: Promote);
800 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f16, DestVT: MVT::v8i32);
801 setOperationAction(Op: ISD::STORE, VT: MVT::v16bf16, Action: Promote);
802 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16bf16, DestVT: MVT::v8i32);
803
804 setOperationAction(Op: ISD::LOAD, VT: MVT::v32i16, Action: Promote);
805 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32i16, DestVT: MVT::v16i32);
806 setOperationAction(Op: ISD::LOAD, VT: MVT::v32f16, Action: Promote);
807 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32f16, DestVT: MVT::v16i32);
808 setOperationAction(Op: ISD::LOAD, VT: MVT::v32bf16, Action: Promote);
809 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32bf16, DestVT: MVT::v16i32);
810
811 setOperationAction(Op: ISD::STORE, VT: MVT::v32i16, Action: Promote);
812 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32i16, DestVT: MVT::v16i32);
813 setOperationAction(Op: ISD::STORE, VT: MVT::v32f16, Action: Promote);
814 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32f16, DestVT: MVT::v16i32);
815 setOperationAction(Op: ISD::STORE, VT: MVT::v32bf16, Action: Promote);
816 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32bf16, DestVT: MVT::v16i32);
817
818 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
819 VT: MVT::v2i32, Action: Expand);
820 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Expand);
821
822 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
823 VT: MVT::v4i32, Action: Expand);
824
825 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
826 VT: MVT::v8i32, Action: Expand);
827
828 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
829 Action: Subtarget->hasVOP3PInsts() ? Legal : Custom);
830
831 setOperationAction(Ops: ISD::FNEG, VTs: {MVT::v2f16, MVT::v2bf16}, Action: Legal);
832 // This isn't really legal, but this avoids the legalizer unrolling it (and
833 // allows matching fneg (fabs x) patterns)
834 setOperationAction(Ops: ISD::FABS, VTs: {MVT::v2f16, MVT::v2bf16}, Action: Legal);
835
836 // Can do this in one BFI plus a constant materialize.
837 setOperationAction(Ops: ISD::FCOPYSIGN,
838 VTs: {MVT::v2f16, MVT::v2bf16, MVT::v4f16, MVT::v4bf16,
839 MVT::v8f16, MVT::v8bf16, MVT::v16f16, MVT::v16bf16,
840 MVT::v32f16, MVT::v32bf16},
841 Action: Custom);
842
843 setOperationAction(
844 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
845 VT: MVT::f16, Action: Custom);
846 setOperationAction(Ops: {ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE}, VT: MVT::f16, Action: Legal);
847
848 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE, ISD::FMINIMUMNUM,
849 ISD::FMAXIMUMNUM},
850 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
851 Action: Custom);
852
853 setOperationAction(Ops: {ISD::FMINNUM, ISD::FMAXNUM},
854 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
855 Action: Expand);
856
857 for (MVT Vec16 :
858 {MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
859 MVT::v16bf16, MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
860 setOperationAction(
861 Ops: {ISD::BUILD_VECTOR, ISD::EXTRACT_VECTOR_ELT, ISD::SCALAR_TO_VECTOR},
862 VT: Vec16, Action: Custom);
863 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec16, Action: Expand);
864 }
865 }
866
867 if (Subtarget->hasVOP3PInsts()) {
868 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::MUL, ISD::SHL, ISD::SRL,
869 ISD::SRA, ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX,
870 ISD::UADDSAT, ISD::USUBSAT, ISD::SADDSAT, ISD::SSUBSAT},
871 VT: MVT::v2i16, Action: Legal);
872
873 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FABS,
874 ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE,
875 ISD::FCANONICALIZE},
876 VT: MVT::v2f16, Action: Legal);
877
878 setOperationAction(Ops: ISD::EXTRACT_VECTOR_ELT,
879 VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Action: Custom);
880
881 setOperationAction(Ops: ISD::VECTOR_SHUFFLE,
882 VTs: {MVT::v4f16, MVT::v4i16, MVT::v4bf16, MVT::v8f16,
883 MVT::v8i16, MVT::v8bf16, MVT::v16f16, MVT::v16i16,
884 MVT::v16bf16, MVT::v32f16, MVT::v32i16, MVT::v32bf16},
885 Action: Custom);
886
887 for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16})
888 // Split vector operations.
889 setOperationAction(Ops: {ISD::SHL, ISD::SRA, ISD::SRL, ISD::ADD, ISD::SUB,
890 ISD::MUL, ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN,
891 ISD::UMAX, ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT,
892 ISD::SSUBSAT},
893 VT, Action: Custom);
894
895 for (MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16})
896 // Split vector operations.
897 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FABS,
898 ISD::FCANONICALIZE},
899 VT, Action: Custom);
900
901 setOperationAction(
902 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
903 VTs: {MVT::v2f16, MVT::v4f16}, Action: Custom);
904
905 setOperationAction(Op: ISD::FEXP, VT: MVT::v2f16, Action: Custom);
906 setOperationAction(Ops: ISD::SELECT, VTs: {MVT::v4i16, MVT::v4f16, MVT::v4bf16},
907 Action: Custom);
908
909 if (Subtarget->hasBF16PackedInsts()) {
910 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMAXNUM, ISD::FMINNUM,
911 ISD::FMA, ISD::FNEG, ISD::FABS, ISD::FCANONICALIZE},
912 VT: MVT::v2bf16, Action: Legal);
913
914 for (MVT VT : {MVT::v4bf16, MVT::v8bf16, MVT::v16bf16, MVT::v32bf16})
915 // Split vector operations.
916 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FCANONICALIZE,
917 ISD::FNEG, ISD::FABS},
918 VT, Action: Custom);
919 }
920
921 if (Subtarget->hasAnyPackedFP32Ops()) {
922 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
923 VT: MVT::v2f32, Action: Legal);
924 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
925 VTs: {MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32},
926 Action: Custom);
927 }
928 if (Subtarget->hasAnyPackedFP64Ops()) {
929 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG,
930 ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE,
931 ISD::FCANONICALIZE, ISD::BUILD_VECTOR},
932 VT: MVT::v2f64, Action: Legal);
933 setOperationAction(
934 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
935 VT: MVT::v2f64, Action: Custom);
936 setOperationAction(
937 Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FMINNUM_IEEE,
938 ISD::FMAXNUM_IEEE, ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM,
939 ISD::FMAXIMUMNUM, ISD::FCANONICALIZE},
940 VTs: {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64}, Action: Custom);
941 }
942
943 if (Subtarget->hasAnyPackedU64Ops()) {
944 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::SHL, ISD::BUILD_VECTOR},
945 VT: MVT::v2i64, Action: Legal);
946 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::SHL},
947 VTs: {MVT::v4i64, MVT::v8i64, MVT::v16i64, MVT::v32i64},
948 Action: Custom);
949 }
950 }
951
952 setOperationAction(Ops: {ISD::FNEG, ISD::FABS}, VT: MVT::v4f16, Action: Custom);
953
954 if (Subtarget->has16BitInsts()) {
955 setOperationAction(Op: ISD::SELECT, VT: MVT::v2i16, Action: Promote);
956 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2i16, DestVT: MVT::i32);
957 setOperationAction(Op: ISD::SELECT, VT: MVT::v2f16, Action: Promote);
958 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2f16, DestVT: MVT::i32);
959 setOperationAction(Op: ISD::SELECT, VT: MVT::v2bf16, Action: Promote);
960 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2bf16, DestVT: MVT::i32);
961 } else {
962 // Legalization hack.
963 setOperationAction(Ops: ISD::SELECT, VTs: {MVT::v2i16, MVT::v2f16}, Action: Custom);
964
965 setOperationAction(Ops: {ISD::FNEG, ISD::FABS}, VT: MVT::v2f16, Action: Custom);
966 }
967
968 setOperationAction(Ops: ISD::SELECT,
969 VTs: {MVT::v4i16, MVT::v4f16, MVT::v4bf16, MVT::v2i8, MVT::v4i8,
970 MVT::v8i8, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
971 MVT::v16i16, MVT::v16f16, MVT::v16bf16, MVT::v32i16,
972 MVT::v32f16, MVT::v32bf16},
973 Action: Custom);
974
975 setOperationAction(Ops: {ISD::SMULO, ISD::UMULO}, VT: MVT::i64, Action: Custom);
976
977 if (Subtarget->hasVMulU64Inst())
978 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Legal);
979 else if (Subtarget->hasScalarSMulU64())
980 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Custom);
981
982 if (Subtarget->hasMad64_32())
983 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT: MVT::i32, Action: Custom);
984
985 if (Subtarget->hasSafeSmemPrefetch() || Subtarget->hasVmemPrefInsts())
986 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
987
988 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
989 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM},
990 VTs: {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16}, Action: Legal);
991 } else {
992 // FIXME: For nnan fmaximum, emit the fmaximum3 instead of fmaxnum
993 if (Subtarget->hasMinimum3Maximum3F32())
994 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f32, Action: Legal);
995
996 if (Subtarget->hasMinimum3Maximum3PKF16()) {
997 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::v2f16, Action: Legal);
998
999 // If only the vector form is available, we need to widen to a vector.
1000 if (!Subtarget->hasMinimum3Maximum3F16())
1001 setOperationPromotedToType(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, OrigVT: MVT::f16,
1002 DestVT: MVT::v2f16);
1003 }
1004 }
1005
1006 if (Subtarget->hasVOP3PInsts()) {
1007 // We want to break these into v2f16 pieces, not scalarize.
1008 setOperationAction(Ops: {ISD::FMINIMUM, ISD::FMAXIMUM},
1009 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
1010 Action: Custom);
1011 }
1012
1013 if (Subtarget->hasMinMaxI64Insts())
1014 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT: MVT::i64,
1015 Action: Legal);
1016
1017 setOperationAction(Ops: ISD::INTRINSIC_WO_CHAIN,
1018 VTs: {MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16,
1019 MVT::bf16, MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::i128,
1020 MVT::i8},
1021 Action: Custom);
1022
1023 setOperationAction(Ops: ISD::INTRINSIC_W_CHAIN,
1024 VTs: {MVT::v2f16, MVT::v2i16, MVT::v2bf16, MVT::v3f16,
1025 MVT::v3i16, MVT::v4f16, MVT::v4i16, MVT::v4bf16,
1026 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::Other, MVT::f16,
1027 MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
1028 Action: Custom);
1029
1030 setOperationAction(Ops: ISD::INTRINSIC_VOID,
1031 VTs: {MVT::Other, MVT::v2i16, MVT::v2f16, MVT::v2bf16,
1032 MVT::v3i16, MVT::v3f16, MVT::v4f16, MVT::v4i16,
1033 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
1034 MVT::f16, MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
1035 Action: Custom);
1036
1037 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Custom);
1038 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
1039 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
1040 setOperationAction(Op: ISD::GET_FPENV, VT: MVT::i64, Action: Custom);
1041 setOperationAction(Op: ISD::SET_FPENV, VT: MVT::i64, Action: Custom);
1042
1043 // TODO: Could move this to custom lowering, could benefit from combines on
1044 // extract of relevant bits.
1045 setOperationAction(Op: ISD::GET_FPMODE, VT: MVT::i32, Action: Legal);
1046
1047 setOperationAction(Op: ISD::MUL, VT: MVT::i1, Action: Promote);
1048
1049 if (Subtarget->hasBF16ConversionInsts()) {
1050 setOperationAction(Ops: {ISD::FP_ROUND, ISD::STRICT_FP_ROUND},
1051 VTs: {MVT::bf16, MVT::v2bf16}, Action: Custom);
1052 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2bf16, Action: Legal);
1053 }
1054
1055 if (Subtarget->hasBF16TransInsts()) {
1056 setOperationAction(Ops: {ISD::FEXP2, ISD::FLOG2, ISD::FSQRT}, VT: MVT::bf16, Action: Legal);
1057 }
1058
1059 const bool HasE5M3ConversionInsts =
1060 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
1061 if (Subtarget->hasOCPFP8ConversionInsts() || HasE5M3ConversionInsts) {
1062 setOperationAction(Ops: ISD::CONVERT_FROM_ARBITRARY_FP, VTs: {MVT::f32, MVT::v2f32},
1063 Action: Custom);
1064 setOperationAction(Op: ISD::CONVERT_FROM_ARBITRARY_FP, VT: MVT::v2i8, Action: Custom);
1065
1066 // i8 result promotes to i16, wider vectors split down to v2i8, and v2i8 is
1067 // handled in ReplaceNodeResults before the legalizer splits it per lane.
1068 setOperationAction(Ops: ISD::CONVERT_TO_ARBITRARY_FP, VTs: {MVT::i16, MVT::v2i8},
1069 Action: Custom);
1070 }
1071
1072 if (Subtarget->hasFP8F16ConversionInsts()) {
1073 setOperationAction(Ops: ISD::CONVERT_FROM_ARBITRARY_FP, VTs: {MVT::f16, MVT::v2f16},
1074 Action: Custom);
1075 }
1076
1077 if (Subtarget->hasCvtPkF16F32Inst()) {
1078 setOperationAction(Ops: ISD::FP_ROUND,
1079 VTs: {MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16},
1080 Action: Custom);
1081 }
1082
1083 setTargetDAGCombine({ISD::ADD,
1084 ISD::PTRADD,
1085 ISD::SUB,
1086 ISD::MUL,
1087 ISD::FADD,
1088 ISD::FSUB,
1089 ISD::FDIV,
1090 ISD::FMUL,
1091 ISD::FMINNUM,
1092 ISD::FMAXNUM,
1093 ISD::FMINNUM_IEEE,
1094 ISD::FMAXNUM_IEEE,
1095 ISD::FMINIMUM,
1096 ISD::FMAXIMUM,
1097 ISD::FMINIMUMNUM,
1098 ISD::FMAXIMUMNUM,
1099 ISD::FMA,
1100 ISD::ABS,
1101 ISD::SMIN,
1102 ISD::SMAX,
1103 ISD::UMIN,
1104 ISD::UMAX,
1105 ISD::SETCC,
1106 ISD::SELECT,
1107 ISD::SMIN,
1108 ISD::SMAX,
1109 ISD::UMIN,
1110 ISD::UMAX,
1111 ISD::USUBSAT,
1112 ISD::AND,
1113 ISD::OR,
1114 ISD::XOR,
1115 ISD::SHL,
1116 ISD::SRL,
1117 ISD::SRA,
1118 ISD::FSHR,
1119 ISD::SINT_TO_FP,
1120 ISD::UINT_TO_FP,
1121 ISD::FCANONICALIZE,
1122 ISD::SCALAR_TO_VECTOR,
1123 ISD::ZERO_EXTEND,
1124 ISD::SIGN_EXTEND_INREG,
1125 ISD::ANY_EXTEND,
1126 ISD::EXTRACT_VECTOR_ELT,
1127 ISD::INSERT_VECTOR_ELT,
1128 ISD::FCOPYSIGN});
1129
1130 if (Subtarget->has16BitInsts() && !Subtarget->hasMed3_16())
1131 setTargetDAGCombine(ISD::FP_ROUND);
1132
1133 // All memory operations. Some folding on the pointer operand is done to help
1134 // matching the constant offsets in the addressing modes.
1135 setTargetDAGCombine({ISD::LOAD,
1136 ISD::STORE,
1137 ISD::ATOMIC_LOAD,
1138 ISD::ATOMIC_STORE,
1139 ISD::ATOMIC_CMP_SWAP,
1140 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
1141 ISD::ATOMIC_SWAP,
1142 ISD::ATOMIC_LOAD_ADD,
1143 ISD::ATOMIC_LOAD_SUB,
1144 ISD::ATOMIC_LOAD_AND,
1145 ISD::ATOMIC_LOAD_OR,
1146 ISD::ATOMIC_LOAD_XOR,
1147 ISD::ATOMIC_LOAD_NAND,
1148 ISD::ATOMIC_LOAD_MIN,
1149 ISD::ATOMIC_LOAD_MAX,
1150 ISD::ATOMIC_LOAD_UMIN,
1151 ISD::ATOMIC_LOAD_UMAX,
1152 ISD::ATOMIC_LOAD_FADD,
1153 ISD::ATOMIC_LOAD_FMIN,
1154 ISD::ATOMIC_LOAD_FMAX,
1155 ISD::ATOMIC_LOAD_UINC_WRAP,
1156 ISD::ATOMIC_LOAD_UDEC_WRAP,
1157 ISD::ATOMIC_LOAD_USUB_COND,
1158 ISD::ATOMIC_LOAD_USUB_SAT,
1159 ISD::INTRINSIC_VOID,
1160 ISD::INTRINSIC_W_CHAIN});
1161
1162 // FIXME: In other contexts we pretend this is a per-function property.
1163 setStackPointerRegisterToSaveRestore(AMDGPU::SGPR32);
1164
1165 setSchedulingPreference(Sched::RegPressure);
1166}
1167
1168const GCNSubtarget *SITargetLowering::getSubtarget() const { return Subtarget; }
1169
1170ArrayRef<MCPhysReg> SITargetLowering::getRoundingControlRegisters() const {
1171 static const MCPhysReg RCRegs[] = {AMDGPU::MODE};
1172 return RCRegs;
1173}
1174
1175//===----------------------------------------------------------------------===//
1176// TargetLowering queries
1177//===----------------------------------------------------------------------===//
1178
1179// v_mad_mix* support a conversion from f16 to f32.
1180//
1181// There is only one special case when denormals are enabled we don't currently,
1182// where this is OK to use.
1183bool SITargetLowering::isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
1184 EVT DestVT, EVT SrcVT) const {
1185 return DestVT.getScalarType() == MVT::f32 &&
1186 ((((Opcode == ISD::FMAD && Subtarget->hasMadMixInsts()) ||
1187 (Opcode == ISD::FMA && Subtarget->hasFmaMixInsts())) &&
1188 SrcVT.getScalarType() == MVT::f16) ||
1189 (Opcode == ISD::FMA && Subtarget->hasFmaMixBF16Insts() &&
1190 SrcVT.getScalarType() == MVT::bf16)) &&
1191 // TODO: This probably only requires no input flushing?
1192 denormalModeIsFlushAllF32(MF: DAG.getMachineFunction());
1193}
1194
1195bool SITargetLowering::isFPExtFoldable(const MachineInstr &MI, unsigned Opcode,
1196 LLT DestTy, LLT SrcTy) const {
1197 return ((Opcode == TargetOpcode::G_FMAD && Subtarget->hasMadMixInsts()) ||
1198 (Opcode == TargetOpcode::G_FMA && Subtarget->hasFmaMixInsts())) &&
1199 DestTy.getScalarSizeInBits() == 32 &&
1200 SrcTy.getScalarSizeInBits() == 16 &&
1201 // TODO: This probably only requires no input flushing?
1202 denormalModeIsFlushAllF32(MF: *MI.getMF());
1203}
1204
1205bool SITargetLowering::isShuffleMaskLegal(ArrayRef<int>, EVT) const {
1206 // SI has some legal vector types, but no legal vector operations. Say no
1207 // shuffles are legal in order to prefer scalarizing some vector operations.
1208 return false;
1209}
1210
1211MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
1212 CallingConv::ID CC,
1213 EVT VT) const {
1214 if (CC == CallingConv::AMDGPU_KERNEL)
1215 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1216
1217 if (VT.isVector()) {
1218 EVT ScalarVT = VT.getScalarType();
1219 unsigned Size = ScalarVT.getSizeInBits();
1220 if (Size == 16) {
1221 return Subtarget->has16BitInsts()
1222 ? MVT::getVectorVT(VT: ScalarVT.getSimpleVT(), NumElements: 2)
1223 : MVT::i32;
1224 }
1225
1226 if (Size < 16)
1227 return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32;
1228 return Size == 32 ? ScalarVT.getSimpleVT() : MVT::i32;
1229 }
1230
1231 if (!Subtarget->has16BitInsts() && VT.getSizeInBits() == 16)
1232 return MVT::i32;
1233
1234 if (VT.getSizeInBits() > 32)
1235 return MVT::i32;
1236
1237 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1238}
1239
1240unsigned SITargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
1241 CallingConv::ID CC,
1242 EVT VT) const {
1243 if (CC == CallingConv::AMDGPU_KERNEL)
1244 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1245
1246 if (VT.isVector()) {
1247 unsigned NumElts = VT.getVectorNumElements();
1248 EVT ScalarVT = VT.getScalarType();
1249 unsigned Size = ScalarVT.getSizeInBits();
1250
1251 // FIXME: Should probably promote 8-bit vectors to i16.
1252 if (Size == 16)
1253 return (NumElts + 1) / 2;
1254
1255 if (Size <= 32)
1256 return NumElts;
1257
1258 if (Size > 32)
1259 return NumElts * ((Size + 31) / 32);
1260 } else if (VT.getSizeInBits() > 32)
1261 return (VT.getSizeInBits() + 31) / 32;
1262
1263 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1264}
1265
1266unsigned SITargetLowering::getVectorTypeBreakdownForCallingConv(
1267 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
1268 unsigned &NumIntermediates, MVT &RegisterVT) const {
1269 if (CC != CallingConv::AMDGPU_KERNEL && VT.isVector()) {
1270 unsigned NumElts = VT.getVectorNumElements();
1271 EVT ScalarVT = VT.getScalarType();
1272 unsigned Size = ScalarVT.getSizeInBits();
1273 // FIXME: We should fix the ABI to be the same on targets without 16-bit
1274 // support, but unless we can properly handle 3-vectors, it will be still be
1275 // inconsistent.
1276 if (Size == 16) {
1277 MVT SimpleIntermediateVT =
1278 MVT::getVectorVT(VT: ScalarVT.getSimpleVT(), EC: ElementCount::getFixed(MinVal: 2));
1279 IntermediateVT = SimpleIntermediateVT;
1280 RegisterVT = Subtarget->has16BitInsts() ? SimpleIntermediateVT : MVT::i32;
1281 NumIntermediates = (NumElts + 1) / 2;
1282 return (NumElts + 1) / 2;
1283 }
1284
1285 if (Size == 32) {
1286 RegisterVT = ScalarVT.getSimpleVT();
1287 IntermediateVT = RegisterVT;
1288 NumIntermediates = NumElts;
1289 return NumIntermediates;
1290 }
1291
1292 if (Size < 16 && Subtarget->has16BitInsts()) {
1293 // FIXME: Should probably form v2i16 pieces
1294 RegisterVT = MVT::i16;
1295 IntermediateVT = ScalarVT;
1296 NumIntermediates = NumElts;
1297 return NumIntermediates;
1298 }
1299
1300 if (Size != 16 && Size <= 32) {
1301 RegisterVT = MVT::i32;
1302 IntermediateVT = ScalarVT;
1303 NumIntermediates = NumElts;
1304 return NumIntermediates;
1305 }
1306
1307 if (Size > 32) {
1308 RegisterVT = MVT::i32;
1309 IntermediateVT = RegisterVT;
1310 NumIntermediates = NumElts * ((Size + 31) / 32);
1311 return NumIntermediates;
1312 }
1313 }
1314
1315 return TargetLowering::getVectorTypeBreakdownForCallingConv(
1316 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
1317}
1318
1319static EVT memVTFromLoadIntrData(const SITargetLowering &TLI,
1320 const DataLayout &DL, Type *Ty,
1321 unsigned MaxNumLanes) {
1322 assert(MaxNumLanes != 0);
1323
1324 LLVMContext &Ctx = Ty->getContext();
1325 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
1326 unsigned NumElts = std::min(a: MaxNumLanes, b: VT->getNumElements());
1327 return EVT::getVectorVT(Context&: Ctx, VT: TLI.getValueType(DL, Ty: VT->getElementType()),
1328 NumElements: NumElts);
1329 }
1330
1331 return TLI.getValueType(DL, Ty);
1332}
1333
1334// Peek through TFE struct returns to only use the data size.
1335static EVT memVTFromLoadIntrReturn(const SITargetLowering &TLI,
1336 const DataLayout &DL, Type *Ty,
1337 unsigned MaxNumLanes) {
1338 auto *ST = dyn_cast<StructType>(Val: Ty);
1339 if (!ST)
1340 return memVTFromLoadIntrData(TLI, DL, Ty, MaxNumLanes);
1341
1342 // TFE intrinsics return an aggregate type.
1343 assert(ST->getNumContainedTypes() == 2 &&
1344 ST->getContainedType(1)->isIntegerTy(32));
1345 return memVTFromLoadIntrData(TLI, DL, Ty: ST->getContainedType(i: 0), MaxNumLanes);
1346}
1347
1348/// Map address space 7 to MVT::amdgpuBufferFatPointer because that's its
1349/// in-memory representation. This return value is a custom type because there
1350/// is no MVT::i160 and adding one breaks integer promotion logic. While this
1351/// could cause issues during codegen, these address space 7 pointers will be
1352/// rewritten away by then. Therefore, we can return MVT::amdgpuBufferFatPointer
1353/// in order to allow pre-codegen passes that query TargetTransformInfo, often
1354/// for cost modeling, to work. (This also sets us up decently for doing the
1355/// buffer lowering in GlobalISel if SelectionDAG ever goes away.)
1356MVT SITargetLowering::getPointerTy(const DataLayout &DL, unsigned AS) const {
1357 if (AMDGPUAS::BUFFER_FAT_POINTER == AS && DL.getPointerSizeInBits(AS) == 160)
1358 return MVT::amdgpuBufferFatPointer;
1359 if (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
1360 DL.getPointerSizeInBits(AS) == 192)
1361 return MVT::amdgpuBufferStridedPointer;
1362 return AMDGPUTargetLowering::getPointerTy(DL, AS);
1363}
1364/// Similarly, the in-memory representation of a p7 is {p8, i32}, aka
1365/// v8i32 when padding is added.
1366/// The in-memory representation of a p9 is {p8, i32, i32}, which is
1367/// also v8i32 with padding.
1368MVT SITargetLowering::getPointerMemTy(const DataLayout &DL, unsigned AS) const {
1369 if ((AMDGPUAS::BUFFER_FAT_POINTER == AS &&
1370 DL.getPointerSizeInBits(AS) == 160) ||
1371 (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
1372 DL.getPointerSizeInBits(AS) == 192))
1373 return MVT::v8i32;
1374 return AMDGPUTargetLowering::getPointerMemTy(DL, AS);
1375}
1376
1377static unsigned getIntrMemWidth(unsigned IntrID) {
1378 switch (IntrID) {
1379 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1380 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1381 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1382 return 8;
1383 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1384 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1385 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1386 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
1387 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
1388 case Intrinsic::amdgcn_flat_load_monitor_b32:
1389 case Intrinsic::amdgcn_global_load_monitor_b32:
1390 return 32;
1391 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1392 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1393 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1394 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
1395 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
1396 case Intrinsic::amdgcn_flat_load_monitor_b64:
1397 case Intrinsic::amdgcn_global_load_monitor_b64:
1398 return 64;
1399 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1400 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
1401 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
1402 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
1403 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
1404 case Intrinsic::amdgcn_flat_load_monitor_b128:
1405 case Intrinsic::amdgcn_global_load_monitor_b128:
1406 return 128;
1407 default:
1408 llvm_unreachable("Unknown width");
1409 }
1410}
1411
1412static AtomicOrdering parseAtomicOrderingCABIArg(const CallBase &CI,
1413 unsigned ArgIdx) {
1414 Value *OrderingArg = CI.getArgOperand(i: ArgIdx);
1415 unsigned Ord = cast<ConstantInt>(Val: OrderingArg)->getZExtValue();
1416 switch (AtomicOrderingCABI(Ord)) {
1417 case AtomicOrderingCABI::acquire:
1418 return AtomicOrdering::Acquire;
1419 break;
1420 case AtomicOrderingCABI::release:
1421 return AtomicOrdering::Release;
1422 break;
1423 case AtomicOrderingCABI::seq_cst:
1424 return AtomicOrdering::SequentiallyConsistent;
1425 break;
1426 default:
1427 return AtomicOrdering::Monotonic;
1428 }
1429}
1430
1431static unsigned parseSyncscopeMDArg(const CallBase &CI, unsigned ArgIdx) {
1432 MDNode *ScopeMD = cast<MDNode>(
1433 Val: cast<MetadataAsValue>(Val: CI.getArgOperand(i: ArgIdx))->getMetadata());
1434 StringRef Scope = cast<MDString>(Val: ScopeMD->getOperand(I: 0))->getString();
1435 return CI.getContext().getOrInsertSyncScopeID(SSN: Scope);
1436}
1437
1438void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
1439 const CallBase &CI,
1440 MachineFunction &MF,
1441 unsigned IntrID) const {
1442 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
1443 if (CI.hasMetadata(KindID: LLVMContext::MD_invariant_load))
1444 Flags |= MachineMemOperand::MOInvariant;
1445 if (CI.hasMetadata(KindID: LLVMContext::MD_nontemporal))
1446 Flags |= MachineMemOperand::MONonTemporal;
1447 Flags |= getTargetMMOFlags(I: CI);
1448
1449 if (const AMDGPU::RsrcIntrinsic *RsrcIntr =
1450 AMDGPU::lookupRsrcIntrinsic(Intr: IntrID)) {
1451 AttributeSet Attr =
1452 Intrinsic::getFnAttributes(C&: CI.getContext(), id: (Intrinsic::ID)IntrID);
1453 MemoryEffects ME = Attr.getMemoryEffects();
1454 if (ME.doesNotAccessMemory())
1455 return;
1456
1457 bool IsSPrefetch = IntrID == Intrinsic::amdgcn_s_buffer_prefetch_data;
1458 if (!IsSPrefetch) {
1459 auto *Aux = cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1));
1460 if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE)
1461 Flags |= MachineMemOperand::MOVolatile;
1462 }
1463
1464 Flags |= MachineMemOperand::MODereferenceable;
1465
1466 IntrinsicInfo Info;
1467 // TODO: Should images get their own address space?
1468 Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
1469
1470 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = nullptr;
1471 if (RsrcIntr->IsImage) {
1472 const AMDGPU::ImageDimIntrinsicInfo *Intr =
1473 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrID);
1474 BaseOpcode = AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Intr->BaseOpcode);
1475 Info.align.reset();
1476 }
1477
1478 Value *RsrcArg = CI.getArgOperand(i: RsrcIntr->RsrcArg);
1479 if (auto *RsrcPtrTy = dyn_cast<PointerType>(Val: RsrcArg->getType())) {
1480 if (RsrcPtrTy->getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE)
1481 // We conservatively set the memory operand of a buffer intrinsic to the
1482 // base resource pointer, so that we can access alias information about
1483 // those pointers. Cases like "this points at the same value
1484 // but with a different offset" are handled in
1485 // areMemAccessesTriviallyDisjoint.
1486 Info.ptrVal = RsrcArg;
1487 }
1488
1489 if (ME.onlyReadsMemory()) {
1490 if (RsrcIntr->IsImage) {
1491 unsigned MaxNumLanes = 4;
1492
1493 if (!BaseOpcode->Gather4) {
1494 // If this isn't a gather, we may have excess loaded elements in the
1495 // IR type. Check the dmask for the real number of elements loaded.
1496 unsigned DMask =
1497 cast<ConstantInt>(Val: CI.getArgOperand(i: 0))->getZExtValue();
1498 MaxNumLanes = DMask == 0 ? 1 : llvm::popcount(Value: DMask);
1499 }
1500
1501 Info.memVT = memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(),
1502 Ty: CI.getType(), MaxNumLanes);
1503 } else {
1504 Info.memVT =
1505 memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(), Ty: CI.getType(),
1506 MaxNumLanes: std::numeric_limits<unsigned>::max());
1507 }
1508
1509 // FIXME: What does alignment mean for an image?
1510 Info.opc = ISD::INTRINSIC_W_CHAIN;
1511 Info.flags = Flags | MachineMemOperand::MOLoad;
1512 } else if (ME.onlyWritesMemory()) {
1513 Info.opc = ISD::INTRINSIC_VOID;
1514
1515 Type *DataTy = CI.getArgOperand(i: 0)->getType();
1516 if (RsrcIntr->IsImage) {
1517 unsigned DMask = cast<ConstantInt>(Val: CI.getArgOperand(i: 1))->getZExtValue();
1518 unsigned DMaskLanes = DMask == 0 ? 1 : llvm::popcount(Value: DMask);
1519 Info.memVT = memVTFromLoadIntrData(TLI: *this, DL: MF.getDataLayout(), Ty: DataTy,
1520 MaxNumLanes: DMaskLanes);
1521 } else
1522 Info.memVT = getValueType(DL: MF.getDataLayout(), Ty: DataTy);
1523
1524 Info.flags = Flags | MachineMemOperand::MOStore;
1525 } else {
1526 // Atomic, NoReturn Sampler or prefetch
1527 Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID
1528 : ISD::INTRINSIC_W_CHAIN;
1529
1530 switch (IntrID) {
1531 default:
1532 Info.flags = Flags | MachineMemOperand::MOLoad;
1533 if (!IsSPrefetch)
1534 Info.flags |= MachineMemOperand::MOStore;
1535
1536 if ((RsrcIntr->IsImage && BaseOpcode->NoReturn) || IsSPrefetch) {
1537 // Fake memory access type for no return sampler intrinsics
1538 Info.memVT = MVT::i32;
1539 } else {
1540 // XXX - Should this be volatile without known ordering?
1541 Info.flags |= MachineMemOperand::MOVolatile;
1542 Info.memVT = MVT::getVT(Ty: CI.getArgOperand(i: 0)->getType());
1543 }
1544 break;
1545 case Intrinsic::amdgcn_raw_buffer_load_lds:
1546 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
1547 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
1548 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
1549 case Intrinsic::amdgcn_struct_buffer_load_lds:
1550 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
1551 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
1552 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
1553 unsigned Width = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getZExtValue();
1554
1555 // Entry 0: Load from buffer.
1556 // Don't set an offset, since the pointer value always represents the
1557 // base of the buffer.
1558 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: Width * 8);
1559 Info.flags = Flags | MachineMemOperand::MOLoad;
1560 Infos.push_back(Elt: Info);
1561
1562 // Entry 1: Store to LDS.
1563 // Instruction offset is applied, and an additional per-lane offset
1564 // which we simulate using a larger memory type.
1565 Info.memVT = EVT::getIntegerVT(
1566 Context&: CI.getContext(), BitWidth: Width * 8 * Subtarget->getWavefrontSize());
1567 Info.ptrVal = CI.getArgOperand(i: 1); // LDS destination pointer
1568 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 2))
1569 ->getZExtValue();
1570 Info.fallbackAddressSpace = AMDGPUAS::LOCAL_ADDRESS;
1571 Info.flags = Flags | MachineMemOperand::MOStore;
1572 Infos.push_back(Elt: Info);
1573 return;
1574 }
1575 case Intrinsic::amdgcn_raw_atomic_buffer_load:
1576 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
1577 case Intrinsic::amdgcn_struct_atomic_buffer_load:
1578 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
1579 Info.memVT =
1580 memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(), Ty: CI.getType(),
1581 MaxNumLanes: std::numeric_limits<unsigned>::max());
1582 Info.flags = Flags | MachineMemOperand::MOLoad;
1583 Infos.push_back(Elt: Info);
1584 return;
1585 }
1586 }
1587 }
1588 Infos.push_back(Elt: Info);
1589 return;
1590 }
1591
1592 IntrinsicInfo Info;
1593 switch (IntrID) {
1594 case Intrinsic::amdgcn_ds_ordered_add:
1595 case Intrinsic::amdgcn_ds_ordered_swap: {
1596 Info.opc = ISD::INTRINSIC_W_CHAIN;
1597 Info.memVT = MVT::getVT(Ty: CI.getType());
1598 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1599 Info.align.reset();
1600 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1601
1602 const ConstantInt *Vol = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 4));
1603 if (!Vol->isZero())
1604 Info.flags |= MachineMemOperand::MOVolatile;
1605
1606 Infos.push_back(Elt: Info);
1607 return;
1608 }
1609 case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
1610 case Intrinsic::amdgcn_ds_sub_gs_reg_rtn: {
1611 Info.opc = ISD::INTRINSIC_W_CHAIN;
1612 Info.memVT = MVT::getVT(Ty: CI.getOperand(i_nocapture: 0)->getType());
1613 Info.ptrVal = nullptr;
1614 Info.fallbackAddressSpace = AMDGPUAS::STREAMOUT_REGISTER;
1615 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1616 Infos.push_back(Elt: Info);
1617 return;
1618 }
1619 case Intrinsic::amdgcn_ds_append:
1620 case Intrinsic::amdgcn_ds_consume: {
1621 Info.opc = ISD::INTRINSIC_W_CHAIN;
1622 Info.memVT = MVT::getVT(Ty: CI.getType());
1623 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1624 Info.align.reset();
1625 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1626
1627 const ConstantInt *Vol = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1));
1628 if (!Vol->isZero())
1629 Info.flags |= MachineMemOperand::MOVolatile;
1630
1631 Infos.push_back(Elt: Info);
1632 return;
1633 }
1634 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
1635 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64: {
1636 Info.opc = (IntrID == Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64)
1637 ? ISD::INTRINSIC_W_CHAIN
1638 : ISD::INTRINSIC_VOID;
1639 Info.memVT = MVT::getVT(Ty: CI.getType());
1640 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1641 Info.memVT = MVT::i64;
1642 Info.size = 8;
1643 Info.align.reset();
1644 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1645 Info.order = AtomicOrdering::Monotonic;
1646 Infos.push_back(Elt: Info);
1647 return;
1648 }
1649 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
1650 case Intrinsic::amdgcn_image_bvh_intersect_ray:
1651 case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
1652 Info.opc = ISD::INTRINSIC_W_CHAIN;
1653 Info.memVT =
1654 MVT::getVT(Ty: IntrID == Intrinsic::amdgcn_image_bvh_intersect_ray
1655 ? CI.getType()
1656 : cast<StructType>(Val: CI.getType())
1657 ->getElementType(N: 0)); // XXX: what is correct VT?
1658
1659 Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
1660 Info.align.reset();
1661 Info.flags = Flags | MachineMemOperand::MOLoad |
1662 MachineMemOperand::MODereferenceable;
1663 Infos.push_back(Elt: Info);
1664 return;
1665 }
1666 case Intrinsic::amdgcn_global_atomic_fmin_num:
1667 case Intrinsic::amdgcn_global_atomic_fmax_num:
1668 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1669 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1670 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
1671 Info.opc = ISD::INTRINSIC_W_CHAIN;
1672 Info.memVT = MVT::getVT(Ty: CI.getType());
1673 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1674 Info.align.reset();
1675 Info.flags =
1676 Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
1677 MachineMemOperand::MODereferenceable | MachineMemOperand::MOVolatile;
1678 Infos.push_back(Elt: Info);
1679 return;
1680 }
1681 case Intrinsic::amdgcn_cluster_load_b32:
1682 case Intrinsic::amdgcn_cluster_load_b64:
1683 case Intrinsic::amdgcn_cluster_load_b128:
1684 case Intrinsic::amdgcn_ds_load_tr6_b96:
1685 case Intrinsic::amdgcn_ds_load_tr4_b64:
1686 case Intrinsic::amdgcn_ds_load_tr8_b64:
1687 case Intrinsic::amdgcn_ds_load_tr16_b128:
1688 case Intrinsic::amdgcn_global_load_tr6_b96:
1689 case Intrinsic::amdgcn_global_load_tr4_b64:
1690 case Intrinsic::amdgcn_global_load_tr_b64:
1691 case Intrinsic::amdgcn_global_load_tr_b128:
1692 case Intrinsic::amdgcn_ds_read_tr4_b64:
1693 case Intrinsic::amdgcn_ds_read_tr6_b96:
1694 case Intrinsic::amdgcn_ds_read_tr8_b64:
1695 case Intrinsic::amdgcn_ds_read_tr16_b64: {
1696 Info.opc = ISD::INTRINSIC_W_CHAIN;
1697 Info.memVT = MVT::getVT(Ty: CI.getType());
1698 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1699 Info.align.reset();
1700 Info.flags = Flags | MachineMemOperand::MOLoad;
1701 Infos.push_back(Elt: Info);
1702 return;
1703 }
1704 case Intrinsic::amdgcn_flat_load_monitor_b32:
1705 case Intrinsic::amdgcn_flat_load_monitor_b64:
1706 case Intrinsic::amdgcn_flat_load_monitor_b128:
1707 case Intrinsic::amdgcn_global_load_monitor_b32:
1708 case Intrinsic::amdgcn_global_load_monitor_b64:
1709 case Intrinsic::amdgcn_global_load_monitor_b128: {
1710 Info.opc = ISD::INTRINSIC_W_CHAIN;
1711 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1712 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1713 Info.align.reset();
1714 Info.flags = MachineMemOperand::MOLoad;
1715 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 1);
1716 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 2);
1717 Infos.push_back(Elt: Info);
1718 return;
1719 }
1720 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
1721 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
1722 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
1723 Info.opc = ISD::INTRINSIC_W_CHAIN;
1724 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1725 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1726 Info.align.reset();
1727 Info.flags = (MachineMemOperand::MOLoad | MOCooperative);
1728 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 1);
1729 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 2);
1730 Infos.push_back(Elt: Info);
1731 return;
1732 }
1733 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
1734 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
1735 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
1736 Info.opc = ISD::INTRINSIC_VOID;
1737 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1738 Info.ptrVal = CI.getArgOperand(i: 0);
1739 Info.align.reset();
1740 Info.flags = (MachineMemOperand::MOStore | MOCooperative);
1741 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 2);
1742 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 3);
1743 Infos.push_back(Elt: Info);
1744 return;
1745 }
1746 case Intrinsic::amdgcn_ds_gws_init:
1747 case Intrinsic::amdgcn_ds_gws_barrier:
1748 case Intrinsic::amdgcn_ds_gws_sema_v:
1749 case Intrinsic::amdgcn_ds_gws_sema_br:
1750 case Intrinsic::amdgcn_ds_gws_sema_p:
1751 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
1752 Info.opc = ISD::INTRINSIC_VOID;
1753
1754 const GCNTargetMachine &TM =
1755 static_cast<const GCNTargetMachine &>(getTargetMachine());
1756
1757 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1758 Info.ptrVal = MFI->getGWSPSV(TM);
1759
1760 // This is an abstract access, but we need to specify a type and size.
1761 Info.memVT = MVT::i32;
1762 Info.size = 4;
1763 Info.align = Align(4);
1764
1765 if (IntrID == Intrinsic::amdgcn_ds_gws_barrier)
1766 Info.flags = Flags | MachineMemOperand::MOLoad;
1767 else
1768 Info.flags = Flags | MachineMemOperand::MOStore;
1769 Infos.push_back(Elt: Info);
1770 return;
1771 }
1772 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1773 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1774 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1775 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1776 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1777 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1778 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1779 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
1780 // Entry 0: Load from source (global/flat).
1781 Info.opc = ISD::INTRINSIC_VOID;
1782 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1783 Info.ptrVal = CI.getArgOperand(i: 0); // Global pointer
1784 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getSExtValue();
1785 Info.flags = Flags | MachineMemOperand::MOLoad;
1786 Infos.push_back(Elt: Info);
1787
1788 // Entry 1: Store to LDS (same offset).
1789 Info.flags = Flags | MachineMemOperand::MOStore;
1790 Info.ptrVal = CI.getArgOperand(i: 1); // LDS pointer
1791 Infos.push_back(Elt: Info);
1792 return;
1793 }
1794 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1795 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1796 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1797 case Intrinsic::amdgcn_global_store_async_from_lds_b128: {
1798 // Entry 0: Load from LDS.
1799 Info.opc = ISD::INTRINSIC_VOID;
1800 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1801 Info.ptrVal = CI.getArgOperand(i: 1); // LDS pointer
1802 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getSExtValue();
1803 Info.flags = Flags | MachineMemOperand::MOLoad;
1804 Infos.push_back(Elt: Info);
1805
1806 // Entry 1: Store to global (same offset).
1807 Info.flags = Flags | MachineMemOperand::MOStore;
1808 Info.ptrVal = CI.getArgOperand(i: 0); // Global pointer
1809 Infos.push_back(Elt: Info);
1810 return;
1811 }
1812 case Intrinsic::amdgcn_av_load_b128:
1813 case Intrinsic::amdgcn_av_store_b128: {
1814 bool IsStore = IntrID == Intrinsic::amdgcn_av_store_b128;
1815 Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
1816 Info.memVT = MVT::v4i32;
1817 Info.ptrVal = CI.getArgOperand(i: 0);
1818 Info.align = Align(16);
1819 Info.flags |=
1820 IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
1821 // Pretend to be atomic so that SIMemoryLegalizer::expandStore sets cache
1822 // flags appropriately.
1823 Info.order = AtomicOrdering::Monotonic;
1824
1825 LLVMContext &Ctx = CI.getContext();
1826 unsigned ScopeIdx = CI.arg_size() - 1;
1827 MDNode *ScopeMD = cast<MDNode>(
1828 Val: cast<MetadataAsValue>(Val: CI.getArgOperand(i: ScopeIdx))->getMetadata());
1829 StringRef Scope = cast<MDString>(Val: ScopeMD->getOperand(I: 0))->getString();
1830 Info.ssid = Ctx.getOrInsertSyncScopeID(SSN: Scope);
1831 Infos.push_back(Elt: Info);
1832 return;
1833 }
1834 case Intrinsic::amdgcn_load_to_lds:
1835 case Intrinsic::amdgcn_load_async_to_lds:
1836 case Intrinsic::amdgcn_global_load_lds:
1837 case Intrinsic::amdgcn_global_load_async_lds: {
1838 unsigned Width = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getZExtValue();
1839 auto *Aux = cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1));
1840 bool IsVolatile = Aux->getZExtValue() & AMDGPU::CPol::VOLATILE;
1841 if (IsVolatile)
1842 Flags |= MachineMemOperand::MOVolatile;
1843
1844 // Entry 0: Load from source (global/flat).
1845 Info.opc = ISD::INTRINSIC_VOID;
1846 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: Width * 8);
1847 Info.ptrVal = CI.getArgOperand(i: 0); // Source pointer
1848 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 3))->getSExtValue();
1849 Info.flags = Flags | MachineMemOperand::MOLoad;
1850 Infos.push_back(Elt: Info);
1851
1852 // Entry 1: Store to LDS.
1853 // Same offset from the instruction, but an additional per-lane offset is
1854 // added. Represent that using a wider memory type.
1855 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(),
1856 BitWidth: Width * 8 * Subtarget->getWavefrontSize());
1857 Info.ptrVal = CI.getArgOperand(i: 1); // LDS destination pointer
1858 Info.flags = Flags | MachineMemOperand::MOStore;
1859 Infos.push_back(Elt: Info);
1860 return;
1861 }
1862 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
1863 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
1864 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
1865 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn: {
1866 Info.opc = ISD::INTRINSIC_W_CHAIN;
1867
1868 const GCNTargetMachine &TM =
1869 static_cast<const GCNTargetMachine &>(getTargetMachine());
1870
1871 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1872 Info.ptrVal = MFI->getGWSPSV(TM);
1873
1874 // This is an abstract access, but we need to specify a type and size.
1875 Info.memVT = MVT::i32;
1876 Info.size = 4;
1877 Info.align = Align(4);
1878
1879 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1880 Infos.push_back(Elt: Info);
1881 return;
1882 }
1883 case Intrinsic::amdgcn_s_prefetch_data:
1884 case Intrinsic::amdgcn_s_prefetch_inst:
1885 case Intrinsic::amdgcn_flat_prefetch:
1886 case Intrinsic::amdgcn_global_prefetch: {
1887 Info.opc = ISD::INTRINSIC_VOID;
1888 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: 8);
1889 Info.ptrVal = CI.getArgOperand(i: 0);
1890 Info.flags = Flags | MachineMemOperand::MOLoad;
1891 Infos.push_back(Elt: Info);
1892 return;
1893 }
1894 default:
1895 return;
1896 }
1897}
1898
1899void SITargetLowering::CollectTargetIntrinsicOperands(
1900 const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
1901 switch (cast<IntrinsicInst>(Val: I).getIntrinsicID()) {
1902 case Intrinsic::amdgcn_addrspacecast_nonnull: {
1903 // The DAG's ValueType loses the addrspaces.
1904 // Add them as 2 extra Constant operands "from" and "to".
1905 unsigned SrcAS = I.getOperand(i_nocapture: 0)->getType()->getPointerAddressSpace();
1906 unsigned DstAS = I.getType()->getPointerAddressSpace();
1907 Ops.push_back(Elt: DAG.getTargetConstant(Val: SrcAS, DL: SDLoc(), VT: MVT::i32));
1908 Ops.push_back(Elt: DAG.getTargetConstant(Val: DstAS, DL: SDLoc(), VT: MVT::i32));
1909 break;
1910 }
1911 default:
1912 break;
1913 }
1914}
1915
1916bool SITargetLowering::getAddrModeArguments(const IntrinsicInst *II,
1917 SmallVectorImpl<Value *> &Ops,
1918 Type *&AccessTy) const {
1919 Value *Ptr = nullptr;
1920 switch (II->getIntrinsicID()) {
1921 case Intrinsic::amdgcn_cluster_load_b128:
1922 case Intrinsic::amdgcn_cluster_load_b64:
1923 case Intrinsic::amdgcn_cluster_load_b32:
1924 case Intrinsic::amdgcn_ds_append:
1925 case Intrinsic::amdgcn_ds_consume:
1926 case Intrinsic::amdgcn_ds_load_tr8_b64:
1927 case Intrinsic::amdgcn_ds_load_tr16_b128:
1928 case Intrinsic::amdgcn_ds_load_tr4_b64:
1929 case Intrinsic::amdgcn_ds_load_tr6_b96:
1930 case Intrinsic::amdgcn_ds_read_tr4_b64:
1931 case Intrinsic::amdgcn_ds_read_tr6_b96:
1932 case Intrinsic::amdgcn_ds_read_tr8_b64:
1933 case Intrinsic::amdgcn_ds_read_tr16_b64:
1934 case Intrinsic::amdgcn_ds_ordered_add:
1935 case Intrinsic::amdgcn_ds_ordered_swap:
1936 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
1937 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
1938 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1939 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1940 case Intrinsic::amdgcn_global_atomic_fmax_num:
1941 case Intrinsic::amdgcn_global_atomic_fmin_num:
1942 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1943 case Intrinsic::amdgcn_global_load_tr_b64:
1944 case Intrinsic::amdgcn_global_load_tr_b128:
1945 case Intrinsic::amdgcn_global_load_tr4_b64:
1946 case Intrinsic::amdgcn_global_load_tr6_b96:
1947 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1948 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1949 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1950 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
1951 case Intrinsic::amdgcn_av_load_b128:
1952 case Intrinsic::amdgcn_av_store_b128:
1953 Ptr = II->getArgOperand(i: 0);
1954 break;
1955 case Intrinsic::amdgcn_load_to_lds:
1956 case Intrinsic::amdgcn_load_async_to_lds:
1957 case Intrinsic::amdgcn_global_load_lds:
1958 case Intrinsic::amdgcn_global_load_async_lds:
1959 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1960 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1961 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1962 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1963 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1964 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1965 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1966 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
1967 Ptr = II->getArgOperand(i: 1);
1968 break;
1969 default:
1970 return false;
1971 }
1972 AccessTy = II->getType();
1973 Ops.push_back(Elt: Ptr);
1974 return true;
1975}
1976
1977bool SITargetLowering::isLegalFlatAddressingMode(const AddrMode &AM,
1978 unsigned AddrSpace) const {
1979 if (!Subtarget->hasFlatInstOffsets()) {
1980 // Flat instructions do not have offsets, and only have the register
1981 // address.
1982 return AM.BaseOffs == 0 && AM.Scale == 0;
1983 }
1984
1985 using AMDGPU::FlatAddrSpace;
1986 FlatAddrSpace FlatVariant =
1987 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS ? FlatAddrSpace::FlatGlobal
1988 : AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ? FlatAddrSpace::FlatScratch
1989 : FlatAddrSpace::FLAT;
1990
1991 return AM.Scale == 0 &&
1992 (AM.BaseOffs == 0 || Subtarget->getInstrInfo()->isLegalFLATOffset(
1993 Offset: AM.BaseOffs, AddrSpace, FlatVariant));
1994}
1995
1996bool SITargetLowering::isLegalGlobalAddressingMode(const AddrMode &AM) const {
1997 if (Subtarget->hasFlatGlobalInsts())
1998 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS);
1999
2000 if (!Subtarget->hasAddr64() || Subtarget->useFlatForGlobal()) {
2001 // Assume the we will use FLAT for all global memory accesses
2002 // on VI.
2003 // FIXME: This assumption is currently wrong. On VI we still use
2004 // MUBUF instructions for the r + i addressing mode. As currently
2005 // implemented, the MUBUF instructions only work on buffer < 4GB.
2006 // It may be possible to support > 4GB buffers with MUBUF instructions,
2007 // by setting the stride value in the resource descriptor which would
2008 // increase the size limit to (stride * 4GB). However, this is risky,
2009 // because it has never been validated.
2010 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::FLAT_ADDRESS);
2011 }
2012
2013 return isLegalMUBUFAddressingMode(AM);
2014}
2015
2016bool SITargetLowering::isLegalMUBUFAddressingMode(const AddrMode &AM) const {
2017 // MUBUF / MTBUF instructions have a 12-bit unsigned byte offset, and
2018 // additionally can do r + r + i with addr64. 32-bit has more addressing
2019 // mode options. Depending on the resource constant, it can also do
2020 // (i64 r0) + (i32 r1) * (i14 i).
2021 //
2022 // Private arrays end up using a scratch buffer most of the time, so also
2023 // assume those use MUBUF instructions. Scratch loads / stores are currently
2024 // implemented as mubuf instructions with offen bit set, so slightly
2025 // different than the normal addr64.
2026 const SIInstrInfo *TII = Subtarget->getInstrInfo();
2027 if (!TII->isLegalMUBUFImmOffset(Imm: AM.BaseOffs))
2028 return false;
2029
2030 // FIXME: Since we can split immediate into soffset and immediate offset,
2031 // would it make sense to allow any immediate?
2032
2033 switch (AM.Scale) {
2034 case 0: // r + i or just i, depending on HasBaseReg.
2035 return true;
2036 case 1:
2037 return true; // We have r + r or r + i.
2038 case 2:
2039 if (AM.HasBaseReg) {
2040 // Reject 2 * r + r.
2041 return false;
2042 }
2043
2044 // Allow 2 * r as r + r
2045 // Or 2 * r + i is allowed as r + r + i.
2046 return true;
2047 default: // Don't allow n * r
2048 return false;
2049 }
2050}
2051
2052bool SITargetLowering::isLegalAddressingMode(const DataLayout &DL,
2053 const AddrMode &AM, Type *Ty,
2054 unsigned AS,
2055 Instruction *I) const {
2056 // No global is ever allowed as a base.
2057 if (AM.BaseGV)
2058 return false;
2059
2060 if (AS == AMDGPUAS::GLOBAL_ADDRESS)
2061 return isLegalGlobalAddressingMode(AM);
2062
2063 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
2064 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
2065 AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE ||
2066 AS == AMDGPUAS::BUFFER_STRIDED_POINTER) {
2067 // If the offset isn't a multiple of 4, it probably isn't going to be
2068 // correctly aligned.
2069 // FIXME: Can we get the real alignment here?
2070 if (AM.BaseOffs % 4 != 0)
2071 return isLegalMUBUFAddressingMode(AM);
2072
2073 if (!Subtarget->hasScalarSubwordLoads()) {
2074 // There are no SMRD extloads, so if we have to do a small type access we
2075 // will use a MUBUF load.
2076 // FIXME?: We also need to do this if unaligned, but we don't know the
2077 // alignment here.
2078 if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4)
2079 return isLegalGlobalAddressingMode(AM);
2080 }
2081
2082 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) {
2083 // SMRD instructions have an 8-bit, dword offset on SI.
2084 if (!isUInt<8>(x: AM.BaseOffs / 4))
2085 return false;
2086 } else if (Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS) {
2087 // On CI+, this can also be a 32-bit literal constant offset. If it fits
2088 // in 8-bits, it can use a smaller encoding.
2089 if (!isUInt<32>(x: AM.BaseOffs / 4))
2090 return false;
2091 } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX9) {
2092 // On VI, these use the SMEM format and the offset is 20-bit in bytes.
2093 if (!isUInt<20>(x: AM.BaseOffs))
2094 return false;
2095 } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX12) {
2096 // On GFX9 the offset is signed 21-bit in bytes (but must not be negative
2097 // for S_BUFFER_* instructions).
2098 if (!isInt<21>(x: AM.BaseOffs))
2099 return false;
2100 } else {
2101 // On GFX12, all offsets are signed 24-bit in bytes.
2102 if (!isInt<24>(x: AM.BaseOffs))
2103 return false;
2104 }
2105
2106 if ((AS == AMDGPUAS::CONSTANT_ADDRESS ||
2107 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
2108 AM.BaseOffs < 0) {
2109 // Scalar (non-buffer) loads can only use a negative offset if
2110 // soffset+offset is non-negative. Since the compiler can only prove that
2111 // in a few special cases, it is safer to claim that negative offsets are
2112 // not supported.
2113 return false;
2114 }
2115
2116 if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
2117 return true;
2118
2119 if (AM.Scale == 1 && AM.HasBaseReg)
2120 return true;
2121
2122 return false;
2123 }
2124
2125 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
2126 return Subtarget->hasFlatScratchEnabled()
2127 ? isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS)
2128 : isLegalMUBUFAddressingMode(AM);
2129
2130 if (AS == AMDGPUAS::LOCAL_ADDRESS ||
2131 (AS == AMDGPUAS::REGION_ADDRESS && Subtarget->hasGDS())) {
2132 // Basic, single offset DS instructions allow a 16-bit unsigned immediate
2133 // field.
2134 // XXX - If doing a 4-byte aligned 8-byte type access, we effectively have
2135 // an 8-bit dword offset but we don't know the alignment here.
2136 if (!isUInt<16>(x: AM.BaseOffs))
2137 return false;
2138
2139 if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
2140 return true;
2141
2142 if (AM.Scale == 1 && AM.HasBaseReg)
2143 return true;
2144
2145 return false;
2146 }
2147
2148 if (AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::UNKNOWN_ADDRESS_SPACE) {
2149 // For an unknown address space, this usually means that this is for some
2150 // reason being used for pure arithmetic, and not based on some addressing
2151 // computation. We don't have instructions that compute pointers with any
2152 // addressing modes, so treat them as having no offset like flat
2153 // instructions.
2154 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::FLAT_ADDRESS);
2155 }
2156
2157 // Assume a user alias of global for unknown address spaces.
2158 return isLegalGlobalAddressingMode(AM);
2159}
2160
2161bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
2162 const MachineFunction &MF) const {
2163 if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS)
2164 return (MemVT.getSizeInBits() <= 4 * 32);
2165 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
2166 unsigned MaxPrivateBits = 8 * getSubtarget()->getMaxPrivateElementSize();
2167 return (MemVT.getSizeInBits() <= MaxPrivateBits);
2168 }
2169 if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS)
2170 return (MemVT.getSizeInBits() <= 2 * 32);
2171 return true;
2172}
2173
2174bool SITargetLowering::allowsMisalignedMemoryAccessesImpl(
2175 unsigned Size, unsigned AddrSpace, Align Alignment,
2176 MachineMemOperand::Flags Flags, unsigned *IsFast) const {
2177 if (IsFast)
2178 *IsFast = 0;
2179
2180 if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
2181 AddrSpace == AMDGPUAS::REGION_ADDRESS) {
2182 // Check if alignment requirements for ds_read/write instructions are
2183 // disabled.
2184 if (!Subtarget->hasUnalignedDSAccessEnabled() && Alignment < Align(4))
2185 return false;
2186
2187 Align RequiredAlignment(
2188 PowerOf2Ceil(A: divideCeil(Numerator: Size, Denominator: 8))); // Natural alignment.
2189 if (Subtarget->hasLDSMisalignedBugInWGPMode() && Size > 32 &&
2190 Alignment < RequiredAlignment)
2191 return false;
2192
2193 // Either, the alignment requirements are "enabled", or there is an
2194 // unaligned LDS access related hardware bug though alignment requirements
2195 // are "disabled". In either case, we need to check for proper alignment
2196 // requirements.
2197 //
2198 switch (Size) {
2199 case 64:
2200 // SI has a hardware bug in the LDS / GDS bounds checking: if the base
2201 // address is negative, then the instruction is incorrectly treated as
2202 // out-of-bounds even if base + offsets is in bounds. Split vectorized
2203 // loads here to avoid emitting ds_read2_b32. We may re-combine the
2204 // load later in the SILoadStoreOptimizer.
2205 if (!Subtarget->hasUsableDSOffset() && Alignment < Align(8))
2206 return false;
2207
2208 // 8 byte accessing via ds_read/write_b64 require 8-byte alignment, but we
2209 // can do a 4 byte aligned, 8 byte access in a single operation using
2210 // ds_read2/write2_b32 with adjacent offsets.
2211 RequiredAlignment = Align(4);
2212
2213 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2214 // We will either select ds_read_b64/ds_write_b64 or ds_read2_b32/
2215 // ds_write2_b32 depending on the alignment. In either case with either
2216 // alignment there is no faster way of doing this.
2217
2218 // The numbers returned here and below are not additive, it is a 'speed
2219 // rank'. They are just meant to be compared to decide if a certain way
2220 // of lowering an operation is faster than another. For that purpose
2221 // naturally aligned operation gets it bitsize to indicate that "it
2222 // operates with a speed comparable to N-bit wide load". With the full
2223 // alignment ds128 is slower than ds96 for example. If underaligned it
2224 // is comparable to a speed of a single dword access, which would then
2225 // mean 32 < 128 and it is faster to issue a wide load regardless.
2226 // 1 is simply "slow, don't do it". I.e. comparing an aligned load to a
2227 // wider load which will not be aligned anymore the latter is slower.
2228 if (IsFast)
2229 *IsFast = (Alignment >= RequiredAlignment) ? 64
2230 : (Alignment < Align(4)) ? 32
2231 : 1;
2232 return true;
2233 }
2234
2235 break;
2236 case 96:
2237 if (!Subtarget->hasDS96AndDS128())
2238 return false;
2239
2240 // 12 byte accessing via ds_read/write_b96 require 16-byte alignment on
2241 // gfx8 and older.
2242
2243 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2244 // Naturally aligned access is fastest. However, also report it is Fast
2245 // if memory is aligned less than DWORD. A narrow load or store will be
2246 // be equally slow as a single ds_read_b96/ds_write_b96, but there will
2247 // be more of them, so overall we will pay less penalty issuing a single
2248 // instruction.
2249
2250 // See comment on the values above.
2251 if (IsFast)
2252 *IsFast = (Alignment >= RequiredAlignment) ? 96
2253 : (Alignment < Align(4)) ? 32
2254 : 1;
2255 return true;
2256 }
2257
2258 break;
2259 case 128:
2260 if (!Subtarget->hasDS96AndDS128() || !Subtarget->useDS128())
2261 return false;
2262
2263 // 16 byte accessing via ds_read/write_b128 require 16-byte alignment on
2264 // gfx8 and older, but we can do a 8 byte aligned, 16 byte access in a
2265 // single operation using ds_read2/write2_b64.
2266 RequiredAlignment = Align(8);
2267
2268 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2269 // Naturally aligned access is fastest. However, also report it is Fast
2270 // if memory is aligned less than DWORD. A narrow load or store will be
2271 // be equally slow as a single ds_read_b128/ds_write_b128, but there
2272 // will be more of them, so overall we will pay less penalty issuing a
2273 // single instruction.
2274
2275 // See comment on the values above.
2276 if (IsFast)
2277 *IsFast = (Alignment >= RequiredAlignment) ? 128
2278 : (Alignment < Align(4)) ? 32
2279 : 1;
2280 return true;
2281 }
2282
2283 break;
2284 default:
2285 if (Size > 32)
2286 return false;
2287
2288 break;
2289 }
2290
2291 // See comment on the values above.
2292 // Note that we have a single-dword or sub-dword here, so if underaligned
2293 // it is a slowest possible access, hence returned value is 0.
2294 if (IsFast)
2295 *IsFast = (Alignment >= RequiredAlignment) ? Size : 0;
2296
2297 return Alignment >= RequiredAlignment ||
2298 Subtarget->hasUnalignedDSAccessEnabled();
2299 }
2300
2301 // FIXME: We have to be conservative here and assume that flat operations
2302 // will access scratch. If we had access to the IR function, then we
2303 // could determine if any private memory was used in the function.
2304 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ||
2305 AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
2306 bool AlignedBy4 = Alignment >= Align(4);
2307 if (Subtarget->hasUnalignedScratchAccessEnabled()) {
2308 if (IsFast)
2309 *IsFast = AlignedBy4 ? Size : 1;
2310 return true;
2311 }
2312
2313 if (IsFast)
2314 *IsFast = AlignedBy4;
2315
2316 return AlignedBy4;
2317 }
2318
2319 // So long as they are correct, wide global memory operations perform better
2320 // than multiple smaller memory ops -- even when misaligned
2321 if (AMDGPU::isExtendedGlobalAddrSpace(AS: AddrSpace)) {
2322 if (IsFast)
2323 *IsFast = Size;
2324
2325 return Alignment >= Align(4) ||
2326 Subtarget->hasUnalignedBufferAccessEnabled();
2327 }
2328
2329 // Ensure robust out-of-bounds guarantees for buffer accesses are met when the
2330 // "amdgpu.buffer.oob.mode" module flag has not enabled relaxed untyped-buffer
2331 // OOB semantics. Normally hardware will ensure proper
2332 // out-of-bounds behavior, but in the edge case where an access starts
2333 // out-of-bounds and then enters in-bounds, the entire access would be treated
2334 // as out-of-bounds. Prevent misaligned memory accesses by requiring the
2335 // natural alignment of buffer accesses.
2336 if (AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
2337 AddrSpace == AMDGPUAS::BUFFER_RESOURCE ||
2338 AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) {
2339 if (!Subtarget->hasRelaxedBufferOOBMode() &&
2340 Alignment < Align(PowerOf2Ceil(A: divideCeil(Numerator: Size, Denominator: 8))))
2341 return false;
2342 }
2343
2344 // Smaller than dword value must be aligned.
2345 if (Size < 32)
2346 return false;
2347
2348 // 8.1.6 - For Dword or larger reads or writes, the two LSBs of the
2349 // byte-address are ignored, thus forcing Dword alignment.
2350 // This applies to private, global, and constant memory.
2351 if (IsFast)
2352 *IsFast = 1;
2353
2354 return Size >= 32 && Alignment >= Align(4);
2355}
2356
2357bool SITargetLowering::allowsMisalignedMemoryAccesses(
2358 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
2359 unsigned *IsFast) const {
2360 return allowsMisalignedMemoryAccessesImpl(Size: VT.getSizeInBits(), AddrSpace,
2361 Alignment, Flags, IsFast);
2362}
2363
2364EVT SITargetLowering::getOptimalMemOpType(
2365 LLVMContext &Context, const MemOp &Op,
2366 const AttributeList &FuncAttributes) const {
2367 // FIXME: Should account for address space here.
2368
2369 // The default fallback uses the private pointer size as a guess for a type to
2370 // use. Make sure we switch these to 64-bit accesses.
2371
2372 if (Op.size() >= 16 &&
2373 Op.isDstAligned(AlignCheck: Align(4))) // XXX: Should only do for global
2374 return MVT::v4i32;
2375
2376 if (Op.size() >= 8 && Op.isDstAligned(AlignCheck: Align(4)))
2377 return MVT::v2i32;
2378
2379 // Use the default.
2380 return MVT::Other;
2381}
2382
2383bool SITargetLowering::isMemOpHasNoClobberedMemOperand(const SDNode *N) const {
2384 const MemSDNode *MemNode = cast<MemSDNode>(Val: N);
2385 return MemNode->getMemOperand()->getFlags() & MONoClobber;
2386}
2387
2388bool SITargetLowering::isNonGlobalAddrSpace(unsigned AS) {
2389 return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS ||
2390 AS == AMDGPUAS::PRIVATE_ADDRESS;
2391}
2392
2393bool SITargetLowering::isFreeAddrSpaceCast(unsigned SrcAS,
2394 unsigned DestAS) const {
2395 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
2396 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
2397 Subtarget->hasGloballyAddressableScratch()) {
2398 // Flat -> private requires subtracting src_flat_scratch_base_lo.
2399 return false;
2400 }
2401
2402 // Flat -> private/local is a simple truncate.
2403 // Flat -> global is no-op
2404 return true;
2405 }
2406
2407 const GCNTargetMachine &TM =
2408 static_cast<const GCNTargetMachine &>(getTargetMachine());
2409 return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
2410}
2411
2412TargetLoweringBase::LegalizeTypeAction
2413SITargetLowering::getPreferredVectorAction(MVT VT) const {
2414 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
2415 VT.getScalarType().bitsLE(VT: MVT::i16))
2416 return VT.isPow2VectorType() ? TypeSplitVector : TypeWidenVector;
2417 return TargetLoweringBase::getPreferredVectorAction(VT);
2418}
2419
2420bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
2421 Type *Ty) const {
2422 // FIXME: Could be smarter if called for vector constants.
2423 return true;
2424}
2425
2426TargetLowering::ExtractSubvectorCost
2427SITargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
2428 unsigned Index) const {
2429 if (!isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: ResVT))
2430 return ExtractSubvectorCost::Expensive;
2431
2432 // TODO: Add more cases that are cheap.
2433 if (Index == 0)
2434 return ExtractSubvectorCost::Free;
2435 return ExtractSubvectorCost::Expensive;
2436}
2437
2438bool SITargetLowering::isExtractVecEltCheap(EVT VT, unsigned Index) const {
2439 // TODO: This should be more aggressive, particular for 16-bit element
2440 // vectors. However there are some mixed improvements and regressions.
2441 EVT EltTy = VT.getVectorElementType();
2442 unsigned MinAlign = Subtarget->useRealTrue16Insts() ? 16 : 32;
2443 return EltTy.getSizeInBits() % MinAlign == 0;
2444}
2445
2446bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT) const {
2447 if (Subtarget->has16BitInsts() && VT == MVT::i16) {
2448 switch (Op) {
2449 case ISD::LOAD:
2450 case ISD::STORE:
2451 return true;
2452 default:
2453 return false;
2454 }
2455 }
2456
2457 // SimplifySetCC uses this function to determine whether or not it should
2458 // create setcc with i1 operands. We don't have instructions for i1 setcc.
2459 if (VT == MVT::i1 && Op == ISD::SETCC)
2460 return false;
2461
2462 return TargetLowering::isTypeDesirableForOp(Op, VT);
2463}
2464
2465MachinePointerInfo
2466SITargetLowering::getKernargSegmentPtrInfo(MachineFunction &MF) const {
2467 // This isn't really a constant pool but close enough.
2468 MachinePointerInfo PtrInfo(MF.getPSVManager().getConstantPool());
2469 PtrInfo.AddrSpace = AMDGPUAS::CONSTANT_ADDRESS;
2470 return PtrInfo;
2471}
2472
2473SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
2474 const SDLoc &SL,
2475 SDValue Chain,
2476 uint64_t Offset) const {
2477 const DataLayout &DL = DAG.getDataLayout();
2478 MachineFunction &MF = DAG.getMachineFunction();
2479 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
2480 MVT PtrVT = getPointerTy(DL, AS: AMDGPUAS::CONSTANT_ADDRESS);
2481
2482 auto [InputPtrReg, RC, ArgTy] =
2483 Info->getPreloadedValue(Value: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
2484
2485 // We may not have the kernarg segment argument if we have no kernel
2486 // arguments.
2487 if (!InputPtrReg)
2488 return DAG.getConstant(Val: Offset, DL: SL, VT: PtrVT);
2489
2490 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
2491 SDValue BasePtr = DAG.getCopyFromReg(
2492 Chain, dl: SL, Reg: MRI.getLiveInVirtReg(PReg: InputPtrReg->getRegister()), VT: PtrVT);
2493
2494 return DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
2495}
2496
2497SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG,
2498 const SDLoc &SL) const {
2499 uint64_t Offset =
2500 getImplicitParameterOffset(MF: DAG.getMachineFunction(), Param: FIRST_IMPLICIT);
2501 return lowerKernArgParameterPtr(DAG, SL, Chain: DAG.getEntryNode(), Offset);
2502}
2503
2504SDValue SITargetLowering::getLDSKernelId(SelectionDAG &DAG,
2505 const SDLoc &SL) const {
2506
2507 Function &F = DAG.getMachineFunction().getFunction();
2508 std::optional<uint32_t> KnownSize =
2509 AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F);
2510 if (KnownSize.has_value())
2511 return DAG.getConstant(Val: *KnownSize, DL: SL, VT: MVT::i32);
2512 return SDValue();
2513}
2514
2515SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
2516 const SDLoc &SL, SDValue Val,
2517 bool Signed,
2518 const ISD::InputArg *Arg) const {
2519 // First, if it is a widened vector, narrow it.
2520 if (VT.isVector() &&
2521 VT.getVectorNumElements() != MemVT.getVectorNumElements()) {
2522 EVT NarrowedVT =
2523 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MemVT.getVectorElementType(),
2524 NumElements: VT.getVectorNumElements());
2525 Val = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: NarrowedVT, N1: Val,
2526 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
2527 }
2528
2529 // Then convert the vector elements or scalar value.
2530 if (Arg && (Arg->Flags.isSExt() || Arg->Flags.isZExt()) && VT.bitsLT(VT: MemVT)) {
2531 unsigned Opc = Arg->Flags.isZExt() ? ISD::AssertZext : ISD::AssertSext;
2532 Val = DAG.getNode(Opcode: Opc, DL: SL, VT: MemVT, N1: Val, N2: DAG.getValueType(VT));
2533 }
2534
2535 if (MemVT.isFloatingPoint()) {
2536 if (VT.isFloatingPoint()) {
2537 Val = getFPExtOrFPRound(DAG, Op: Val, DL: SL, VT);
2538 } else {
2539 assert(!MemVT.isVector());
2540 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
2541 SDValue Cast = DAG.getBitcast(VT: IntVT, V: Val);
2542 Val = DAG.getAnyExtOrTrunc(Op: Cast, DL: SL, VT);
2543 }
2544 } else if (Signed)
2545 Val = DAG.getSExtOrTrunc(Op: Val, DL: SL, VT);
2546 else
2547 Val = DAG.getZExtOrTrunc(Op: Val, DL: SL, VT);
2548
2549 return Val;
2550}
2551
2552SDValue SITargetLowering::lowerKernargMemParameter(
2553 SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain,
2554 uint64_t Offset, Align Alignment, bool Signed,
2555 const ISD::InputArg *Arg) const {
2556
2557 MachinePointerInfo PtrInfo =
2558 getKernargSegmentPtrInfo(MF&: DAG.getMachineFunction());
2559
2560 // Try to avoid using an extload by loading earlier than the argument address,
2561 // and extracting the relevant bits. The load should hopefully be merged with
2562 // the previous argument.
2563 if (MemVT.getStoreSize() < 4 && Alignment < 4) {
2564 // TODO: Handle align < 4 and size >= 4 (can happen with packed structs).
2565 int64_t AlignDownOffset = alignDown(Value: Offset, Align: 4);
2566 int64_t OffsetDiff = Offset - AlignDownOffset;
2567
2568 EVT IntVT = MemVT.changeTypeToInteger();
2569
2570 // TODO: If we passed in the base kernel offset we could have a better
2571 // alignment than 4, but we don't really need it.
2572 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset: AlignDownOffset);
2573 SDValue Load = DAG.getLoad(VT: MVT::i32, dl: SL, Chain, Ptr,
2574 PtrInfo: PtrInfo.getWithOffset(O: AlignDownOffset), Alignment: Align(4),
2575 MMOFlags: MachineMemOperand::MODereferenceable |
2576 MachineMemOperand::MOInvariant);
2577
2578 SDValue ShiftAmt = DAG.getConstant(Val: OffsetDiff * 8, DL: SL, VT: MVT::i32);
2579 SDValue Extract = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: Load, N2: ShiftAmt);
2580
2581 SDValue ArgVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: IntVT, Operand: Extract);
2582 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MemVT, Operand: ArgVal);
2583 ArgVal = convertArgType(DAG, VT, MemVT, SL, Val: ArgVal, Signed, Arg);
2584
2585 return DAG.getMergeValues(Ops: {ArgVal, Load.getValue(R: 1)}, dl: SL);
2586 }
2587
2588 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
2589 SDValue Load = DAG.getLoad(
2590 VT: MemVT, dl: SL, Chain, Ptr, PtrInfo: PtrInfo.getWithOffset(O: Offset), Alignment,
2591 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
2592
2593 SDValue Val = convertArgType(DAG, VT, MemVT, SL, Val: Load, Signed, Arg);
2594 return DAG.getMergeValues(Ops: {Val, Load.getValue(R: 1)}, dl: SL);
2595}
2596
2597/// Coerce an argument which was passed in a different ABI type to the original
2598/// expected value type.
2599SDValue SITargetLowering::convertABITypeToValueType(SelectionDAG &DAG,
2600 SDValue Val,
2601 CCValAssign &VA,
2602 const SDLoc &SL) const {
2603 EVT ValVT = VA.getValVT();
2604
2605 // If this is an 8 or 16-bit value, it is really passed promoted
2606 // to 32 bits. Insert an assert[sz]ext to capture this, then
2607 // truncate to the right size.
2608 switch (VA.getLocInfo()) {
2609 case CCValAssign::Full:
2610 return Val;
2611 case CCValAssign::BCvt:
2612 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ValVT, Operand: Val);
2613 case CCValAssign::SExt:
2614 Val = DAG.getNode(Opcode: ISD::AssertSext, DL: SL, VT: VA.getLocVT(), N1: Val,
2615 N2: DAG.getValueType(ValVT));
2616 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2617 case CCValAssign::ZExt:
2618 Val = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: VA.getLocVT(), N1: Val,
2619 N2: DAG.getValueType(ValVT));
2620 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2621 case CCValAssign::AExt:
2622 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2623 default:
2624 llvm_unreachable("Unknown loc info!");
2625 }
2626}
2627
2628SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG,
2629 CCValAssign &VA, const SDLoc &SL,
2630 SDValue Chain,
2631 const ISD::InputArg &Arg) const {
2632 MachineFunction &MF = DAG.getMachineFunction();
2633 MachineFrameInfo &MFI = MF.getFrameInfo();
2634
2635 if (Arg.Flags.isByVal()) {
2636 unsigned Size = Arg.Flags.getByValSize();
2637 int FrameIdx = MFI.CreateFixedObject(Size, SPOffset: VA.getLocMemOffset(), IsImmutable: false);
2638 return DAG.getFrameIndex(FI: FrameIdx, VT: MVT::i32);
2639 }
2640
2641 unsigned ArgOffset = VA.getLocMemOffset();
2642 unsigned ArgSize = VA.getValVT().getStoreSize();
2643
2644 int FI = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: true);
2645
2646 // Create load nodes to retrieve arguments from the stack.
2647 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
2648
2649 // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
2650 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
2651 MVT MemVT = VA.getValVT();
2652
2653 switch (VA.getLocInfo()) {
2654 default:
2655 break;
2656 case CCValAssign::BCvt:
2657 MemVT = VA.getLocVT();
2658 break;
2659 case CCValAssign::SExt:
2660 ExtType = ISD::SEXTLOAD;
2661 break;
2662 case CCValAssign::ZExt:
2663 ExtType = ISD::ZEXTLOAD;
2664 break;
2665 case CCValAssign::AExt:
2666 ExtType = ISD::EXTLOAD;
2667 break;
2668 }
2669
2670 SDValue ArgValue = DAG.getExtLoad(
2671 ExtType, dl: SL, VT: VA.getLocVT(), Chain, Ptr: FIN,
2672 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), MemVT);
2673
2674 SDValue ConvertedVal = convertABITypeToValueType(DAG, Val: ArgValue, VA, SL);
2675 if (ConvertedVal == ArgValue)
2676 return ConvertedVal;
2677
2678 return DAG.getMergeValues(Ops: {ConvertedVal, ArgValue.getValue(R: 1)}, dl: SL);
2679}
2680
2681SDValue SITargetLowering::lowerWorkGroupId(
2682 SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
2683 AMDGPUFunctionArgInfo::PreloadedValue WorkGroupIdPV,
2684 AMDGPUFunctionArgInfo::PreloadedValue ClusterMaxIdPV,
2685 AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const {
2686 if (!Subtarget->hasClusters())
2687 return getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
2688
2689 // Clusters are supported. Return the global position in the grid. If clusters
2690 // are enabled, WorkGroupIdPV returns the cluster ID not the workgroup ID.
2691
2692 // WorkGroupIdXYZ = ClusterId == 0 ?
2693 // ClusterIdXYZ :
2694 // ClusterIdXYZ * (ClusterMaxIdXYZ + 1) + ClusterWorkGroupIdXYZ
2695 SDValue ClusterIdXYZ = getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
2696 SDLoc SL(ClusterIdXYZ);
2697 SDValue ClusterMaxIdXYZ = getPreloadedValue(DAG, MFI, VT, ClusterMaxIdPV);
2698 SDValue One = DAG.getConstant(Val: 1, DL: SL, VT);
2699 SDValue ClusterSizeXYZ = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ClusterMaxIdXYZ, N2: One);
2700 SDValue ClusterWorkGroupIdXYZ =
2701 getPreloadedValue(DAG, MFI, VT, ClusterWorkGroupIdPV);
2702 SDValue GlobalIdXYZ =
2703 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ClusterWorkGroupIdXYZ,
2704 N2: DAG.getNode(Opcode: ISD::MUL, DL: SL, VT, N1: ClusterIdXYZ, N2: ClusterSizeXYZ));
2705
2706 switch (MFI.getClusterDims().getKind()) {
2707 case AMDGPU::ClusterDimsAttr::Kind::FixedDims:
2708 case AMDGPU::ClusterDimsAttr::Kind::VariableDims:
2709 return GlobalIdXYZ;
2710 case AMDGPU::ClusterDimsAttr::Kind::NoCluster:
2711 return ClusterIdXYZ;
2712 case AMDGPU::ClusterDimsAttr::Kind::Unknown: {
2713 using namespace AMDGPU::Hwreg;
2714 SDValue ClusterIdField =
2715 DAG.getTargetConstant(Val: HwregEncoding::encode(Values: ID_IB_STS2, Values: 6, Values: 4), DL: SL, VT);
2716 SDNode *GetReg =
2717 DAG.getMachineNode(Opcode: AMDGPU::S_GETREG_B32_const, dl: SL, VT, Op1: ClusterIdField);
2718 SDValue ClusterId(GetReg, 0);
2719 SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT);
2720 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SL, VT, N1: ClusterId, N2: Zero, N3: ClusterIdXYZ,
2721 N4: GlobalIdXYZ, N5: DAG.getCondCode(Cond: ISD::SETEQ));
2722 }
2723 }
2724
2725 llvm_unreachable("nothing should reach here");
2726}
2727
2728SDValue SITargetLowering::getPreloadedValue(
2729 SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
2730 AMDGPUFunctionArgInfo::PreloadedValue PVID) const {
2731 const ArgDescriptor *Reg = nullptr;
2732 const TargetRegisterClass *RC = nullptr;
2733 LLT Ty;
2734
2735 CallingConv::ID CC = DAG.getMachineFunction().getFunction().getCallingConv();
2736 const ArgDescriptor WorkGroupIDX =
2737 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP9);
2738 // If GridZ is not programmed in an entry function then the hardware will set
2739 // it to all zeros, so there is no need to mask the GridY value in the low
2740 // order bits.
2741 const ArgDescriptor WorkGroupIDY = ArgDescriptor::createRegister(
2742 Reg: AMDGPU::TTMP7,
2743 Mask: AMDGPU::isEntryFunctionCC(CC) && !MFI.hasWorkGroupIDZ() ? ~0u : 0xFFFFu);
2744 const ArgDescriptor WorkGroupIDZ =
2745 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP7, Mask: 0xFFFF0000u);
2746 const ArgDescriptor ClusterWorkGroupIDX =
2747 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0000000Fu);
2748 const ArgDescriptor ClusterWorkGroupIDY =
2749 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x000000F0u);
2750 const ArgDescriptor ClusterWorkGroupIDZ =
2751 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x00000F00u);
2752 const ArgDescriptor ClusterWorkGroupMaxIDX =
2753 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0000F000u);
2754 const ArgDescriptor ClusterWorkGroupMaxIDY =
2755 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x000F0000u);
2756 const ArgDescriptor ClusterWorkGroupMaxIDZ =
2757 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x00F00000u);
2758 const ArgDescriptor ClusterWorkGroupMaxFlatID =
2759 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0F000000u);
2760
2761 auto LoadConstant = [&](unsigned N) {
2762 return DAG.getConstant(Val: N, DL: SDLoc(), VT);
2763 };
2764
2765 if (Subtarget->hasArchitectedSGPRs() &&
2766 (AMDGPU::isCompute(CC) || CC == CallingConv::AMDGPU_Gfx)) {
2767 AMDGPU::ClusterDimsAttr ClusterDims = MFI.getClusterDims();
2768 bool HasFixedDims = ClusterDims.isFixedDims();
2769
2770 switch (PVID) {
2771 case AMDGPUFunctionArgInfo::WORKGROUP_ID_X:
2772 Reg = &WorkGroupIDX;
2773 RC = &AMDGPU::SReg_32RegClass;
2774 Ty = LLT::scalar(SizeInBits: 32);
2775 break;
2776 case AMDGPUFunctionArgInfo::WORKGROUP_ID_Y:
2777 Reg = &WorkGroupIDY;
2778 RC = &AMDGPU::SReg_32RegClass;
2779 Ty = LLT::scalar(SizeInBits: 32);
2780 break;
2781 case AMDGPUFunctionArgInfo::WORKGROUP_ID_Z:
2782 Reg = &WorkGroupIDZ;
2783 RC = &AMDGPU::SReg_32RegClass;
2784 Ty = LLT::scalar(SizeInBits: 32);
2785 break;
2786 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X:
2787 if (HasFixedDims && ClusterDims.getDims()[0] == 1)
2788 return LoadConstant(0);
2789 Reg = &ClusterWorkGroupIDX;
2790 RC = &AMDGPU::SReg_32RegClass;
2791 Ty = LLT::scalar(SizeInBits: 32);
2792 break;
2793 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y:
2794 if (HasFixedDims && ClusterDims.getDims()[1] == 1)
2795 return LoadConstant(0);
2796 Reg = &ClusterWorkGroupIDY;
2797 RC = &AMDGPU::SReg_32RegClass;
2798 Ty = LLT::scalar(SizeInBits: 32);
2799 break;
2800 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z:
2801 if (HasFixedDims && ClusterDims.getDims()[2] == 1)
2802 return LoadConstant(0);
2803 Reg = &ClusterWorkGroupIDZ;
2804 RC = &AMDGPU::SReg_32RegClass;
2805 Ty = LLT::scalar(SizeInBits: 32);
2806 break;
2807 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X:
2808 if (HasFixedDims)
2809 return LoadConstant(ClusterDims.getDims()[0] - 1);
2810 Reg = &ClusterWorkGroupMaxIDX;
2811 RC = &AMDGPU::SReg_32RegClass;
2812 Ty = LLT::scalar(SizeInBits: 32);
2813 break;
2814 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y:
2815 if (HasFixedDims)
2816 return LoadConstant(ClusterDims.getDims()[1] - 1);
2817 Reg = &ClusterWorkGroupMaxIDY;
2818 RC = &AMDGPU::SReg_32RegClass;
2819 Ty = LLT::scalar(SizeInBits: 32);
2820 break;
2821 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z:
2822 if (HasFixedDims)
2823 return LoadConstant(ClusterDims.getDims()[2] - 1);
2824 Reg = &ClusterWorkGroupMaxIDZ;
2825 RC = &AMDGPU::SReg_32RegClass;
2826 Ty = LLT::scalar(SizeInBits: 32);
2827 break;
2828 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID:
2829 Reg = &ClusterWorkGroupMaxFlatID;
2830 RC = &AMDGPU::SReg_32RegClass;
2831 Ty = LLT::scalar(SizeInBits: 32);
2832 break;
2833 default:
2834 break;
2835 }
2836 }
2837
2838 if (!Reg)
2839 std::tie(args&: Reg, args&: RC, args&: Ty) = MFI.getPreloadedValue(Value: PVID);
2840 if (!Reg) {
2841 if (PVID == AMDGPUFunctionArgInfo::PreloadedValue::KERNARG_SEGMENT_PTR) {
2842 // It's possible for a kernarg intrinsic call to appear in a kernel with
2843 // no allocated segment, in which case we do not add the user sgpr
2844 // argument, so just return null.
2845 return DAG.getConstant(Val: 0, DL: SDLoc(), VT);
2846 }
2847
2848 // It's undefined behavior if a function marked with the amdgpu-no-*
2849 // attributes uses the corresponding intrinsic.
2850 return DAG.getPOISON(VT);
2851 }
2852
2853 return loadInputValue(DAG, RC, VT, SL: SDLoc(DAG.getEntryNode()), Arg: *Reg);
2854}
2855
2856static void processPSInputArgs(SmallVectorImpl<ISD::InputArg> &Splits,
2857 CallingConv::ID CallConv,
2858 ArrayRef<ISD::InputArg> Ins, BitVector &Skipped,
2859 FunctionType *FType,
2860 SIMachineFunctionInfo *Info) {
2861 for (unsigned I = 0, E = Ins.size(), PSInputNum = 0; I != E; ++I) {
2862 const ISD::InputArg *Arg = &Ins[I];
2863
2864 assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
2865 "vector type argument should have been split");
2866
2867 // First check if it's a PS input addr.
2868 if (CallConv == CallingConv::AMDGPU_PS && !Arg->Flags.isInReg() &&
2869 PSInputNum <= 15) {
2870 bool SkipArg = !Arg->Used && !Info->isPSInputAllocated(Index: PSInputNum);
2871
2872 // Inconveniently only the first part of the split is marked as isSplit,
2873 // so skip to the end. We only want to increment PSInputNum once for the
2874 // entire split argument.
2875 if (Arg->Flags.isSplit()) {
2876 while (!Arg->Flags.isSplitEnd()) {
2877 assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
2878 "unexpected vector split in ps argument type");
2879 if (!SkipArg)
2880 Splits.push_back(Elt: *Arg);
2881 Arg = &Ins[++I];
2882 }
2883 }
2884
2885 if (SkipArg) {
2886 // We can safely skip PS inputs.
2887 Skipped.set(Arg->getOrigArgIndex());
2888 ++PSInputNum;
2889 continue;
2890 }
2891
2892 Info->markPSInputAllocated(Index: PSInputNum);
2893 if (Arg->Used)
2894 Info->markPSInputEnabled(Index: PSInputNum);
2895
2896 ++PSInputNum;
2897 }
2898
2899 Splits.push_back(Elt: *Arg);
2900 }
2901}
2902
2903// Allocate special inputs passed in VGPRs.
2904void SITargetLowering::allocateSpecialEntryInputVGPRs(
2905 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
2906 SIMachineFunctionInfo &Info) const {
2907 const LLT S32 = LLT::scalar(SizeInBits: 32);
2908 MachineRegisterInfo &MRI = MF.getRegInfo();
2909
2910 if (Info.hasWorkItemIDX()) {
2911 Register Reg = AMDGPU::VGPR0;
2912 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: S32);
2913
2914 CCInfo.AllocateReg(Reg);
2915 unsigned Mask =
2916 (Subtarget->hasPackedTID() && Info.hasWorkItemIDY()) ? 0x3ff : ~0u;
2917 Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
2918 }
2919
2920 if (Info.hasWorkItemIDY()) {
2921 assert(Info.hasWorkItemIDX());
2922 if (Subtarget->hasPackedTID()) {
2923 Info.setWorkItemIDY(
2924 ArgDescriptor::createRegister(Reg: AMDGPU::VGPR0, Mask: 0x3ff << 10));
2925 } else {
2926 unsigned Reg = AMDGPU::VGPR1;
2927 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: S32);
2928
2929 CCInfo.AllocateReg(Reg);
2930 Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg));
2931 }
2932 }
2933
2934 if (Info.hasWorkItemIDZ()) {
2935 assert(Info.hasWorkItemIDX() && Info.hasWorkItemIDY());
2936 if (Subtarget->hasPackedTID()) {
2937 Info.setWorkItemIDZ(
2938 ArgDescriptor::createRegister(Reg: AMDGPU::VGPR0, Mask: 0x3ff << 20));
2939 } else {
2940 unsigned Reg = AMDGPU::VGPR2;
2941 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: S32);
2942
2943 CCInfo.AllocateReg(Reg);
2944 Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg));
2945 }
2946 }
2947}
2948
2949// Try to allocate a VGPR at the end of the argument list, or if no argument
2950// VGPRs are left allocating a stack slot.
2951// If \p Mask is given it indicates bitfield position in the register.
2952// If \p Arg is given use it with new ]p Mask instead of allocating new.
2953static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u,
2954 ArgDescriptor Arg = ArgDescriptor()) {
2955 if (Arg.isSet())
2956 return ArgDescriptor::createArg(Arg, Mask);
2957
2958 ArrayRef<MCPhysReg> ArgVGPRs = ArrayRef(AMDGPU::VGPR_32RegClass.begin(), 32);
2959 unsigned RegIdx = CCInfo.getFirstUnallocated(Regs: ArgVGPRs);
2960 if (RegIdx == ArgVGPRs.size()) {
2961 // Spill to stack required.
2962 int64_t Offset = CCInfo.AllocateStack(Size: 4, Alignment: Align(4));
2963
2964 return ArgDescriptor::createStack(Offset, Mask);
2965 }
2966
2967 unsigned Reg = ArgVGPRs[RegIdx];
2968 Reg = CCInfo.AllocateReg(Reg);
2969 assert(Reg != AMDGPU::NoRegister);
2970
2971 MachineFunction &MF = CCInfo.getMachineFunction();
2972 Register LiveInVReg = MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass);
2973 MF.getRegInfo().setType(VReg: LiveInVReg, Ty: LLT::scalar(SizeInBits: 32));
2974 return ArgDescriptor::createRegister(Reg, Mask);
2975}
2976
2977static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo,
2978 const TargetRegisterClass *RC,
2979 unsigned NumArgRegs) {
2980 ArrayRef<MCPhysReg> ArgSGPRs = ArrayRef(RC->begin(), 32);
2981 unsigned RegIdx = CCInfo.getFirstUnallocated(Regs: ArgSGPRs);
2982 if (RegIdx == ArgSGPRs.size())
2983 report_fatal_error(reason: "ran out of SGPRs for arguments");
2984
2985 unsigned Reg = ArgSGPRs[RegIdx];
2986 Reg = CCInfo.AllocateReg(Reg);
2987 assert(Reg != AMDGPU::NoRegister);
2988
2989 MachineFunction &MF = CCInfo.getMachineFunction();
2990 MF.addLiveIn(PReg: Reg, RC);
2991 return ArgDescriptor::createRegister(Reg);
2992}
2993
2994// If this has a fixed position, we still should allocate the register in the
2995// CCInfo state. Technically we could get away with this for values passed
2996// outside of the normal argument range.
2997static void allocateFixedSGPRInputImpl(CCState &CCInfo,
2998 const TargetRegisterClass *RC,
2999 MCRegister Reg) {
3000 Reg = CCInfo.AllocateReg(Reg);
3001 assert(Reg != AMDGPU::NoRegister);
3002 MachineFunction &MF = CCInfo.getMachineFunction();
3003 MF.addLiveIn(PReg: Reg, RC);
3004}
3005
3006static void allocateSGPR32Input(CCState &CCInfo, ArgDescriptor &Arg) {
3007 if (Arg) {
3008 allocateFixedSGPRInputImpl(CCInfo, RC: &AMDGPU::SGPR_32RegClass,
3009 Reg: Arg.getRegister());
3010 } else
3011 Arg = allocateSGPR32InputImpl(CCInfo, RC: &AMDGPU::SGPR_32RegClass, NumArgRegs: 32);
3012}
3013
3014static void allocateSGPR64Input(CCState &CCInfo, ArgDescriptor &Arg) {
3015 if (Arg) {
3016 allocateFixedSGPRInputImpl(CCInfo, RC: &AMDGPU::SGPR_64RegClass,
3017 Reg: Arg.getRegister());
3018 } else
3019 Arg = allocateSGPR32InputImpl(CCInfo, RC: &AMDGPU::SGPR_64RegClass, NumArgRegs: 16);
3020}
3021
3022/// Allocate implicit function VGPR arguments at the end of allocated user
3023/// arguments.
3024void SITargetLowering::allocateSpecialInputVGPRs(
3025 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
3026 SIMachineFunctionInfo &Info) const {
3027 const unsigned Mask = 0x3ff;
3028 ArgDescriptor Arg;
3029
3030 if (Info.hasWorkItemIDX()) {
3031 Arg = allocateVGPR32Input(CCInfo, Mask);
3032 Info.setWorkItemIDX(Arg);
3033 }
3034
3035 if (Info.hasWorkItemIDY()) {
3036 Arg = allocateVGPR32Input(CCInfo, Mask: Mask << 10, Arg);
3037 Info.setWorkItemIDY(Arg);
3038 }
3039
3040 if (Info.hasWorkItemIDZ())
3041 Info.setWorkItemIDZ(allocateVGPR32Input(CCInfo, Mask: Mask << 20, Arg));
3042}
3043
3044/// Allocate implicit function VGPR arguments in fixed registers.
3045void SITargetLowering::allocateSpecialInputVGPRsFixed(
3046 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
3047 SIMachineFunctionInfo &Info) const {
3048 Register Reg = CCInfo.AllocateReg(Reg: AMDGPU::VGPR31);
3049 if (!Reg)
3050 report_fatal_error(reason: "failed to allocate VGPR for implicit arguments");
3051
3052 const unsigned Mask = 0x3ff;
3053 Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
3054 Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg, Mask: Mask << 10));
3055 Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg, Mask: Mask << 20));
3056}
3057
3058void SITargetLowering::allocateSpecialInputSGPRs(
3059 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
3060 SIMachineFunctionInfo &Info) const {
3061 auto &ArgInfo = Info.getArgInfo();
3062 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
3063
3064 // TODO: Unify handling with private memory pointers.
3065 if (UserSGPRInfo.hasDispatchPtr())
3066 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.DispatchPtr);
3067
3068 if (UserSGPRInfo.hasQueuePtr())
3069 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.QueuePtr);
3070
3071 // Implicit arg ptr takes the place of the kernarg segment pointer. This is a
3072 // constant offset from the kernarg segment.
3073 if (Info.hasImplicitArgPtr())
3074 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.ImplicitArgPtr);
3075
3076 if (UserSGPRInfo.hasDispatchID())
3077 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.DispatchID);
3078
3079 // flat_scratch_init is not applicable for non-kernel functions.
3080
3081 if (Info.hasWorkGroupIDX())
3082 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDX);
3083
3084 if (Info.hasWorkGroupIDY())
3085 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDY);
3086
3087 if (Info.hasWorkGroupIDZ())
3088 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDZ);
3089
3090 if (Info.hasLDSKernelId())
3091 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.LDSKernelId);
3092}
3093
3094// Allocate special inputs passed in user SGPRs.
3095void SITargetLowering::allocateHSAUserSGPRs(CCState &CCInfo,
3096 MachineFunction &MF,
3097 const SIRegisterInfo &TRI,
3098 SIMachineFunctionInfo &Info) const {
3099 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
3100 if (UserSGPRInfo.hasImplicitBufferPtr()) {
3101 Register ImplicitBufferPtrReg = Info.addImplicitBufferPtr(TRI);
3102 MF.addLiveIn(PReg: ImplicitBufferPtrReg, RC: &AMDGPU::SGPR_64RegClass);
3103 CCInfo.AllocateReg(Reg: ImplicitBufferPtrReg);
3104 }
3105
3106 // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
3107 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
3108 Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
3109 MF.addLiveIn(PReg: PrivateSegmentBufferReg, RC: &AMDGPU::SGPR_128RegClass);
3110 CCInfo.AllocateReg(Reg: PrivateSegmentBufferReg);
3111 }
3112
3113 if (UserSGPRInfo.hasDispatchPtr()) {
3114 Register DispatchPtrReg = Info.addDispatchPtr(TRI);
3115 MF.addLiveIn(PReg: DispatchPtrReg, RC: &AMDGPU::SGPR_64RegClass);
3116 CCInfo.AllocateReg(Reg: DispatchPtrReg);
3117 }
3118
3119 if (UserSGPRInfo.hasQueuePtr()) {
3120 Register QueuePtrReg = Info.addQueuePtr(TRI);
3121 MF.addLiveIn(PReg: QueuePtrReg, RC: &AMDGPU::SGPR_64RegClass);
3122 CCInfo.AllocateReg(Reg: QueuePtrReg);
3123 }
3124
3125 if (UserSGPRInfo.hasKernargSegmentPtr()) {
3126 MachineRegisterInfo &MRI = MF.getRegInfo();
3127 Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
3128 CCInfo.AllocateReg(Reg: InputPtrReg);
3129
3130 Register VReg = MF.addLiveIn(PReg: InputPtrReg, RC: &AMDGPU::SGPR_64RegClass);
3131 MRI.setType(VReg, Ty: LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64));
3132 }
3133
3134 if (UserSGPRInfo.hasDispatchID()) {
3135 Register DispatchIDReg = Info.addDispatchID(TRI);
3136 MF.addLiveIn(PReg: DispatchIDReg, RC: &AMDGPU::SGPR_64RegClass);
3137 CCInfo.AllocateReg(Reg: DispatchIDReg);
3138 }
3139
3140 if (UserSGPRInfo.hasFlatScratchInit() && !getSubtarget()->isAmdPalOS()) {
3141 Register FlatScratchInitReg = Info.addFlatScratchInit(TRI);
3142 MF.addLiveIn(PReg: FlatScratchInitReg, RC: &AMDGPU::SGPR_64RegClass);
3143 CCInfo.AllocateReg(Reg: FlatScratchInitReg);
3144 }
3145
3146 if (UserSGPRInfo.hasPrivateSegmentSize()) {
3147 Register PrivateSegmentSizeReg = Info.addPrivateSegmentSize(TRI);
3148 MF.addLiveIn(PReg: PrivateSegmentSizeReg, RC: &AMDGPU::SGPR_32RegClass);
3149 CCInfo.AllocateReg(Reg: PrivateSegmentSizeReg);
3150 }
3151
3152 // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
3153 // these from the dispatch pointer.
3154}
3155
3156// Allocate pre-loaded kernel arguemtns. Arguments to be preloading must be
3157// sequential starting from the first argument.
3158void SITargetLowering::allocatePreloadKernArgSGPRs(
3159 CCState &CCInfo, SmallVectorImpl<CCValAssign> &ArgLocs,
3160 const SmallVectorImpl<ISD::InputArg> &Ins, MachineFunction &MF,
3161 const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const {
3162 Function &F = MF.getFunction();
3163 unsigned LastExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
3164 GCNUserSGPRUsageInfo &SGPRInfo = Info.getUserSGPRInfo();
3165 bool InPreloadSequence = true;
3166 unsigned InIdx = 0;
3167 bool AlignedForImplictArgs = false;
3168 unsigned ImplicitArgOffset = 0;
3169 for (auto &Arg : F.args()) {
3170 if (!InPreloadSequence || !Arg.hasInRegAttr())
3171 break;
3172
3173 unsigned ArgIdx = Arg.getArgNo();
3174 // Don't preload non-original args or parts not in the current preload
3175 // sequence.
3176 if (InIdx < Ins.size() &&
3177 (!Ins[InIdx].isOrigArg() || Ins[InIdx].getOrigArgIndex() != ArgIdx))
3178 break;
3179
3180 for (; InIdx < Ins.size() && Ins[InIdx].isOrigArg() &&
3181 Ins[InIdx].getOrigArgIndex() == ArgIdx;
3182 InIdx++) {
3183 assert(ArgLocs[ArgIdx].isMemLoc());
3184 auto &ArgLoc = ArgLocs[InIdx];
3185 const Align KernelArgBaseAlign = Align(16);
3186 unsigned ArgOffset = ArgLoc.getLocMemOffset();
3187 Align Alignment = commonAlignment(A: KernelArgBaseAlign, Offset: ArgOffset);
3188 unsigned NumAllocSGPRs =
3189 alignTo(Value: ArgLoc.getLocVT().getFixedSizeInBits(), Align: 32) / 32;
3190
3191 // Fix alignment for hidden arguments.
3192 if (Arg.hasAttribute(Kind: "amdgpu-hidden-argument")) {
3193 if (!AlignedForImplictArgs) {
3194 ImplicitArgOffset =
3195 alignTo(Size: LastExplicitArgOffset,
3196 A: Subtarget->getAlignmentForImplicitArgPtr()) -
3197 LastExplicitArgOffset;
3198 AlignedForImplictArgs = true;
3199 }
3200 ArgOffset += ImplicitArgOffset;
3201 }
3202
3203 // Arg is preloaded into the previous SGPR.
3204 if (ArgLoc.getLocVT().getStoreSize() < 4 && Alignment < 4) {
3205 assert(InIdx >= 1 && "No previous SGPR");
3206 Info.getArgInfo().PreloadKernArgs[InIdx].Regs.push_back(
3207 Elt: Info.getArgInfo().PreloadKernArgs[InIdx - 1].Regs[0]);
3208 continue;
3209 }
3210
3211 unsigned Padding = ArgOffset - LastExplicitArgOffset;
3212 unsigned PaddingSGPRs = alignTo(Value: Padding, Align: 4) / 4;
3213 // Check for free user SGPRs for preloading.
3214 if (PaddingSGPRs + NumAllocSGPRs > SGPRInfo.getNumFreeUserSGPRs()) {
3215 InPreloadSequence = false;
3216 break;
3217 }
3218
3219 // Preload this argument.
3220 const TargetRegisterClass *RC =
3221 TRI.getSGPRClassForBitWidth(BitWidth: NumAllocSGPRs * 32);
3222 SmallVectorImpl<MCRegister> *PreloadRegs =
3223 Info.addPreloadedKernArg(TRI, RC, AllocSizeDWord: NumAllocSGPRs, KernArgIdx: InIdx, PaddingSGPRs);
3224
3225 if (PreloadRegs->size() > 1)
3226 RC = &AMDGPU::SGPR_32RegClass;
3227 for (auto &Reg : *PreloadRegs) {
3228 assert(Reg);
3229 MF.addLiveIn(PReg: Reg, RC);
3230 CCInfo.AllocateReg(Reg);
3231 }
3232
3233 LastExplicitArgOffset = NumAllocSGPRs * 4 + ArgOffset;
3234 }
3235 }
3236}
3237
3238void SITargetLowering::allocateLDSKernelId(CCState &CCInfo, MachineFunction &MF,
3239 const SIRegisterInfo &TRI,
3240 SIMachineFunctionInfo &Info) const {
3241 // Always allocate this last since it is a synthetic preload.
3242 if (Info.hasLDSKernelId()) {
3243 Register Reg = Info.addLDSKernelId();
3244 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3245 CCInfo.AllocateReg(Reg);
3246 }
3247}
3248
3249// Allocate special input registers that are initialized per-wave.
3250void SITargetLowering::allocateSystemSGPRs(CCState &CCInfo, MachineFunction &MF,
3251 SIMachineFunctionInfo &Info,
3252 CallingConv::ID CallConv,
3253 bool IsShader) const {
3254 bool HasArchitectedSGPRs = Subtarget->hasArchitectedSGPRs();
3255 if (Subtarget->hasUserSGPRInit16BugInWave32() && !IsShader) {
3256 // Note: user SGPRs are handled by the front-end for graphics shaders
3257 // Pad up the used user SGPRs with dead inputs.
3258
3259 // TODO: NumRequiredSystemSGPRs computation should be adjusted appropriately
3260 // before enabling architected SGPRs for workgroup IDs.
3261 assert(!HasArchitectedSGPRs && "Unhandled feature for the subtarget");
3262
3263 unsigned CurrentUserSGPRs = Info.getNumUserSGPRs();
3264 // Note we do not count the PrivateSegmentWaveByteOffset. We do not want to
3265 // rely on it to reach 16 since if we end up having no stack usage, it will
3266 // not really be added.
3267 unsigned NumRequiredSystemSGPRs =
3268 Info.hasWorkGroupIDX() + Info.hasWorkGroupIDY() +
3269 Info.hasWorkGroupIDZ() + Info.hasWorkGroupInfo();
3270 for (unsigned i = NumRequiredSystemSGPRs + CurrentUserSGPRs; i < 16; ++i) {
3271 Register Reg = Info.addReservedUserSGPR();
3272 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3273 CCInfo.AllocateReg(Reg);
3274 }
3275 }
3276
3277 if (!HasArchitectedSGPRs) {
3278 if (Info.hasWorkGroupIDX()) {
3279 Register Reg = Info.addWorkGroupIDX();
3280 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3281 CCInfo.AllocateReg(Reg);
3282 }
3283
3284 if (Info.hasWorkGroupIDY()) {
3285 Register Reg = Info.addWorkGroupIDY();
3286 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3287 CCInfo.AllocateReg(Reg);
3288 }
3289
3290 if (Info.hasWorkGroupIDZ()) {
3291 Register Reg = Info.addWorkGroupIDZ();
3292 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3293 CCInfo.AllocateReg(Reg);
3294 }
3295 }
3296
3297 if (Info.hasWorkGroupInfo()) {
3298 Register Reg = Info.addWorkGroupInfo();
3299 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3300 CCInfo.AllocateReg(Reg);
3301 }
3302
3303 if (Info.hasPrivateSegmentWaveByteOffset()) {
3304 // Scratch wave offset passed in system SGPR.
3305 unsigned PrivateSegmentWaveByteOffsetReg;
3306
3307 if (IsShader) {
3308 PrivateSegmentWaveByteOffsetReg =
3309 Info.getPrivateSegmentWaveByteOffsetSystemSGPR();
3310
3311 // This is true if the scratch wave byte offset doesn't have a fixed
3312 // location.
3313 if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) {
3314 PrivateSegmentWaveByteOffsetReg = findFirstFreeSGPR(CCInfo);
3315 Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
3316 }
3317 } else
3318 PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset();
3319
3320 MF.addLiveIn(PReg: PrivateSegmentWaveByteOffsetReg, RC: &AMDGPU::SGPR_32RegClass);
3321 CCInfo.AllocateReg(Reg: PrivateSegmentWaveByteOffsetReg);
3322 }
3323
3324 assert(!Subtarget->hasUserSGPRInit16BugInWave32() || IsShader ||
3325 Info.getNumPreloadedSGPRs() >= 16);
3326}
3327
3328static void reservePrivateMemoryRegs(const TargetMachine &TM,
3329 MachineFunction &MF,
3330 const SIRegisterInfo &TRI,
3331 SIMachineFunctionInfo &Info) {
3332 // Now that we've figured out where the scratch register inputs are, see if
3333 // should reserve the arguments and use them directly.
3334 MachineFrameInfo &MFI = MF.getFrameInfo();
3335 bool HasStackObjects = MFI.hasStackObjects();
3336 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
3337
3338 // Record that we know we have non-spill stack objects so we don't need to
3339 // check all stack objects later.
3340 if (HasStackObjects)
3341 Info.setHasNonSpillStackObjects(true);
3342
3343 // Everything live out of a block is spilled with fast regalloc, so it's
3344 // almost certain that spilling will be required.
3345 if (TM.getOptLevel() == CodeGenOptLevel::None)
3346 HasStackObjects = true;
3347
3348 // For now assume stack access is needed in any callee functions, so we need
3349 // the scratch registers to pass in.
3350 bool RequiresStackAccess = HasStackObjects || MFI.hasCalls();
3351
3352 if (!ST.hasFlatScratchEnabled()) {
3353 if (RequiresStackAccess && ST.isAmdHsaOrMesa(F: MF.getFunction())) {
3354 // If we have stack objects, we unquestionably need the private buffer
3355 // resource. For the Code Object V2 ABI, this will be the first 4 user
3356 // SGPR inputs. We can reserve those and use them directly.
3357
3358 Register PrivateSegmentBufferReg =
3359 Info.getPreloadedReg(Value: AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
3360 Info.setScratchRSrcReg(PrivateSegmentBufferReg);
3361 } else {
3362 unsigned ReservedBufferReg = TRI.reservedPrivateSegmentBufferReg(MF);
3363 // We tentatively reserve the last registers (skipping the last registers
3364 // which may contain VCC, FLAT_SCR, and XNACK). After register allocation,
3365 // we'll replace these with the ones immediately after those which were
3366 // really allocated. In the prologue copies will be inserted from the
3367 // argument to these reserved registers.
3368
3369 // Without HSA, relocations are used for the scratch pointer and the
3370 // buffer resource setup is always inserted in the prologue. Scratch wave
3371 // offset is still in an input SGPR.
3372 Info.setScratchRSrcReg(ReservedBufferReg);
3373 }
3374 }
3375
3376 MachineRegisterInfo &MRI = MF.getRegInfo();
3377
3378 // For entry functions we have to set up the stack pointer if we use it,
3379 // whereas non-entry functions get this "for free". This means there is no
3380 // intrinsic advantage to using S32 over S34 in cases where we do not have
3381 // calls but do need a frame pointer (i.e. if we are requested to have one
3382 // because frame pointer elimination is disabled). To keep things simple we
3383 // only ever use S32 as the call ABI stack pointer, and so using it does not
3384 // imply we need a separate frame pointer.
3385 //
3386 // Try to use s32 as the SP, but move it if it would interfere with input
3387 // arguments. This won't work with calls though.
3388 //
3389 // FIXME: Move SP to avoid any possible inputs, or find a way to spill input
3390 // registers.
3391 if (!MRI.isLiveIn(Reg: AMDGPU::SGPR32)) {
3392 Info.setStackPtrOffsetReg(AMDGPU::SGPR32);
3393 } else {
3394 assert(AMDGPU::isShader(MF.getFunction().getCallingConv()));
3395
3396 if (MFI.hasCalls())
3397 report_fatal_error(reason: "call in graphics shader with too many input SGPRs");
3398
3399 for (unsigned Reg : AMDGPU::SGPR_32RegClass) {
3400 if (!MRI.isLiveIn(Reg)) {
3401 Info.setStackPtrOffsetReg(Reg);
3402 break;
3403 }
3404 }
3405
3406 if (Info.getStackPtrOffsetReg() == AMDGPU::SP_REG)
3407 report_fatal_error(reason: "failed to find register for SP");
3408 }
3409
3410 // hasFP should be accurate for entry functions even before the frame is
3411 // finalized, because it does not rely on the known stack size, only
3412 // properties like whether variable sized objects are present.
3413 if (ST.getFrameLowering()->hasFP(MF)) {
3414 Info.setFrameOffsetReg(AMDGPU::SGPR33);
3415 }
3416}
3417
3418bool SITargetLowering::supportSplitCSR(MachineFunction *MF) const {
3419 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
3420 return !Info->isEntryFunction();
3421}
3422
3423void SITargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {}
3424
3425void SITargetLowering::insertCopiesSplitCSR(
3426 MachineBasicBlock *Entry,
3427 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
3428 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3429
3430 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(MF: Entry->getParent());
3431 if (!IStart)
3432 return;
3433
3434 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
3435 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
3436 MachineBasicBlock::iterator MBBI = Entry->begin();
3437 for (const MCPhysReg *I = IStart; *I; ++I) {
3438 const TargetRegisterClass *RC = nullptr;
3439 if (AMDGPU::SReg_64RegClass.contains(Reg: *I))
3440 RC = &AMDGPU::SGPR_64RegClass;
3441 else if (AMDGPU::SReg_32RegClass.contains(Reg: *I))
3442 RC = &AMDGPU::SGPR_32RegClass;
3443 else
3444 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3445
3446 Register NewVR = MRI->createVirtualRegister(RegClass: RC);
3447 // Create copy from CSR to a virtual register.
3448 Entry->addLiveIn(PhysReg: *I);
3449 BuildMI(BB&: *Entry, I: MBBI, MIMD: DebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewVR)
3450 .addReg(RegNo: *I);
3451
3452 // Insert the copy-back instructions right before the terminator.
3453 for (auto *Exit : Exits)
3454 BuildMI(BB&: *Exit, I: Exit->getFirstTerminator(), MIMD: DebugLoc(),
3455 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: *I)
3456 .addReg(RegNo: NewVR);
3457 }
3458}
3459
3460SDValue SITargetLowering::LowerFormalArguments(
3461 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3462 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3463 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3464 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3465
3466 MachineFunction &MF = DAG.getMachineFunction();
3467 const Function &Fn = MF.getFunction();
3468 FunctionType *FType = MF.getFunction().getFunctionType();
3469 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
3470 bool IsError = false;
3471
3472 if (Subtarget->isAmdHsaOS() && AMDGPU::isGraphics(CC: CallConv)) {
3473 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
3474 Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc()));
3475 IsError = true;
3476 }
3477
3478 SmallVector<ISD::InputArg, 16> Splits;
3479 SmallVector<CCValAssign, 16> ArgLocs;
3480 BitVector Skipped(Fn.arg_size());
3481 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
3482 *DAG.getContext());
3483
3484 bool IsGraphics = AMDGPU::isGraphics(CC: CallConv);
3485 bool IsKernel = AMDGPU::isKernel(CC: CallConv);
3486 bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CC: CallConv);
3487
3488 if (IsGraphics) {
3489 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info->getUserSGPRInfo();
3490 assert(!UserSGPRInfo.hasDispatchPtr() &&
3491 !UserSGPRInfo.hasKernargSegmentPtr() && !Info->hasWorkGroupInfo() &&
3492 !Info->hasLDSKernelId() && !Info->hasWorkItemIDX() &&
3493 !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ());
3494 (void)UserSGPRInfo;
3495 if (!Subtarget->hasFlatScratchEnabled())
3496 assert(!UserSGPRInfo.hasFlatScratchInit());
3497 if ((CallConv != CallingConv::AMDGPU_CS &&
3498 CallConv != CallingConv::AMDGPU_Gfx &&
3499 CallConv != CallingConv::AMDGPU_Gfx_WholeWave) ||
3500 !Subtarget->hasArchitectedSGPRs())
3501 assert(!Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() &&
3502 !Info->hasWorkGroupIDZ());
3503 }
3504
3505 bool IsWholeWaveFunc = Info->isWholeWaveFunction();
3506
3507 if (CallConv == CallingConv::AMDGPU_PS) {
3508 processPSInputArgs(Splits, CallConv, Ins, Skipped, FType, Info);
3509
3510 // At least one interpolation mode must be enabled or else the GPU will
3511 // hang.
3512 //
3513 // Check PSInputAddr instead of PSInputEnable. The idea is that if the user
3514 // set PSInputAddr, the user wants to enable some bits after the compilation
3515 // based on run-time states. Since we can't know what the final PSInputEna
3516 // will look like, so we shouldn't do anything here and the user should take
3517 // responsibility for the correct programming.
3518 //
3519 // Otherwise, the following restrictions apply:
3520 // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
3521 // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
3522 // enabled too.
3523 if ((Info->getPSInputAddr() & 0x7F) == 0 ||
3524 ((Info->getPSInputAddr() & 0xF) == 0 && Info->isPSInputAllocated(Index: 11))) {
3525 CCInfo.AllocateReg(Reg: AMDGPU::VGPR0);
3526 CCInfo.AllocateReg(Reg: AMDGPU::VGPR1);
3527 Info->markPSInputAllocated(Index: 0);
3528 Info->markPSInputEnabled(Index: 0);
3529 }
3530 if (Subtarget->isAmdPalOS()) {
3531 // For isAmdPalOS, the user does not enable some bits after compilation
3532 // based on run-time states; the register values being generated here are
3533 // the final ones set in hardware. Therefore we need to apply the
3534 // workaround to PSInputAddr and PSInputEnable together. (The case where
3535 // a bit is set in PSInputAddr but not PSInputEnable is where the
3536 // frontend set up an input arg for a particular interpolation mode, but
3537 // nothing uses that input arg. Really we should have an earlier pass
3538 // that removes such an arg.)
3539 unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
3540 if ((PsInputBits & 0x7F) == 0 ||
3541 ((PsInputBits & 0xF) == 0 && (PsInputBits >> 11 & 1)))
3542 Info->markPSInputEnabled(Index: llvm::countr_zero(Val: Info->getPSInputAddr()));
3543 }
3544 } else if (IsKernel) {
3545 assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX());
3546 } else {
3547 Splits.append(in_start: IsWholeWaveFunc ? std::next(x: Ins.begin()) : Ins.begin(),
3548 in_end: Ins.end());
3549 }
3550
3551 if (IsKernel)
3552 analyzeFormalArgumentsCompute(State&: CCInfo, Ins);
3553
3554 if (IsEntryFunc) {
3555 allocateSpecialEntryInputVGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3556 allocateHSAUserSGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3557 if (IsKernel && Subtarget->hasKernargPreload())
3558 allocatePreloadKernArgSGPRs(CCInfo, ArgLocs, Ins, MF, TRI: *TRI, Info&: *Info);
3559
3560 allocateLDSKernelId(CCInfo, MF, TRI: *TRI, Info&: *Info);
3561 } else if (!IsGraphics) {
3562 // For the fixed ABI, pass workitem IDs in the last argument register.
3563 allocateSpecialInputVGPRsFixed(CCInfo, MF, TRI: *TRI, Info&: *Info);
3564
3565 // FIXME: Sink this into allocateSpecialInputSGPRs
3566 if (!Subtarget->hasFlatScratchEnabled())
3567 CCInfo.AllocateReg(Reg: Info->getScratchRSrcReg());
3568
3569 allocateSpecialInputSGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3570 }
3571
3572 if (!IsKernel) {
3573 CCAssignFn *AssignFn = CCAssignFnForCall(CC: CallConv, IsVarArg: isVarArg);
3574 CCInfo.AnalyzeFormalArguments(Ins: Splits, Fn: AssignFn);
3575
3576 // This assumes the registers are allocated by CCInfo in ascending order
3577 // with no gaps.
3578 Info->setNumWaveDispatchSGPRs(
3579 CCInfo.getFirstUnallocated(Regs: AMDGPU::SGPR_32RegClass.getRegisters()));
3580 Info->setNumWaveDispatchVGPRs(
3581 CCInfo.getFirstUnallocated(Regs: AMDGPU::VGPR_32RegClass.getRegisters()));
3582 } else if (Info->getNumKernargPreloadedSGPRs()) {
3583 Info->setNumWaveDispatchSGPRs(Info->getNumUserSGPRs());
3584 }
3585
3586 SmallVector<SDValue, 16> Chains;
3587
3588 if (IsWholeWaveFunc) {
3589 SDValue Setup = DAG.getNode(Opcode: AMDGPUISD::WHOLE_WAVE_SETUP, DL,
3590 ResultTys: {MVT::i1, MVT::Other}, Ops: Chain);
3591 InVals.push_back(Elt: Setup.getValue(R: 0));
3592 Chains.push_back(Elt: Setup.getValue(R: 1));
3593 }
3594
3595 // FIXME: This is the minimum kernel argument alignment. We should improve
3596 // this to the maximum alignment of the arguments.
3597 //
3598 // FIXME: Alignment of explicit arguments totally broken with non-0 explicit
3599 // kern arg offset.
3600 const Align KernelArgBaseAlign = Align(16);
3601
3602 for (unsigned i = IsWholeWaveFunc ? 1 : 0, e = Ins.size(), ArgIdx = 0; i != e;
3603 ++i) {
3604 const ISD::InputArg &Arg = Ins[i];
3605 if ((Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) || IsError) {
3606 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
3607 continue;
3608 }
3609
3610 CCValAssign &VA = ArgLocs[ArgIdx++];
3611 MVT VT = VA.getLocVT();
3612
3613 if (IsEntryFunc && VA.isMemLoc()) {
3614 VT = Ins[i].VT;
3615 EVT MemVT = VA.getLocVT();
3616
3617 const uint64_t Offset = VA.getLocMemOffset();
3618 Align Alignment = commonAlignment(A: KernelArgBaseAlign, Offset);
3619
3620 if (Arg.Flags.isByRef()) {
3621 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL: DL, Chain, Offset);
3622
3623 const GCNTargetMachine &TM =
3624 static_cast<const GCNTargetMachine &>(getTargetMachine());
3625 if (!TM.isNoopAddrSpaceCast(SrcAS: AMDGPUAS::CONSTANT_ADDRESS,
3626 DestAS: Arg.Flags.getPointerAddrSpace())) {
3627 Ptr = DAG.getAddrSpaceCast(dl: DL, VT, Ptr, SrcAS: AMDGPUAS::CONSTANT_ADDRESS,
3628 DestAS: Arg.Flags.getPointerAddrSpace());
3629 }
3630
3631 InVals.push_back(Elt: Ptr);
3632 continue;
3633 }
3634
3635 SDValue NewArg;
3636 if (Arg.isOrigArg() && Info->getArgInfo().PreloadKernArgs.count(Val: i)) {
3637 if (MemVT.getStoreSize() < 4 && Alignment < 4) {
3638 // In this case the argument is packed into the previous preload SGPR.
3639 int64_t AlignDownOffset = alignDown(Value: Offset, Align: 4);
3640 int64_t OffsetDiff = Offset - AlignDownOffset;
3641 EVT IntVT = MemVT.changeTypeToInteger();
3642
3643 const SIMachineFunctionInfo *Info =
3644 MF.getInfo<SIMachineFunctionInfo>();
3645 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
3646 Register Reg =
3647 Info->getArgInfo().PreloadKernArgs.find(Val: i)->getSecond().Regs[0];
3648
3649 assert(Reg);
3650 Register VReg = MRI.getLiveInVirtReg(PReg: Reg);
3651 SDValue Copy = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::i32);
3652
3653 SDValue ShiftAmt = DAG.getConstant(Val: OffsetDiff * 8, DL, VT: MVT::i32);
3654 SDValue Extract = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Copy, N2: ShiftAmt);
3655
3656 SDValue ArgVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: Extract);
3657 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MemVT, Operand: ArgVal);
3658 NewArg = convertArgType(DAG, VT, MemVT, SL: DL, Val: ArgVal,
3659 Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3660
3661 NewArg = DAG.getMergeValues(Ops: {NewArg, Copy.getValue(R: 1)}, dl: DL);
3662 } else {
3663 const SIMachineFunctionInfo *Info =
3664 MF.getInfo<SIMachineFunctionInfo>();
3665 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
3666 const SmallVectorImpl<MCRegister> &PreloadRegs =
3667 Info->getArgInfo().PreloadKernArgs.find(Val: i)->getSecond().Regs;
3668
3669 SDValue Copy;
3670 if (PreloadRegs.size() == 1) {
3671 Register VReg = MRI.getLiveInVirtReg(PReg: PreloadRegs[0]);
3672 const TargetRegisterClass *RC = MRI.getRegClass(Reg: VReg);
3673 NewArg = DAG.getCopyFromReg(
3674 Chain, dl: DL, Reg: VReg,
3675 VT: EVT::getIntegerVT(Context&: *DAG.getContext(),
3676 BitWidth: TRI->getRegSizeInBits(RC: *RC)));
3677
3678 } else {
3679 // If the kernarg alignment does not match the alignment of the SGPR
3680 // tuple RC that can accommodate this argument, it will be built up
3681 // via copies from from the individual SGPRs that the argument was
3682 // preloaded to.
3683 SmallVector<SDValue, 4> Elts;
3684 for (auto Reg : PreloadRegs) {
3685 Register VReg = MRI.getLiveInVirtReg(PReg: Reg);
3686 Copy = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::i32);
3687 Elts.push_back(Elt: Copy);
3688 }
3689 NewArg =
3690 DAG.getBuildVector(VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
3691 NumElements: PreloadRegs.size()),
3692 DL, Ops: Elts);
3693 }
3694
3695 // If the argument was preloaded to multiple consecutive 32-bit
3696 // registers because of misalignment between addressable SGPR tuples
3697 // and the argument size, we can still assume that because of kernarg
3698 // segment alignment restrictions that NewArg's size is the same as
3699 // MemVT and just do a bitcast. If MemVT is less than 32-bits we add a
3700 // truncate since we cannot preload to less than a single SGPR and the
3701 // MemVT may be smaller.
3702 EVT MemVTInt =
3703 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
3704 if (MemVT.bitsLT(VT: NewArg.getSimpleValueType()))
3705 NewArg = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MemVTInt, Operand: NewArg);
3706
3707 NewArg = DAG.getBitcast(VT: MemVT, V: NewArg);
3708 NewArg = convertArgType(DAG, VT, MemVT, SL: DL, Val: NewArg,
3709 Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3710 NewArg = DAG.getMergeValues(Ops: {NewArg, Chain}, dl: DL);
3711 }
3712 } else {
3713 // Hidden arguments that are in the kernel signature must be preloaded
3714 // to user SGPRs. Print a diagnostic error if a hidden argument is in
3715 // the argument list and is not preloaded.
3716 if (Arg.isOrigArg()) {
3717 Argument *OrigArg = Fn.getArg(i: Arg.getOrigArgIndex());
3718 if (OrigArg->hasAttribute(Kind: "amdgpu-hidden-argument")) {
3719 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
3720 *OrigArg->getParent(),
3721 "hidden argument in kernel signature was not preloaded",
3722 DL.getDebugLoc()));
3723 }
3724 }
3725
3726 NewArg =
3727 lowerKernargMemParameter(DAG, VT, MemVT, SL: DL, Chain, Offset,
3728 Alignment, Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3729 }
3730 Chains.push_back(Elt: NewArg.getValue(R: 1));
3731
3732 auto *ParamTy =
3733 dyn_cast<PointerType>(Val: FType->getParamType(i: Ins[i].getOrigArgIndex()));
3734 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS &&
3735 ParamTy &&
3736 (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
3737 ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS)) {
3738 // On SI local pointers are just offsets into LDS, so they are always
3739 // less than 16-bits. On CI and newer they could potentially be
3740 // real pointers, so we can't guarantee their size.
3741 NewArg = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: NewArg.getValueType(), N1: NewArg,
3742 N2: DAG.getValueType(MVT::i16));
3743 }
3744
3745 InVals.push_back(Elt: NewArg);
3746 continue;
3747 }
3748 if (!IsEntryFunc && VA.isMemLoc()) {
3749 SDValue Val = lowerStackParameter(DAG, VA, SL: DL, Chain, Arg);
3750 InVals.push_back(Elt: Val);
3751 if (!Arg.Flags.isByVal())
3752 Chains.push_back(Elt: Val.getValue(R: 1));
3753 continue;
3754 }
3755
3756 assert(VA.isRegLoc() && "Parameter must be in a register!");
3757
3758 Register Reg = VA.getLocReg();
3759 const TargetRegisterClass *RC = nullptr;
3760 if (AMDGPU::VGPR_32RegClass.contains(Reg))
3761 RC = &AMDGPU::VGPR_32RegClass;
3762 else if (AMDGPU::SGPR_32RegClass.contains(Reg))
3763 RC = &AMDGPU::SGPR_32RegClass;
3764 else
3765 llvm_unreachable("Unexpected register class in LowerFormalArguments!");
3766
3767 Reg = MF.addLiveIn(PReg: Reg, RC);
3768 SDValue Val = DAG.getCopyFromReg(Chain, dl: DL, Reg, VT);
3769 if (Arg.Flags.isInReg() && RC == &AMDGPU::VGPR_32RegClass) {
3770 // FIXME: Need to forward the chains created by `CopyFromReg`s, make sure
3771 // they will read physical regs before any side effect instructions.
3772 SDValue ReadFirstLane =
3773 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
3774 Val = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Val.getValueType(),
3775 N1: ReadFirstLane, N2: Val);
3776 }
3777
3778 if (Arg.Flags.isSRet()) {
3779 // The return object should be reasonably addressable.
3780 Val = annotateStackObjectPointer(Ptr: Val, DAG, DL,
3781 Alignment: Arg.Flags.getNonZeroMemAlign());
3782 }
3783
3784 Val = convertABITypeToValueType(DAG, Val, VA, SL: DL);
3785 InVals.push_back(Elt: Val);
3786 }
3787
3788 // Start adding system SGPRs.
3789 if (IsEntryFunc)
3790 allocateSystemSGPRs(CCInfo, MF, Info&: *Info, CallConv, IsShader: IsGraphics);
3791
3792 unsigned StackArgSize = CCInfo.getStackSize();
3793 Info->setBytesInStackArgArea(StackArgSize);
3794
3795 return Chains.empty() ? Chain
3796 : DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
3797}
3798
3799// TODO: If return values can't fit in registers, we should return as many as
3800// possible in registers before passing on stack.
3801bool SITargetLowering::CanLowerReturn(
3802 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
3803 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
3804 const Type *RetTy) const {
3805 // Replacing returns with sret/stack usage doesn't make sense for shaders.
3806 // FIXME: Also sort of a workaround for custom vector splitting in LowerReturn
3807 // for shaders. Vector types should be explicitly handled by CC.
3808 if (AMDGPU::isEntryFunctionCC(CC: CallConv))
3809 return true;
3810
3811 SmallVector<CCValAssign, 16> RVLocs;
3812 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
3813 if (!CCInfo.CheckReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, IsVarArg)))
3814 return false;
3815
3816 // We must use the stack if return would require unavailable registers.
3817 unsigned MaxNumVGPRs = Subtarget->getMaxNumVGPRs(MF);
3818 unsigned TotalNumVGPRs = Subtarget->getAddressableNumArchVGPRs();
3819 for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i)
3820 if (CCInfo.isAllocated(Reg: AMDGPU::VGPR_32RegClass.getRegister(i)))
3821 return false;
3822
3823 return true;
3824}
3825
3826SDValue
3827SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
3828 bool isVarArg,
3829 const SmallVectorImpl<ISD::OutputArg> &Outs,
3830 const SmallVectorImpl<SDValue> &OutVals,
3831 const SDLoc &DL, SelectionDAG &DAG) const {
3832 MachineFunction &MF = DAG.getMachineFunction();
3833 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
3834 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3835
3836 if (AMDGPU::isKernel(CC: CallConv)) {
3837 return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs,
3838 OutVals, DL, DAG);
3839 }
3840
3841 bool IsShader = AMDGPU::isShader(CC: CallConv);
3842
3843 Info->setIfReturnsVoid(Outs.empty());
3844 bool IsWaveEnd = Info->returnsVoid() && IsShader;
3845
3846 // CCValAssign - represent the assignment of the return value to a location.
3847 SmallVector<CCValAssign, 48> RVLocs;
3848
3849 // CCState - Info about the registers and stack slots.
3850 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
3851 *DAG.getContext());
3852
3853 // Analyze outgoing return values.
3854 CCInfo.AnalyzeReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, IsVarArg: isVarArg));
3855
3856 SDValue Glue;
3857 SmallVector<SDValue, 48> RetOps;
3858 RetOps.push_back(Elt: Chain); // Operand #0 = Chain (updated below)
3859
3860 SDValue ReadFirstLane =
3861 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
3862 // Copy the result values into the output registers.
3863 for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E;
3864 ++I, ++RealRVLocIdx) {
3865 CCValAssign &VA = RVLocs[I];
3866 assert(VA.isRegLoc() && "Can only return in registers!");
3867 // TODO: Partially return in registers if return values don't fit.
3868 SDValue Arg = OutVals[RealRVLocIdx];
3869
3870 // Copied from other backends.
3871 switch (VA.getLocInfo()) {
3872 case CCValAssign::Full:
3873 break;
3874 case CCValAssign::BCvt:
3875 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getLocVT(), Operand: Arg);
3876 break;
3877 case CCValAssign::SExt:
3878 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3879 break;
3880 case CCValAssign::ZExt:
3881 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3882 break;
3883 case CCValAssign::AExt:
3884 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3885 break;
3886 default:
3887 llvm_unreachable("Unknown loc info!");
3888 }
3889 if (TRI->isSGPRPhysReg(Reg: VA.getLocReg()))
3890 Arg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Arg.getValueType(),
3891 N1: ReadFirstLane, N2: Arg);
3892 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VA.getLocReg(), N: Arg, Glue);
3893 Glue = Chain.getValue(R: 1);
3894 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
3895 }
3896
3897 // FIXME: Does sret work properly?
3898 if (!Info->isEntryFunction()) {
3899 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
3900 const MCPhysReg *I =
3901 TRI->getCalleeSavedRegsViaCopy(MF: &DAG.getMachineFunction());
3902 if (I) {
3903 for (; *I; ++I) {
3904 if (AMDGPU::SReg_64RegClass.contains(Reg: *I))
3905 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::i64));
3906 else if (AMDGPU::SReg_32RegClass.contains(Reg: *I))
3907 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::i32));
3908 else
3909 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3910 }
3911 }
3912 }
3913
3914 // Update chain and glue.
3915 RetOps[0] = Chain;
3916 if (Glue.getNode())
3917 RetOps.push_back(Elt: Glue);
3918
3919 unsigned Opc = AMDGPUISD::ENDPGM;
3920 if (!IsWaveEnd)
3921 Opc = Info->isWholeWaveFunction() ? AMDGPUISD::WHOLE_WAVE_RETURN
3922 : IsShader ? AMDGPUISD::RETURN_TO_EPILOG
3923 : AMDGPUISD::RET_GLUE;
3924 return DAG.getNode(Opcode: Opc, DL, VT: MVT::Other, Ops: RetOps);
3925}
3926
3927SDValue SITargetLowering::LowerCallResult(
3928 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool IsVarArg,
3929 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3930 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn,
3931 SDValue ThisVal) const {
3932 CCAssignFn *RetCC = CCAssignFnForReturn(CC: CallConv, IsVarArg);
3933
3934 // Assign locations to each value returned by this call.
3935 SmallVector<CCValAssign, 16> RVLocs;
3936 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
3937 *DAG.getContext());
3938 CCInfo.AnalyzeCallResult(Ins, Fn: RetCC);
3939
3940 // Copy all of the result registers out of their specified physreg.
3941 for (CCValAssign VA : RVLocs) {
3942 SDValue Val;
3943
3944 if (VA.isRegLoc()) {
3945 Val =
3946 DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue: InGlue);
3947 Chain = Val.getValue(R: 1);
3948 InGlue = Val.getValue(R: 2);
3949 } else if (VA.isMemLoc()) {
3950 report_fatal_error(reason: "TODO: return values in memory");
3951 } else
3952 llvm_unreachable("unknown argument location type");
3953
3954 switch (VA.getLocInfo()) {
3955 case CCValAssign::Full:
3956 break;
3957 case CCValAssign::BCvt:
3958 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Val);
3959 break;
3960 case CCValAssign::ZExt:
3961 Val = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: VA.getLocVT(), N1: Val,
3962 N2: DAG.getValueType(VA.getValVT()));
3963 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
3964 break;
3965 case CCValAssign::SExt:
3966 Val = DAG.getNode(Opcode: ISD::AssertSext, DL, VT: VA.getLocVT(), N1: Val,
3967 N2: DAG.getValueType(VA.getValVT()));
3968 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
3969 break;
3970 case CCValAssign::AExt:
3971 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
3972 break;
3973 default:
3974 llvm_unreachable("Unknown loc info!");
3975 }
3976
3977 InVals.push_back(Elt: Val);
3978 }
3979
3980 return Chain;
3981}
3982
3983// Add code to pass special inputs required depending on used features separate
3984// from the explicit user arguments present in the IR.
3985void SITargetLowering::passSpecialInputs(
3986 CallLoweringInfo &CLI, CCState &CCInfo, const SIMachineFunctionInfo &Info,
3987 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
3988 SmallVectorImpl<SDValue> &MemOpChains, SDValue Chain) const {
3989 // If we don't have a call site, this was a call inserted by
3990 // legalization. These can never use special inputs.
3991 if (!CLI.CB)
3992 return;
3993
3994 SelectionDAG &DAG = CLI.DAG;
3995 const SDLoc &DL = CLI.DL;
3996 const Function &F = DAG.getMachineFunction().getFunction();
3997
3998 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
3999 const AMDGPUFunctionArgInfo &CallerArgInfo = Info.getArgInfo();
4000
4001 const AMDGPUFunctionArgInfo &CalleeArgInfo =
4002 AMDGPUFunctionArgInfo::FixedABIFunctionInfo;
4003
4004 // TODO: Unify with private memory register handling. This is complicated by
4005 // the fact that at least in kernels, the input argument is not necessarily
4006 // in the same location as the input.
4007 // clang-format off
4008 static constexpr std::pair<AMDGPUFunctionArgInfo::PreloadedValue,
4009 std::array<StringLiteral, 2>> ImplicitAttrs[] = {
4010 {AMDGPUFunctionArgInfo::DISPATCH_PTR, {"amdgpu-no-dispatch-ptr", ""}},
4011 {AMDGPUFunctionArgInfo::QUEUE_PTR, {"amdgpu-no-queue-ptr", ""}},
4012 {AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR, {"amdgpu-no-implicitarg-ptr", ""}},
4013 {AMDGPUFunctionArgInfo::DISPATCH_ID, {"amdgpu-no-dispatch-id", ""}},
4014 {AMDGPUFunctionArgInfo::WORKGROUP_ID_X, {"amdgpu-no-workgroup-id-x", "amdgpu-no-cluster-id-x"}},
4015 {AMDGPUFunctionArgInfo::WORKGROUP_ID_Y, {"amdgpu-no-workgroup-id-y", "amdgpu-no-cluster-id-y"}},
4016 {AMDGPUFunctionArgInfo::WORKGROUP_ID_Z, {"amdgpu-no-workgroup-id-z", "amdgpu-no-cluster-id-z"}},
4017 {AMDGPUFunctionArgInfo::LDS_KERNEL_ID, {"amdgpu-no-lds-kernel-id", ""}},
4018 };
4019 // clang-format on
4020
4021 for (auto [InputID, Attrs] : ImplicitAttrs) {
4022 // If the callee does not use the attribute value, skip copying the value.
4023 if (all_of(Range&: Attrs, P: [&](StringRef Attr) {
4024 return Attr.empty() || CLI.CB->hasFnAttr(Kind: Attr);
4025 }))
4026 continue;
4027
4028 const auto [OutgoingArg, ArgRC, ArgTy] =
4029 CalleeArgInfo.getPreloadedValue(Value: InputID);
4030 if (!OutgoingArg)
4031 continue;
4032
4033 const auto [IncomingArg, IncomingArgRC, Ty] =
4034 CallerArgInfo.getPreloadedValue(Value: InputID);
4035 assert(IncomingArgRC == ArgRC);
4036
4037 // All special arguments are ints for now.
4038 EVT ArgVT = TRI->getSpillSize(RC: *ArgRC) == 8 ? MVT::i64 : MVT::i32;
4039 SDValue InputReg;
4040
4041 if (IncomingArg) {
4042 InputReg = loadInputValue(DAG, RC: ArgRC, VT: ArgVT, SL: DL, Arg: *IncomingArg);
4043 } else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) {
4044 // The implicit arg ptr is special because it doesn't have a corresponding
4045 // input for kernels, and is computed from the kernarg segment pointer.
4046 InputReg = getImplicitArgPtr(DAG, SL: DL);
4047 } else if (InputID == AMDGPUFunctionArgInfo::LDS_KERNEL_ID) {
4048 std::optional<uint32_t> Id =
4049 AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F);
4050 if (Id.has_value()) {
4051 InputReg = DAG.getConstant(Val: *Id, DL, VT: ArgVT);
4052 } else {
4053 InputReg = DAG.getPOISON(VT: ArgVT);
4054 }
4055 } else {
4056 // We may have proven the input wasn't needed, although the ABI is
4057 // requiring it. We just need to allocate the register appropriately.
4058 InputReg = DAG.getPOISON(VT: ArgVT);
4059 }
4060
4061 if (OutgoingArg->isRegister()) {
4062 RegsToPass.emplace_back(Args: OutgoingArg->getRegister(), Args&: InputReg);
4063 if (!CCInfo.AllocateReg(Reg: OutgoingArg->getRegister()))
4064 report_fatal_error(reason: "failed to allocate implicit input argument");
4065 } else {
4066 unsigned SpecialArgOffset =
4067 CCInfo.AllocateStack(Size: ArgVT.getStoreSize(), Alignment: Align(4));
4068 SDValue ArgStore =
4069 storeStackInputValue(DAG, SL: DL, Chain, ArgVal: InputReg, Offset: SpecialArgOffset);
4070 MemOpChains.push_back(Elt: ArgStore);
4071 }
4072 }
4073
4074 // Pack workitem IDs into a single register or pass it as is if already
4075 // packed.
4076
4077 auto [OutgoingArg, ArgRC, Ty] =
4078 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_X);
4079 if (!OutgoingArg)
4080 std::tie(args&: OutgoingArg, args&: ArgRC, args&: Ty) =
4081 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
4082 if (!OutgoingArg)
4083 std::tie(args&: OutgoingArg, args&: ArgRC, args&: Ty) =
4084 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
4085 if (!OutgoingArg)
4086 return;
4087
4088 const ArgDescriptor *IncomingArgX = std::get<0>(
4089 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_X));
4090 const ArgDescriptor *IncomingArgY = std::get<0>(
4091 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Y));
4092 const ArgDescriptor *IncomingArgZ = std::get<0>(
4093 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Z));
4094
4095 SDValue InputReg;
4096 SDLoc SL;
4097
4098 const bool NeedWorkItemIDX = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-x");
4099 const bool NeedWorkItemIDY = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-y");
4100 const bool NeedWorkItemIDZ = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-z");
4101
4102 // If incoming ids are not packed we need to pack them.
4103 if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo.WorkItemIDX &&
4104 NeedWorkItemIDX) {
4105 if (Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 0) != 0) {
4106 InputReg = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgX);
4107 } else {
4108 InputReg = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
4109 }
4110 }
4111
4112 if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo.WorkItemIDY &&
4113 NeedWorkItemIDY && Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 1) != 0) {
4114 SDValue Y = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgY);
4115 Y = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Y,
4116 N2: DAG.getShiftAmountConstant(Val: 10, VT: MVT::i32, DL: SL));
4117 InputReg = InputReg.getNode()
4118 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: InputReg, N2: Y)
4119 : Y;
4120 }
4121
4122 if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo.WorkItemIDZ &&
4123 NeedWorkItemIDZ && Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 2) != 0) {
4124 SDValue Z = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgZ);
4125 Z = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Z,
4126 N2: DAG.getShiftAmountConstant(Val: 20, VT: MVT::i32, DL: SL));
4127 InputReg = InputReg.getNode()
4128 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: InputReg, N2: Z)
4129 : Z;
4130 }
4131
4132 if (!InputReg && (NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ)) {
4133 if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {
4134 // We're in a situation where the outgoing function requires the workitem
4135 // ID, but the calling function does not have it (e.g a graphics function
4136 // calling a C calling convention function). This is illegal, but we need
4137 // to produce something.
4138 InputReg = DAG.getPOISON(VT: MVT::i32);
4139 } else {
4140 // Workitem ids are already packed, any of present incoming arguments
4141 // will carry all required fields.
4142 ArgDescriptor IncomingArg =
4143 ArgDescriptor::createArg(Arg: IncomingArgX ? *IncomingArgX
4144 : IncomingArgY ? *IncomingArgY
4145 : *IncomingArgZ,
4146 Mask: ~0u);
4147 InputReg = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: IncomingArg);
4148 }
4149 }
4150
4151 if (OutgoingArg->isRegister()) {
4152 if (InputReg)
4153 RegsToPass.emplace_back(Args: OutgoingArg->getRegister(), Args&: InputReg);
4154
4155 CCInfo.AllocateReg(Reg: OutgoingArg->getRegister());
4156 } else {
4157 unsigned SpecialArgOffset = CCInfo.AllocateStack(Size: 4, Alignment: Align(4));
4158 if (InputReg) {
4159 SDValue ArgStore =
4160 storeStackInputValue(DAG, SL: DL, Chain, ArgVal: InputReg, Offset: SpecialArgOffset);
4161 MemOpChains.push_back(Elt: ArgStore);
4162 }
4163 }
4164}
4165
4166bool SITargetLowering::isEligibleForTailCallOptimization(
4167 SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
4168 const SmallVectorImpl<ISD::OutputArg> &Outs,
4169 const SmallVectorImpl<SDValue> &OutVals,
4170 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
4171 if (AMDGPU::isChainCC(CC: CalleeCC))
4172 return true;
4173
4174 if (!AMDGPU::mayTailCallThisCC(CC: CalleeCC))
4175 return false;
4176
4177 // For a divergent call target, we need to do a waterfall loop over the
4178 // possible callees which precludes us from using a simple jump.
4179 if (Callee->isDivergent())
4180 return false;
4181
4182 MachineFunction &MF = DAG.getMachineFunction();
4183 const Function &CallerF = MF.getFunction();
4184 CallingConv::ID CallerCC = CallerF.getCallingConv();
4185 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
4186 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
4187
4188 // Kernels aren't callable, and don't have a live in return address so it
4189 // doesn't make sense to do a tail call with entry functions.
4190 if (!CallerPreserved)
4191 return false;
4192
4193 bool CCMatch = CallerCC == CalleeCC;
4194
4195 if (DAG.getTarget().Options.GuaranteedTailCallOpt) {
4196 if (AMDGPU::canGuaranteeTCO(CC: CalleeCC) && CCMatch)
4197 return true;
4198 return false;
4199 }
4200
4201 // TODO: Can we handle var args?
4202 if (IsVarArg)
4203 return false;
4204
4205 for (const Argument &Arg : CallerF.args()) {
4206 if (Arg.hasByValAttr())
4207 return false;
4208 }
4209
4210 LLVMContext &Ctx = *DAG.getContext();
4211
4212 // Check that the call results are passed in the same way.
4213 if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C&: Ctx, Ins,
4214 CalleeFn: CCAssignFnForCall(CC: CalleeCC, IsVarArg),
4215 CallerFn: CCAssignFnForCall(CC: CallerCC, IsVarArg)))
4216 return false;
4217
4218 // The callee has to preserve all registers the caller needs to preserve.
4219 if (!CCMatch) {
4220 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
4221 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved))
4222 return false;
4223 }
4224
4225 // Nothing more to check if the callee is taking no arguments.
4226 if (Outs.empty())
4227 return true;
4228
4229 SmallVector<CCValAssign, 16> ArgLocs;
4230 CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx);
4231
4232 // FIXME: We are not allocating special input registers, so we will be
4233 // deciding based on incorrect register assignments.
4234 CCInfo.AnalyzeCallOperands(Outs, Fn: CCAssignFnForCall(CC: CalleeCC, IsVarArg));
4235
4236 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
4237 // If the stack arguments for this call do not fit into our own save area then
4238 // the call cannot be made tail.
4239 // TODO: Is this really necessary?
4240 if (CCInfo.getStackSize() > FuncInfo->getBytesInStackArgArea())
4241 return false;
4242
4243 for (const auto &[CCVA, ArgVal] : zip_equal(t&: ArgLocs, u: OutVals)) {
4244 // FIXME: What about inreg arguments that end up passed in memory?
4245 if (!CCVA.isRegLoc())
4246 continue;
4247
4248 // If we are passing an argument in an SGPR, and the value is divergent,
4249 // this call requires a waterfall loop.
4250 if (ArgVal->isDivergent() && TRI->isSGPRPhysReg(Reg: CCVA.getLocReg())) {
4251 LLVM_DEBUG(
4252 dbgs() << "Cannot tail call due to divergent outgoing argument in "
4253 << printReg(CCVA.getLocReg(), TRI) << '\n');
4254 return false;
4255 }
4256 }
4257
4258 const MachineRegisterInfo &MRI = MF.getRegInfo();
4259 return parametersInCSRMatch(MRI, CallerPreservedMask: CallerPreserved, ArgLocs, OutVals);
4260}
4261
4262bool SITargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
4263 if (!CI->isTailCall())
4264 return false;
4265
4266 const Function *ParentFn = CI->getFunction();
4267 if (AMDGPU::isEntryFunctionCC(CC: ParentFn->getCallingConv()))
4268 return false;
4269 return true;
4270}
4271
4272namespace {
4273// Chain calls have special arguments that we need to handle. These are
4274// tagging along at the end of the arguments list(s), after the SGPR and VGPR
4275// arguments (index 0 and 1 respectively).
4276enum ChainCallArgIdx {
4277 Exec = 2,
4278 Flags,
4279 NumVGPRs,
4280 FallbackExec,
4281 FallbackCallee
4282};
4283} // anonymous namespace
4284
4285// The wave scratch offset register is used as the global base pointer.
4286SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
4287 SmallVectorImpl<SDValue> &InVals) const {
4288 CallingConv::ID CallConv = CLI.CallConv;
4289 bool IsChainCallConv = AMDGPU::isChainCC(CC: CallConv);
4290
4291 SelectionDAG &DAG = CLI.DAG;
4292
4293 const SDLoc &DL = CLI.DL;
4294 SDValue Chain = CLI.Chain;
4295 SDValue Callee = CLI.Callee;
4296
4297 llvm::SmallVector<SDValue, 6> ChainCallSpecialArgs;
4298 bool UsesDynamicVGPRs = false;
4299 if (IsChainCallConv) {
4300 // The last arguments should be the value that we need to put in EXEC,
4301 // followed by the flags and any other arguments with special meanings.
4302 // Pop them out of CLI.Outs and CLI.OutVals before we do any processing so
4303 // we don't treat them like the "real" arguments.
4304 auto RequestedExecIt =
4305 llvm::find_if(Range&: CLI.Outs, P: [](const ISD::OutputArg &Arg) {
4306 return Arg.OrigArgIndex == 2;
4307 });
4308 assert(RequestedExecIt != CLI.Outs.end() && "No node for EXEC");
4309
4310 size_t SpecialArgsBeginIdx = RequestedExecIt - CLI.Outs.begin();
4311 CLI.OutVals.erase(CS: CLI.OutVals.begin() + SpecialArgsBeginIdx,
4312 CE: CLI.OutVals.end());
4313 CLI.Outs.erase(CS: RequestedExecIt, CE: CLI.Outs.end());
4314
4315 assert(CLI.Outs.back().OrigArgIndex < 2 &&
4316 "Haven't popped all the special args");
4317
4318 TargetLowering::ArgListEntry RequestedExecArg =
4319 CLI.Args[ChainCallArgIdx::Exec];
4320 if (!RequestedExecArg.Ty->isIntegerTy(BitWidth: Subtarget->getWavefrontSize()))
4321 return lowerUnhandledCall(CLI, InVals, Reason: "Invalid value for EXEC");
4322
4323 // Convert constants into TargetConstants, so they become immediate operands
4324 // instead of being selected into S_MOV.
4325 auto PushNodeOrTargetConstant = [&](TargetLowering::ArgListEntry Arg) {
4326 if (const auto *ArgNode = dyn_cast<ConstantSDNode>(Val&: Arg.Node)) {
4327 ChainCallSpecialArgs.push_back(Elt: DAG.getTargetConstant(
4328 Val: ArgNode->getAPIntValue(), DL, VT: ArgNode->getValueType(ResNo: 0)));
4329 } else
4330 ChainCallSpecialArgs.push_back(Elt: Arg.Node);
4331 };
4332
4333 PushNodeOrTargetConstant(RequestedExecArg);
4334
4335 // Process any other special arguments depending on the value of the flags.
4336 TargetLowering::ArgListEntry Flags = CLI.Args[ChainCallArgIdx::Flags];
4337
4338 const APInt &FlagsValue = cast<ConstantSDNode>(Val&: Flags.Node)->getAPIntValue();
4339 if (FlagsValue.isZero()) {
4340 if (CLI.Args.size() > ChainCallArgIdx::Flags + 1)
4341 return lowerUnhandledCall(CLI, InVals,
4342 Reason: "no additional args allowed if flags == 0");
4343 } else if (FlagsValue.isOneBitSet(BitNo: 0)) {
4344 if (CLI.Args.size() != ChainCallArgIdx::FallbackCallee + 1) {
4345 return lowerUnhandledCall(CLI, InVals, Reason: "expected 3 additional args");
4346 }
4347
4348 if (!Subtarget->isWave32()) {
4349 return lowerUnhandledCall(
4350 CLI, InVals, Reason: "dynamic VGPR mode is only supported for wave32");
4351 }
4352
4353 UsesDynamicVGPRs = true;
4354 std::for_each(first: CLI.Args.begin() + ChainCallArgIdx::NumVGPRs,
4355 last: CLI.Args.end(), f: PushNodeOrTargetConstant);
4356 }
4357 }
4358
4359 SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
4360 SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
4361 SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
4362 bool &IsTailCall = CLI.IsTailCall;
4363 bool IsVarArg = CLI.IsVarArg;
4364 bool IsSibCall = false;
4365 MachineFunction &MF = DAG.getMachineFunction();
4366
4367 if (Callee.isUndef() || isNullConstant(V: Callee)) {
4368 if (!CLI.IsTailCall) {
4369 for (ISD::InputArg &Arg : CLI.Ins)
4370 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
4371 }
4372
4373 return Chain;
4374 }
4375
4376 if (IsVarArg) {
4377 return lowerUnhandledCall(CLI, InVals,
4378 Reason: "unsupported call to variadic function ");
4379 }
4380
4381 if (!CLI.CB)
4382 return lowerUnhandledCall(CLI, InVals, Reason: "unsupported libcall legalization");
4383
4384 if (IsTailCall && MF.getTarget().Options.GuaranteedTailCallOpt) {
4385 return lowerUnhandledCall(CLI, InVals,
4386 Reason: "unsupported required tail call to function ");
4387 }
4388
4389 if (IsTailCall) {
4390 IsTailCall = isEligibleForTailCallOptimization(Callee, CalleeCC: CallConv, IsVarArg,
4391 Outs, OutVals, Ins, DAG);
4392 if (!IsTailCall &&
4393 ((CLI.CB && CLI.CB->isMustTailCall()) || IsChainCallConv)) {
4394 report_fatal_error(reason: "failed to perform tail call elimination on a call "
4395 "site marked musttail or on llvm.amdgcn.cs.chain");
4396 }
4397
4398 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
4399
4400 // A sibling call is one where we're under the usual C ABI and not planning
4401 // to change that but can still do a tail call:
4402 if (!TailCallOpt && IsTailCall)
4403 IsSibCall = true;
4404
4405 if (IsTailCall)
4406 ++NumTailCalls;
4407 }
4408
4409 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
4410 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
4411 SmallVector<SDValue, 8> MemOpChains;
4412
4413 // Analyze operands of the call, assigning locations to each operand.
4414 SmallVector<CCValAssign, 16> ArgLocs;
4415 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
4416 CCAssignFn *AssignFn = CCAssignFnForCall(CC: CallConv, IsVarArg);
4417
4418 if (CallConv != CallingConv::AMDGPU_Gfx && !AMDGPU::isChainCC(CC: CallConv) &&
4419 CallConv != CallingConv::AMDGPU_Gfx_WholeWave) {
4420 // With a fixed ABI, allocate fixed registers before user arguments.
4421 passSpecialInputs(CLI, CCInfo, Info: *Info, RegsToPass, MemOpChains, Chain);
4422 }
4423
4424 // Mark the scratch resource descriptor as allocated so the CC analysis
4425 // does not assign user arguments to these registers, matching the callee.
4426 if (!Subtarget->hasFlatScratchEnabled())
4427 CCInfo.AllocateReg(Reg: Info->getScratchRSrcReg());
4428
4429 CCInfo.AnalyzeCallOperands(Outs, Fn: AssignFn);
4430
4431 // Get a count of how many bytes are to be pushed on the stack.
4432 unsigned NumBytes = CCInfo.getStackSize();
4433
4434 if (IsSibCall) {
4435 // Since we're not changing the ABI to make this a tail call, the memory
4436 // operands are already available in the caller's incoming argument space.
4437 NumBytes = 0;
4438 }
4439
4440 // FPDiff is the byte offset of the call's argument area from the callee's.
4441 // Stores to callee stack arguments will be placed in FixedStackSlots offset
4442 // by this amount for a tail call. In a sibling call it must be 0 because the
4443 // caller will deallocate the entire stack and the callee still expects its
4444 // arguments to begin at SP+0. Completely unused for non-tail calls.
4445 int32_t FPDiff = 0;
4446 MachineFrameInfo &MFI = MF.getFrameInfo();
4447 auto *TRI = Subtarget->getRegisterInfo();
4448
4449 // Adjust the stack pointer for the new arguments...
4450 // These operations are automatically eliminated by the prolog/epilog pass
4451 if (!IsSibCall)
4452 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL);
4453
4454 if (!IsSibCall || IsChainCallConv) {
4455 if (!Subtarget->hasFlatScratchEnabled()) {
4456 SmallVector<SDValue, 4> CopyFromChains;
4457
4458 // In the HSA case, this should be an identity copy.
4459 SDValue ScratchRSrcReg =
4460 DAG.getCopyFromReg(Chain, dl: DL, Reg: Info->getScratchRSrcReg(), VT: MVT::v4i32);
4461 RegsToPass.emplace_back(Args: IsChainCallConv
4462 ? AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51
4463 : AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3,
4464 Args&: ScratchRSrcReg);
4465 CopyFromChains.push_back(Elt: ScratchRSrcReg.getValue(R: 1));
4466 Chain = DAG.getTokenFactor(DL, Vals&: CopyFromChains);
4467 }
4468 }
4469
4470 const unsigned NumSpecialInputs = RegsToPass.size();
4471
4472 MVT PtrVT = MVT::i32;
4473
4474 // Walk the register/memloc assignments, inserting copies/loads.
4475 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4476 CCValAssign &VA = ArgLocs[i];
4477 SDValue Arg = OutVals[i];
4478
4479 // Promote the value if needed.
4480 switch (VA.getLocInfo()) {
4481 case CCValAssign::Full:
4482 break;
4483 case CCValAssign::BCvt:
4484 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getLocVT(), Operand: Arg);
4485 break;
4486 case CCValAssign::ZExt:
4487 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4488 break;
4489 case CCValAssign::SExt:
4490 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4491 break;
4492 case CCValAssign::AExt:
4493 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4494 break;
4495 case CCValAssign::FPExt:
4496 Arg = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4497 break;
4498 default:
4499 llvm_unreachable("Unknown loc info!");
4500 }
4501
4502 if (VA.isRegLoc()) {
4503 RegsToPass.push_back(Elt: std::pair(VA.getLocReg(), Arg));
4504 } else {
4505 assert(VA.isMemLoc());
4506
4507 SDValue DstAddr;
4508 MachinePointerInfo DstInfo;
4509
4510 unsigned LocMemOffset = VA.getLocMemOffset();
4511 int32_t Offset = LocMemOffset;
4512
4513 SDValue PtrOff = DAG.getConstant(Val: Offset, DL, VT: PtrVT);
4514 MaybeAlign Alignment;
4515
4516 if (IsTailCall) {
4517 ISD::ArgFlagsTy Flags = Outs[i].Flags;
4518 unsigned OpSize = Flags.isByVal() ? Flags.getByValSize()
4519 : VA.getValVT().getStoreSize();
4520
4521 // FIXME: We can have better than the minimum byval required alignment.
4522 Alignment =
4523 Flags.isByVal()
4524 ? Flags.getNonZeroByValAlign()
4525 : commonAlignment(A: Subtarget->getStackAlignment(), Offset);
4526
4527 Offset = Offset + FPDiff;
4528 int FI = MFI.CreateFixedObject(Size: OpSize, SPOffset: Offset, IsImmutable: true);
4529
4530 DstAddr = DAG.getFrameIndex(FI, VT: PtrVT);
4531 DstInfo = MachinePointerInfo::getFixedStack(MF, FI);
4532
4533 // Make sure any stack arguments overlapping with where we're storing
4534 // are loaded before this eventual operation. Otherwise they'll be
4535 // clobbered.
4536
4537 // FIXME: Why is this really necessary? This seems to just result in a
4538 // lot of code to copy the stack and write them back to the same
4539 // locations, which are supposed to be immutable?
4540 Chain = addTokenForArgument(Chain, DAG, MFI, ClobberedFI: FI);
4541 } else {
4542 // Stores to the argument stack area are relative to the stack pointer.
4543 SDValue SP = DAG.getCopyFromReg(Chain, dl: DL, Reg: Info->getStackPtrOffsetReg(),
4544 VT: MVT::i32);
4545 DstAddr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SP, N2: PtrOff);
4546 DstInfo = MachinePointerInfo::getStack(MF, Offset: LocMemOffset);
4547 Alignment =
4548 commonAlignment(A: Subtarget->getStackAlignment(), Offset: LocMemOffset);
4549 }
4550
4551 if (Outs[i].Flags.isByVal()) {
4552 SDValue SizeNode =
4553 DAG.getConstant(Val: Outs[i].Flags.getByValSize(), DL, VT: MVT::i32);
4554 SDValue Cpy =
4555 DAG.getMemcpy(Chain, dl: DL, Dst: DstAddr, Src: Arg, Size: SizeNode,
4556 DstAlign: Outs[i].Flags.getNonZeroByValAlign(),
4557 SrcAlign: Outs[i].Flags.getNonZeroByValAlign(),
4558 /*isVol = */ false, /*AlwaysInline = */ true,
4559 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: DstInfo,
4560 SrcPtrInfo: MachinePointerInfo(AMDGPUAS::PRIVATE_ADDRESS));
4561
4562 MemOpChains.push_back(Elt: Cpy);
4563 } else {
4564 SDValue Store =
4565 DAG.getStore(Chain, dl: DL, Val: Arg, Ptr: DstAddr, PtrInfo: DstInfo, Alignment);
4566 MemOpChains.push_back(Elt: Store);
4567 }
4568 }
4569 }
4570
4571 if (!MemOpChains.empty())
4572 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
4573
4574 SDValue ReadFirstLaneID =
4575 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
4576
4577 SDValue TokenGlue;
4578 if (CLI.ConvergenceControlToken) {
4579 TokenGlue = DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL, VT: MVT::Glue,
4580 Operand: CLI.ConvergenceControlToken);
4581 }
4582
4583 // Build a sequence of copy-to-reg nodes chained together with token chain
4584 // and flag operands which copy the outgoing args into the appropriate regs.
4585 SDValue InGlue;
4586
4587 unsigned ArgIdx = 0;
4588 for (auto [Reg, Val] : RegsToPass) {
4589 if (ArgIdx++ >= NumSpecialInputs &&
4590 (IsChainCallConv || !Val->isDivergent()) && TRI->isSGPRPhysReg(Reg)) {
4591 // For chain calls, the inreg arguments are required to be
4592 // uniform. Speculatively Insert a readfirstlane in case we cannot prove
4593 // they are uniform.
4594 //
4595 // For other calls, if an inreg arguments is known to be uniform,
4596 // speculatively insert a readfirstlane in case it is in a VGPR.
4597 //
4598 // FIXME: We need to execute this in a waterfall loop if it is a divergent
4599 // value, so let that continue to produce invalid code.
4600
4601 SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Val});
4602 if (TokenGlue)
4603 ReadfirstlaneArgs.push_back(Elt: TokenGlue);
4604 Val = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Val.getValueType(),
4605 Ops: ReadfirstlaneArgs);
4606 }
4607
4608 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N: Val, Glue: InGlue);
4609 InGlue = Chain.getValue(R: 1);
4610 }
4611
4612 // We don't usually want to end the call-sequence here because we would tidy
4613 // the frame up *after* the call, however in the ABI-changing tail-call case
4614 // we've carefully laid out the parameters so that when sp is reset they'll be
4615 // in the correct location.
4616 if (IsTailCall && !IsSibCall) {
4617 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue: InGlue, DL);
4618 InGlue = Chain.getValue(R: 1);
4619 }
4620
4621 std::vector<SDValue> Ops({Chain});
4622
4623 // Add a redundant copy of the callee global which will not be legalized, as
4624 // we need direct access to the callee later.
4625 if (GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
4626 const GlobalValue *GV = GSD->getGlobal();
4627 Ops.push_back(x: Callee);
4628 Ops.push_back(x: DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i64));
4629 } else {
4630 if (IsTailCall) {
4631 // isEligibleForTailCallOptimization considered whether the call target is
4632 // divergent, but we may still end up with a uniform value in a VGPR.
4633 // Insert a readfirstlane just in case.
4634 SDValue ReadFirstLaneID =
4635 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
4636
4637 SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Callee});
4638 if (TokenGlue)
4639 ReadfirstlaneArgs.push_back(Elt: TokenGlue); // Wire up convergence token.
4640 Callee = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Callee.getValueType(),
4641 Ops: ReadfirstlaneArgs);
4642 }
4643
4644 Ops.push_back(x: Callee);
4645 Ops.push_back(x: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i64));
4646 }
4647
4648 if (IsTailCall) {
4649 // Each tail call may have to adjust the stack by a different amount, so
4650 // this information must travel along with the operation for eventual
4651 // consumption by emitEpilogue.
4652 Ops.push_back(x: DAG.getTargetConstant(Val: FPDiff, DL, VT: MVT::i32));
4653 }
4654
4655 if (IsChainCallConv)
4656 llvm::append_range(C&: Ops, R&: ChainCallSpecialArgs);
4657
4658 // Add argument registers to the end of the list so that they are known live
4659 // into the call.
4660 for (auto &[Reg, Val] : RegsToPass)
4661 Ops.push_back(x: DAG.getRegister(Reg, VT: Val.getValueType()));
4662
4663 // Add a register mask operand representing the call-preserved registers.
4664 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
4665 assert(Mask && "Missing call preserved mask for calling convention");
4666 Ops.push_back(x: DAG.getRegisterMask(RegMask: Mask));
4667
4668 if (SDValue Token = CLI.ConvergenceControlToken) {
4669 SmallVector<SDValue, 2> GlueOps;
4670 GlueOps.push_back(Elt: Token);
4671 if (InGlue)
4672 GlueOps.push_back(Elt: InGlue);
4673
4674 InGlue = SDValue(DAG.getMachineNode(Opcode: TargetOpcode::CONVERGENCECTRL_GLUE, dl: DL,
4675 VT: MVT::Glue, Ops: GlueOps),
4676 0);
4677 }
4678
4679 if (InGlue)
4680 Ops.push_back(x: InGlue);
4681
4682 // If we're doing a tall call, use a TC_RETURN here rather than an
4683 // actual call instruction.
4684 if (IsTailCall) {
4685 MFI.setHasTailCall();
4686 unsigned OPC = AMDGPUISD::TC_RETURN;
4687 switch (CallConv) {
4688 case CallingConv::AMDGPU_Gfx:
4689 OPC = AMDGPUISD::TC_RETURN_GFX;
4690 break;
4691 case CallingConv::AMDGPU_CS_Chain:
4692 case CallingConv::AMDGPU_CS_ChainPreserve:
4693 OPC = UsesDynamicVGPRs ? AMDGPUISD::TC_RETURN_CHAIN_DVGPR
4694 : AMDGPUISD::TC_RETURN_CHAIN;
4695 break;
4696 }
4697
4698 // If the caller is a whole wave function, we need to use a special opcode
4699 // so we can patch up EXEC.
4700 if (Info->isWholeWaveFunction())
4701 OPC = AMDGPUISD::TC_RETURN_GFX_WholeWave;
4702
4703 return DAG.getNode(Opcode: OPC, DL, VT: MVT::Other, Ops);
4704 }
4705
4706 // Returns a chain and a flag for retval copy to use.
4707 SDValue Call = DAG.getNode(Opcode: AMDGPUISD::CALL, DL, ResultTys: {MVT::Other, MVT::Glue}, Ops);
4708 Chain = Call.getValue(R: 0);
4709 InGlue = Call.getValue(R: 1);
4710
4711 uint64_t CalleePopBytes = NumBytes;
4712 Chain = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: CalleePopBytes, Glue: InGlue, DL);
4713 if (!Ins.empty())
4714 InGlue = Chain.getValue(R: 1);
4715
4716 // Handle result values, copying them out of physregs into vregs that we
4717 // return.
4718 return LowerCallResult(Chain, InGlue, CallConv, IsVarArg, Ins, DL, DAG,
4719 InVals, /*IsThisReturn=*/false, ThisVal: SDValue());
4720}
4721
4722// This is similar to the default implementation in ExpandDYNAMIC_STACKALLOC,
4723// except for:
4724// 1. Stack growth direction(default: downwards, AMDGPU: upwards), and
4725// 2. Scale size where, scale = wave-reduction(alloca-size) * wave-size
4726SDValue SITargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
4727 SelectionDAG &DAG) const {
4728 const MachineFunction &MF = DAG.getMachineFunction();
4729 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
4730
4731 SDLoc dl(Op);
4732 EVT VT = Op.getValueType();
4733 SDValue Chain = Op.getOperand(i: 0);
4734 Register SPReg = Info->getStackPtrOffsetReg();
4735
4736 // Chain the dynamic stack allocation so that it doesn't modify the stack
4737 // pointer when other instructions are using the stack.
4738 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
4739
4740 SDValue Size = Op.getOperand(i: 1);
4741 SDValue BaseAddr = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
4742 Align Alignment = cast<ConstantSDNode>(Val: Op.getOperand(i: 2))->getAlignValue();
4743
4744 const TargetFrameLowering *TFL = Subtarget->getFrameLowering();
4745 assert(TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp &&
4746 "Stack grows upwards for AMDGPU");
4747
4748 Chain = BaseAddr.getValue(R: 1);
4749 // When using flat-scratch, the stack offset is unscaled.
4750 const bool HasFlatScratch = Subtarget->hasFlatScratchEnabled();
4751 const unsigned WavefrontSizeLog2 = Subtarget->getWavefrontSizeLog2();
4752
4753 Align StackAlign = TFL->getStackAlign();
4754 if (Alignment > StackAlign) {
4755 uint64_t ScaledAlignment = Alignment.value()
4756 << (HasFlatScratch ? 0 : WavefrontSizeLog2);
4757 uint64_t StackAlignMask = ScaledAlignment - 1;
4758 SDValue TmpAddr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr,
4759 N2: DAG.getConstant(Val: StackAlignMask, DL: dl, VT));
4760 BaseAddr = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: TmpAddr,
4761 N2: DAG.getSignedConstant(Val: -ScaledAlignment, DL: dl, VT));
4762 }
4763
4764 assert(Size.getValueType() == MVT::i32 && "Size must be 32-bit");
4765 SDValue NewSP;
4766 if (isa<ConstantSDNode>(Val: Size)) {
4767 // Increase the stack pointer by the size of the alloca.
4768 // If not using flat-scratch, we have to scale the size by the wave-size.
4769 SDValue ScaledSize =
4770 HasFlatScratch
4771 ? Size
4772 : DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Size,
4773 N2: DAG.getConstant(Val: WavefrontSizeLog2, DL: dl, VT: MVT::i32));
4774 NewSP = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr, N2: ScaledSize); // Value
4775 } else {
4776 // For dynamic sized alloca, perform wave-wide reduction to get max of
4777 // alloca size(divergent), and then scale it (when not using flat-scratch)
4778 // by wave-size.
4779 SDValue WaveReduction =
4780 DAG.getTargetConstant(Val: Intrinsic::amdgcn_wave_reduce_umax, DL: dl, VT: MVT::i32);
4781 Size = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::i32, N1: WaveReduction,
4782 N2: Size, N3: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32));
4783 SDValue ScaledSize = Size;
4784 if (!HasFlatScratch) {
4785 ScaledSize =
4786 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Size,
4787 N2: DAG.getConstant(Val: WavefrontSizeLog2, DL: dl, VT: MVT::i32));
4788 }
4789 NewSP =
4790 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr, N2: ScaledSize); // Value in vgpr.
4791 SDValue ReadFirstLaneID =
4792 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: dl, VT: MVT::i32);
4793 NewSP = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::i32, N1: ReadFirstLaneID,
4794 N2: NewSP);
4795 }
4796
4797 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: NewSP); // Output chain
4798 SDValue CallSeqEnd = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
4799
4800 return DAG.getMergeValues(Ops: {BaseAddr, CallSeqEnd}, dl);
4801}
4802
4803SDValue SITargetLowering::LowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const {
4804 if (Op.getValueType() != MVT::i32)
4805 return Op; // Defer to cannot select error.
4806
4807 Register SP = getStackPointerRegisterToSaveRestore();
4808 SDLoc SL(Op);
4809
4810 SDValue CopyFromSP = DAG.getCopyFromReg(Chain: Op->getOperand(Num: 0), dl: SL, Reg: SP, VT: MVT::i32);
4811
4812 // Convert from wave uniform to swizzled vector address. This should protect
4813 // from any edge cases where the stacksave result isn't directly used with
4814 // stackrestore.
4815 SDValue VectorAddress =
4816 DAG.getNode(Opcode: AMDGPUISD::WAVE_ADDRESS, DL: SL, VT: MVT::i32, Operand: CopyFromSP);
4817 return DAG.getMergeValues(Ops: {VectorAddress, CopyFromSP.getValue(R: 1)}, dl: SL);
4818}
4819
4820SDValue SITargetLowering::lowerGET_ROUNDING(SDValue Op,
4821 SelectionDAG &DAG) const {
4822 SDLoc SL(Op);
4823 assert(Op.getValueType() == MVT::i32);
4824
4825 uint32_t BothRoundHwReg =
4826 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
4827 SDValue GetRoundBothImm = DAG.getTargetConstant(Val: BothRoundHwReg, DL: SL, VT: MVT::i32);
4828
4829 SDValue IntrinID =
4830 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_getreg, DL: SL, VT: MVT::i32);
4831 SDValue GetReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList: Op->getVTList(),
4832 N1: Op.getOperand(i: 0), N2: IntrinID, N3: GetRoundBothImm);
4833
4834 // There are two rounding modes, one for f32 and one for f64/f16. We only
4835 // report in the standard value range if both are the same.
4836 //
4837 // The raw values also differ from the expected FLT_ROUNDS values. Nearest
4838 // ties away from zero is not supported, and the other values are rotated by
4839 // 1.
4840 //
4841 // If the two rounding modes are not the same, report a target defined value.
4842
4843 // Mode register rounding mode fields:
4844 //
4845 // [1:0] Single-precision round mode.
4846 // [3:2] Double/Half-precision round mode.
4847 //
4848 // 0=nearest even; 1= +infinity; 2= -infinity, 3= toward zero.
4849 //
4850 // Hardware Spec
4851 // Toward-0 3 0
4852 // Nearest Even 0 1
4853 // +Inf 1 2
4854 // -Inf 2 3
4855 // NearestAway0 N/A 4
4856 //
4857 // We have to handle 16 permutations of a 4-bit value, so we create a 64-bit
4858 // table we can index by the raw hardware mode.
4859 //
4860 // (trunc (FltRoundConversionTable >> MODE.fp_round)) & 0xf
4861
4862 SDValue BitTable =
4863 DAG.getConstant(Val: AMDGPU::FltRoundConversionTable, DL: SL, VT: MVT::i64);
4864
4865 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4866 SDValue RoundModeTimesNumBits =
4867 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: GetReg, N2: Two);
4868
4869 // TODO: We could possibly avoid a 64-bit shift and use a simpler table if we
4870 // knew only one mode was demanded.
4871 SDValue TableValue =
4872 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i64, N1: BitTable, N2: RoundModeTimesNumBits);
4873 SDValue TruncTable = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: TableValue);
4874
4875 SDValue EntryMask = DAG.getConstant(Val: 0xf, DL: SL, VT: MVT::i32);
4876 SDValue TableEntry =
4877 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: TruncTable, N2: EntryMask);
4878
4879 // There's a gap in the 4-bit encoded table and actual enum values, so offset
4880 // if it's an extended value.
4881 SDValue Four = DAG.getConstant(Val: 4, DL: SL, VT: MVT::i32);
4882 SDValue IsStandardValue =
4883 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: TableEntry, RHS: Four, Cond: ISD::SETULT);
4884 SDValue EnumOffset = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: TableEntry, N2: Four);
4885 SDValue Result = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i32, N1: IsStandardValue,
4886 N2: TableEntry, N3: EnumOffset);
4887
4888 return DAG.getMergeValues(Ops: {Result, GetReg.getValue(R: 1)}, dl: SL);
4889}
4890
4891SDValue SITargetLowering::lowerSET_ROUNDING(SDValue Op,
4892 SelectionDAG &DAG) const {
4893 SDLoc SL(Op);
4894
4895 SDValue NewMode = Op.getOperand(i: 1);
4896 assert(NewMode.getValueType() == MVT::i32);
4897
4898 // Index a table of 4-bit entries mapping from the C FLT_ROUNDS values to the
4899 // hardware MODE.fp_round values.
4900 if (auto *ConstMode = dyn_cast<ConstantSDNode>(Val&: NewMode)) {
4901 uint32_t ClampedVal = std::min(
4902 a: static_cast<uint32_t>(ConstMode->getZExtValue()),
4903 b: static_cast<uint32_t>(AMDGPU::TowardZeroF32_TowardNegativeF64));
4904 NewMode = DAG.getConstant(
4905 Val: AMDGPU::decodeFltRoundToHWConversionTable(FltRounds: ClampedVal), DL: SL, VT: MVT::i32);
4906 } else {
4907 // If we know the input can only be one of the supported standard modes in
4908 // the range 0-3, we can use a simplified mapping to hardware values.
4909 KnownBits KB = DAG.computeKnownBits(Op: NewMode);
4910 const bool UseReducedTable = KB.countMinLeadingZeros() >= 30;
4911 // The supported standard values are 0-3. The extended values start at 8. We
4912 // need to offset by 4 if the value is in the extended range.
4913
4914 if (UseReducedTable) {
4915 // Truncate to the low 32-bits.
4916 SDValue BitTable = DAG.getConstant(
4917 Val: AMDGPU::FltRoundToHWConversionTable & 0xffff, DL: SL, VT: MVT::i32);
4918
4919 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4920 SDValue RoundModeTimesNumBits =
4921 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: NewMode, N2: Two);
4922
4923 NewMode =
4924 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: BitTable, N2: RoundModeTimesNumBits);
4925
4926 // TODO: SimplifyDemandedBits on the setreg source here can likely reduce
4927 // the table extracted bits into inline immediates.
4928 } else {
4929 // table_index = umin(value, value - 4)
4930 // MODE.fp_round = (bit_table >> (table_index << 2)) & 0xf
4931 SDValue BitTable =
4932 DAG.getConstant(Val: AMDGPU::FltRoundToHWConversionTable, DL: SL, VT: MVT::i64);
4933
4934 SDValue Four = DAG.getConstant(Val: 4, DL: SL, VT: MVT::i32);
4935 SDValue OffsetEnum = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: NewMode, N2: Four);
4936 SDValue IndexVal =
4937 DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewMode, N2: OffsetEnum);
4938
4939 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4940 SDValue RoundModeTimesNumBits =
4941 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: IndexVal, N2: Two);
4942
4943 SDValue TableValue =
4944 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i64, N1: BitTable, N2: RoundModeTimesNumBits);
4945 SDValue TruncTable = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: TableValue);
4946
4947 // No need to mask out the high bits since the setreg will ignore them
4948 // anyway.
4949 NewMode = TruncTable;
4950 }
4951
4952 // Insert a readfirstlane in case the value is a VGPR. We could do this
4953 // earlier and keep more operations scalar, but that interferes with
4954 // combining the source.
4955 SDValue ReadFirstLaneID =
4956 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: SL, VT: MVT::i32);
4957 NewMode = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
4958 N1: ReadFirstLaneID, N2: NewMode);
4959 }
4960
4961 // N.B. The setreg will be later folded into s_round_mode on supported
4962 // targets.
4963 SDValue IntrinID =
4964 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_setreg, DL: SL, VT: MVT::i32);
4965 uint32_t BothRoundHwReg =
4966 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
4967 SDValue RoundBothImm = DAG.getTargetConstant(Val: BothRoundHwReg, DL: SL, VT: MVT::i32);
4968
4969 SDValue SetReg =
4970 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VTList: Op->getVTList(), N1: Op.getOperand(i: 0),
4971 N2: IntrinID, N3: RoundBothImm, N4: NewMode);
4972
4973 return SetReg;
4974}
4975
4976SDValue SITargetLowering::lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const {
4977 if (Op->isDivergent() &&
4978 (!Subtarget->hasVmemPrefInsts() || !Op.getConstantOperandVal(i: 4)))
4979 // Cannot do I$ prefetch with divergent pointer.
4980 return SDValue();
4981
4982 switch (cast<MemSDNode>(Val&: Op)->getAddressSpace()) {
4983 case AMDGPUAS::FLAT_ADDRESS:
4984 case AMDGPUAS::GLOBAL_ADDRESS:
4985 case AMDGPUAS::CONSTANT_ADDRESS:
4986 break;
4987 case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
4988 if (Subtarget->hasSafeSmemPrefetch())
4989 break;
4990 [[fallthrough]];
4991 default:
4992 return SDValue();
4993 }
4994
4995 // I$ prefetch
4996 if (!Subtarget->hasSafeSmemPrefetch() && !Op.getConstantOperandVal(i: 4))
4997 return SDValue();
4998
4999 return Op;
5000}
5001
5002// Work around DAG legality rules only based on the result type.
5003SDValue SITargetLowering::lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
5004 bool IsStrict = Op.getOpcode() == ISD::STRICT_FP_EXTEND;
5005 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
5006 EVT SrcVT = Src.getValueType();
5007
5008 if (SrcVT.getScalarType() != MVT::bf16)
5009 return Op;
5010
5011 SDLoc SL(Op);
5012 SDValue BitCast =
5013 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: SrcVT.changeTypeToInteger(), Operand: Src);
5014
5015 EVT DstVT = Op.getValueType();
5016 if (IsStrict)
5017 llvm_unreachable("Need STRICT_BF16_TO_FP");
5018
5019 return DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SL, VT: DstVT, Operand: BitCast);
5020}
5021
5022SDValue SITargetLowering::lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const {
5023 SDLoc SL(Op);
5024 if (Op.getValueType() != MVT::i64)
5025 return Op;
5026
5027 uint32_t ModeHwReg =
5028 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 23);
5029 SDValue ModeHwRegImm = DAG.getTargetConstant(Val: ModeHwReg, DL: SL, VT: MVT::i32);
5030 uint32_t TrapHwReg =
5031 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: 0, Values: 5);
5032 SDValue TrapHwRegImm = DAG.getTargetConstant(Val: TrapHwReg, DL: SL, VT: MVT::i32);
5033
5034 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
5035 SDValue IntrinID =
5036 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_getreg, DL: SL, VT: MVT::i32);
5037 SDValue GetModeReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList,
5038 N1: Op.getOperand(i: 0), N2: IntrinID, N3: ModeHwRegImm);
5039 SDValue GetTrapReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList,
5040 N1: Op.getOperand(i: 0), N2: IntrinID, N3: TrapHwRegImm);
5041 SDValue TokenReg =
5042 DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: GetModeReg.getValue(R: 1),
5043 N2: GetTrapReg.getValue(R: 1));
5044
5045 SDValue CvtPtr =
5046 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: GetModeReg, N2: GetTrapReg);
5047 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
5048
5049 return DAG.getMergeValues(Ops: {Result, TokenReg}, dl: SL);
5050}
5051
5052SDValue SITargetLowering::lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const {
5053 SDLoc SL(Op);
5054 if (Op.getOperand(i: 1).getValueType() != MVT::i64)
5055 return Op;
5056
5057 SDValue Input = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
5058 SDValue NewModeReg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Input,
5059 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
5060 SDValue NewTrapReg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Input,
5061 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
5062
5063 SDValue ReadFirstLaneID =
5064 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: SL, VT: MVT::i32);
5065 NewModeReg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
5066 N1: ReadFirstLaneID, N2: NewModeReg);
5067 NewTrapReg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
5068 N1: ReadFirstLaneID, N2: NewTrapReg);
5069
5070 unsigned ModeHwReg =
5071 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 23);
5072 SDValue ModeHwRegImm = DAG.getTargetConstant(Val: ModeHwReg, DL: SL, VT: MVT::i32);
5073 unsigned TrapHwReg =
5074 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: 0, Values: 5);
5075 SDValue TrapHwRegImm = DAG.getTargetConstant(Val: TrapHwReg, DL: SL, VT: MVT::i32);
5076
5077 SDValue IntrinID =
5078 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_setreg, DL: SL, VT: MVT::i32);
5079 SDValue SetModeReg =
5080 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VT: MVT::Other, N1: Op.getOperand(i: 0),
5081 N2: IntrinID, N3: ModeHwRegImm, N4: NewModeReg);
5082 SDValue SetTrapReg =
5083 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VT: MVT::Other, N1: Op.getOperand(i: 0),
5084 N2: IntrinID, N3: TrapHwRegImm, N4: NewTrapReg);
5085 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: SetTrapReg, N2: SetModeReg);
5086}
5087
5088Register SITargetLowering::getRegisterByName(const char *RegName, LLT VT,
5089 const MachineFunction &MF) const {
5090 const Function &Fn = MF.getFunction();
5091
5092 Register Reg = StringSwitch<Register>(RegName)
5093 .Case(S: "m0", Value: AMDGPU::M0)
5094 .Case(S: "exec", Value: AMDGPU::EXEC)
5095 .Case(S: "exec_lo", Value: AMDGPU::EXEC_LO)
5096 .Case(S: "exec_hi", Value: AMDGPU::EXEC_HI)
5097 .Case(S: "flat_scratch", Value: AMDGPU::FLAT_SCR)
5098 .Case(S: "flat_scratch_lo", Value: AMDGPU::FLAT_SCR_LO)
5099 .Case(S: "flat_scratch_hi", Value: AMDGPU::FLAT_SCR_HI)
5100 .Default(Value: Register());
5101 if (!Reg)
5102 return Reg;
5103
5104 if (!Subtarget->hasFlatScrRegister() &&
5105 Subtarget->getRegisterInfo()->regsOverlap(RegA: Reg, RegB: AMDGPU::FLAT_SCR)) {
5106 Fn.getContext().emitError(ErrorStr: Twine("invalid register \"" + StringRef(RegName) +
5107 "\" for subtarget."));
5108 }
5109
5110 switch (Reg) {
5111 case AMDGPU::M0:
5112 case AMDGPU::EXEC_LO:
5113 case AMDGPU::EXEC_HI:
5114 case AMDGPU::FLAT_SCR_LO:
5115 case AMDGPU::FLAT_SCR_HI:
5116 if (VT.getSizeInBits() == 32)
5117 return Reg;
5118 break;
5119 case AMDGPU::EXEC:
5120 case AMDGPU::FLAT_SCR:
5121 if (VT.getSizeInBits() == 64)
5122 return Reg;
5123 break;
5124 default:
5125 llvm_unreachable("missing register type checking");
5126 }
5127
5128 report_fatal_error(
5129 reason: Twine("invalid type for register \"" + StringRef(RegName) + "\"."));
5130}
5131
5132// If kill is not the last instruction, split the block so kill is always a
5133// proper terminator.
5134MachineBasicBlock *
5135SITargetLowering::splitKillBlock(MachineInstr &MI,
5136 MachineBasicBlock *BB) const {
5137 MachineBasicBlock *SplitBB = BB->splitAt(SplitInst&: MI, /*UpdateLiveIns=*/true);
5138 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5139 MI.setDesc(TII->getKillTerminatorFromPseudo(Opcode: MI.getOpcode()));
5140 return SplitBB;
5141}
5142
5143// Split block \p MBB at \p MI, as to insert a loop. If \p InstInLoop is true,
5144// \p MI will be the only instruction in the loop body block. Otherwise, it will
5145// be the first instruction in the remainder block.
5146//
5147/// \returns { LoopBody, Remainder }
5148static std::pair<MachineBasicBlock *, MachineBasicBlock *>
5149splitBlockForLoop(MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop) {
5150 MachineFunction *MF = MBB.getParent();
5151 MachineBasicBlock::iterator I(&MI);
5152
5153 // To insert the loop we need to split the block. Move everything after this
5154 // point to a new block, and insert a new empty block between the two.
5155 MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
5156 MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
5157 MachineFunction::iterator MBBI(MBB);
5158 ++MBBI;
5159
5160 MF->insert(MBBI, MBB: LoopBB);
5161 MF->insert(MBBI, MBB: RemainderBB);
5162
5163 LoopBB->addSuccessor(Succ: LoopBB);
5164 LoopBB->addSuccessor(Succ: RemainderBB);
5165
5166 // Move the rest of the block into a new block.
5167 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
5168
5169 if (InstInLoop) {
5170 auto Next = std::next(x: I);
5171
5172 // Move instruction to loop body.
5173 LoopBB->splice(Where: LoopBB->begin(), Other: &MBB, From: I, To: Next);
5174
5175 // Move the rest of the block.
5176 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: Next, To: MBB.end());
5177 } else {
5178 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: I, To: MBB.end());
5179 }
5180
5181 MBB.addSuccessor(Succ: LoopBB);
5182
5183 return std::pair(LoopBB, RemainderBB);
5184}
5185
5186/// Insert \p MI into a BUNDLE with an S_WAITCNT 0 immediately following it.
5187void SITargetLowering::bundleInstWithWaitcnt(MachineInstr &MI) const {
5188 MachineBasicBlock *MBB = MI.getParent();
5189 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5190 auto I = MI.getIterator();
5191 auto E = std::next(x: I);
5192
5193 // clang-format off
5194 BuildMI(BB&: *MBB, I: E, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_WAITCNT))
5195 .addImm(Val: 0);
5196 // clang-format on
5197
5198 MIBundleBuilder Bundler(*MBB, I, E);
5199 finalizeBundle(MBB&: *MBB, FirstMI: Bundler.begin());
5200}
5201
5202MachineBasicBlock *
5203SITargetLowering::emitGWSMemViolTestLoop(MachineInstr &MI,
5204 MachineBasicBlock *BB) const {
5205 const DebugLoc &DL = MI.getDebugLoc();
5206
5207 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
5208
5209 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5210
5211 // Apparently kill flags are only valid if the def is in the same block?
5212 if (MachineOperand *Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::data0))
5213 Src->setIsKill(false);
5214
5215 auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, MBB&: *BB, InstInLoop: true);
5216
5217 MachineBasicBlock::iterator I = LoopBB->end();
5218
5219 const unsigned EncodedReg = AMDGPU::Hwreg::HwregEncoding::encode(
5220 Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: AMDGPU::Hwreg::OFFSET_MEM_VIOL, Values: 1);
5221
5222 // Clear TRAP_STS.MEM_VIOL
5223 BuildMI(BB&: *LoopBB, I: LoopBB->begin(), MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SETREG_IMM32_B32))
5224 .addImm(Val: 0)
5225 .addImm(Val: EncodedReg);
5226
5227 bundleInstWithWaitcnt(MI);
5228
5229 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5230
5231 // Load and check TRAP_STS.MEM_VIOL
5232 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: Reg)
5233 .addImm(Val: EncodedReg);
5234
5235 // FIXME: Do we need to use an isel pseudo that may clobber scc?
5236 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
5237 .addReg(RegNo: Reg, Flags: RegState::Kill)
5238 .addImm(Val: 0);
5239 // clang-format off
5240 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
5241 .addMBB(MBB: LoopBB);
5242 // clang-format on
5243
5244 return RemainderBB;
5245}
5246
5247// Do a v_movrels_b32 or v_movreld_b32 for each unique value of \p IdxReg in the
5248// wavefront. If the value is uniform and just happens to be in a VGPR, this
5249// will only do one iteration. In the worst case, this will loop 64 times.
5250//
5251// TODO: Just use v_readlane_b32 if we know the VGPR has a uniform value.
5252static MachineBasicBlock::iterator
5253emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI,
5254 MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
5255 const DebugLoc &DL, const MachineOperand &Idx,
5256 unsigned InitReg, unsigned ResultReg, unsigned PhiReg,
5257 unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode,
5258 Register &SGPRIdxReg) {
5259
5260 MachineFunction *MF = OrigBB.getParent();
5261 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5262 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5263 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
5264 MachineBasicBlock::iterator I = LoopBB.begin();
5265
5266 const TargetRegisterClass *BoolRC = TRI->getBoolRC();
5267 Register PhiExec = MRI.createVirtualRegister(RegClass: BoolRC);
5268 Register NewExec = MRI.createVirtualRegister(RegClass: BoolRC);
5269 Register CurrentIdxReg =
5270 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5271 Register CondReg = MRI.createVirtualRegister(RegClass: BoolRC);
5272
5273 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::PHI), DestReg: PhiReg)
5274 .addReg(RegNo: InitReg)
5275 .addMBB(MBB: &OrigBB)
5276 .addReg(RegNo: ResultReg)
5277 .addMBB(MBB: &LoopBB);
5278
5279 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::PHI), DestReg: PhiExec)
5280 .addReg(RegNo: InitSaveExecReg)
5281 .addMBB(MBB: &OrigBB)
5282 .addReg(RegNo: NewExec)
5283 .addMBB(MBB: &LoopBB);
5284
5285 // Read the next variant <- also loop target.
5286 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurrentIdxReg)
5287 .addReg(RegNo: Idx.getReg(), Flags: getUndefRegState(B: Idx.isUndef()));
5288
5289 // Compare the just read M0 value to all possible Idx values.
5290 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: CondReg)
5291 .addReg(RegNo: CurrentIdxReg)
5292 .addReg(RegNo: Idx.getReg(), Flags: {}, SubReg: Idx.getSubReg());
5293
5294 // Update EXEC, save the original EXEC value to VCC.
5295 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.AndSaveExecOpc), DestReg: NewExec)
5296 .addReg(RegNo: CondReg, Flags: RegState::Kill);
5297
5298 MRI.setSimpleHint(VReg: NewExec, PrefReg: CondReg);
5299
5300 if (UseGPRIdxMode) {
5301 if (Offset == 0) {
5302 SGPRIdxReg = CurrentIdxReg;
5303 } else {
5304 SGPRIdxReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
5305 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: SGPRIdxReg)
5306 .addReg(RegNo: CurrentIdxReg, Flags: RegState::Kill)
5307 .addImm(Val: Offset);
5308 }
5309 } else {
5310 // Move index from VCC into M0
5311 if (Offset == 0) {
5312 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
5313 .addReg(RegNo: CurrentIdxReg, Flags: RegState::Kill);
5314 } else {
5315 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: AMDGPU::M0)
5316 .addReg(RegNo: CurrentIdxReg, Flags: RegState::Kill)
5317 .addImm(Val: Offset);
5318 }
5319 }
5320
5321 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
5322 MachineInstr *InsertPt =
5323 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.XorTermOpc), DestReg: LMC.ExecReg)
5324 .addReg(RegNo: LMC.ExecReg)
5325 .addReg(RegNo: NewExec);
5326
5327 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
5328 // s_cbranch_scc0?
5329
5330 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
5331 // clang-format off
5332 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_EXECNZ))
5333 .addMBB(MBB: &LoopBB);
5334 // clang-format on
5335
5336 return InsertPt->getIterator();
5337}
5338
5339// This has slightly sub-optimal regalloc when the source vector is killed by
5340// the read. The register allocator does not understand that the kill is
5341// per-workitem, so is kept alive for the whole loop so we end up not re-using a
5342// subregister from it, using 1 more VGPR than necessary. This was saved when
5343// this was expanded after register allocation.
5344static MachineBasicBlock::iterator
5345loadM0FromVGPR(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI,
5346 unsigned InitResultReg, unsigned PhiReg, int Offset,
5347 bool UseGPRIdxMode, Register &SGPRIdxReg) {
5348 MachineFunction *MF = MBB.getParent();
5349 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5350 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5351 MachineRegisterInfo &MRI = MF->getRegInfo();
5352 const DebugLoc &DL = MI.getDebugLoc();
5353 MachineBasicBlock::iterator I(&MI);
5354
5355 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
5356 Register DstReg = MI.getOperand(i: 0).getReg();
5357 Register SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
5358 Register TmpExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
5359 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
5360
5361 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: TmpExec);
5362
5363 // Save the EXEC mask
5364 // clang-format off
5365 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: SaveExec)
5366 .addReg(RegNo: LMC.ExecReg);
5367 // clang-format on
5368
5369 auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, MBB, InstInLoop: false);
5370
5371 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5372
5373 auto InsPt = emitLoadM0FromVGPRLoop(TII, MRI, OrigBB&: MBB, LoopBB&: *LoopBB, DL, Idx: *Idx,
5374 InitReg: InitResultReg, ResultReg: DstReg, PhiReg, InitSaveExecReg: TmpExec,
5375 Offset, UseGPRIdxMode, SGPRIdxReg);
5376
5377 MachineBasicBlock *LandingPad = MF->CreateMachineBasicBlock();
5378 MachineFunction::iterator MBBI(LoopBB);
5379 ++MBBI;
5380 MF->insert(MBBI, MBB: LandingPad);
5381 LoopBB->removeSuccessor(Succ: RemainderBB);
5382 LandingPad->addSuccessor(Succ: RemainderBB);
5383 LoopBB->addSuccessor(Succ: LandingPad);
5384 MachineBasicBlock::iterator First = LandingPad->begin();
5385 // clang-format off
5386 BuildMI(BB&: *LandingPad, I: First, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
5387 .addReg(RegNo: SaveExec);
5388 // clang-format on
5389
5390 return InsPt;
5391}
5392
5393// Returns subreg index, offset
5394static std::pair<unsigned, int>
5395computeIndirectRegAndOffset(const SIRegisterInfo &TRI,
5396 const TargetRegisterClass *SuperRC, unsigned VecReg,
5397 int Offset) {
5398 int NumElts = TRI.getRegSizeInBits(RC: *SuperRC) / 32;
5399
5400 // Skip out of bounds offsets, or else we would end up using an undefined
5401 // register.
5402 if (Offset >= NumElts || Offset < 0)
5403 return std::pair(AMDGPU::sub0, Offset);
5404
5405 return std::pair(SIRegisterInfo::getSubRegFromChannel(Channel: Offset), 0);
5406}
5407
5408static void setM0ToIndexFromSGPR(const SIInstrInfo *TII,
5409 MachineRegisterInfo &MRI, MachineInstr &MI,
5410 int Offset) {
5411 MachineBasicBlock *MBB = MI.getParent();
5412 const DebugLoc &DL = MI.getDebugLoc();
5413 MachineBasicBlock::iterator I(&MI);
5414
5415 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5416
5417 assert(Idx->getReg() != AMDGPU::NoRegister);
5418
5419 if (Offset == 0) {
5420 // clang-format off
5421 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
5422 .add(MO: *Idx);
5423 // clang-format on
5424 } else {
5425 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: AMDGPU::M0)
5426 .add(MO: *Idx)
5427 .addImm(Val: Offset);
5428 }
5429}
5430
5431static Register getIndirectSGPRIdx(const SIInstrInfo *TII,
5432 MachineRegisterInfo &MRI, MachineInstr &MI,
5433 int Offset) {
5434 MachineBasicBlock *MBB = MI.getParent();
5435 const DebugLoc &DL = MI.getDebugLoc();
5436 MachineBasicBlock::iterator I(&MI);
5437
5438 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5439
5440 if (Offset == 0)
5441 return Idx->getReg();
5442
5443 Register Tmp = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5444 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: Tmp)
5445 .add(MO: *Idx)
5446 .addImm(Val: Offset);
5447 return Tmp;
5448}
5449
5450static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
5451 MachineBasicBlock &MBB,
5452 const GCNSubtarget &ST) {
5453 const SIInstrInfo *TII = ST.getInstrInfo();
5454 const SIRegisterInfo &TRI = TII->getRegisterInfo();
5455 MachineFunction *MF = MBB.getParent();
5456 MachineRegisterInfo &MRI = MF->getRegInfo();
5457
5458 Register Dst = MI.getOperand(i: 0).getReg();
5459 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5460 Register SrcReg = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src)->getReg();
5461 int Offset = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::offset)->getImm();
5462
5463 const TargetRegisterClass *VecRC = MRI.getRegClass(Reg: SrcReg);
5464 const TargetRegisterClass *IdxRC = MRI.getRegClass(Reg: Idx->getReg());
5465
5466 unsigned SubReg;
5467 std::tie(args&: SubReg, args&: Offset) =
5468 computeIndirectRegAndOffset(TRI, SuperRC: VecRC, VecReg: SrcReg, Offset);
5469
5470 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
5471
5472 // Check for a SGPR index.
5473 if (TII->getRegisterInfo().isSGPRClass(RC: IdxRC)) {
5474 MachineBasicBlock::iterator I(&MI);
5475 const DebugLoc &DL = MI.getDebugLoc();
5476
5477 if (UseGPRIdxMode) {
5478 // TODO: Look at the uses to avoid the copy. This may require rescheduling
5479 // to avoid interfering with other uses, so probably requires a new
5480 // optimization pass.
5481 Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
5482
5483 const MCInstrDesc &GPRIDXDesc =
5484 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: true);
5485 BuildMI(BB&: MBB, I, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5486 .addReg(RegNo: SrcReg)
5487 .addReg(RegNo: Idx)
5488 .addImm(Val: SubReg);
5489 } else {
5490 setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
5491
5492 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: Dst)
5493 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
5494 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5495 }
5496
5497 MI.eraseFromParent();
5498
5499 return &MBB;
5500 }
5501
5502 // Control flow needs to be inserted if indexing with a VGPR.
5503 const DebugLoc &DL = MI.getDebugLoc();
5504 MachineBasicBlock::iterator I(&MI);
5505
5506 Register PhiReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5507 Register InitReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5508
5509 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: InitReg);
5510
5511 Register SGPRIdxReg;
5512 auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitResultReg: InitReg, PhiReg, Offset,
5513 UseGPRIdxMode, SGPRIdxReg);
5514
5515 MachineBasicBlock *LoopBB = InsPt->getParent();
5516
5517 if (UseGPRIdxMode) {
5518 const MCInstrDesc &GPRIDXDesc =
5519 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: true);
5520
5521 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5522 .addReg(RegNo: SrcReg)
5523 .addReg(RegNo: SGPRIdxReg)
5524 .addImm(Val: SubReg);
5525 } else {
5526 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: Dst)
5527 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
5528 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5529 }
5530
5531 MI.eraseFromParent();
5532
5533 return LoopBB;
5534}
5535
5536static MachineBasicBlock *emitIndirectDst(MachineInstr &MI,
5537 MachineBasicBlock &MBB,
5538 const GCNSubtarget &ST) {
5539 const SIInstrInfo *TII = ST.getInstrInfo();
5540 const SIRegisterInfo &TRI = TII->getRegisterInfo();
5541 MachineFunction *MF = MBB.getParent();
5542 MachineRegisterInfo &MRI = MF->getRegInfo();
5543
5544 Register Dst = MI.getOperand(i: 0).getReg();
5545 const MachineOperand *SrcVec = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src);
5546 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5547 const MachineOperand *Val = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::val);
5548 int Offset = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::offset)->getImm();
5549 const TargetRegisterClass *VecRC = MRI.getRegClass(Reg: SrcVec->getReg());
5550 const TargetRegisterClass *IdxRC = MRI.getRegClass(Reg: Idx->getReg());
5551
5552 // This can be an immediate, but will be folded later.
5553 assert(Val->getReg());
5554
5555 unsigned SubReg;
5556 std::tie(args&: SubReg, args&: Offset) =
5557 computeIndirectRegAndOffset(TRI, SuperRC: VecRC, VecReg: SrcVec->getReg(), Offset);
5558 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
5559
5560 if (Idx->getReg() == AMDGPU::NoRegister) {
5561 MachineBasicBlock::iterator I(&MI);
5562 const DebugLoc &DL = MI.getDebugLoc();
5563
5564 assert(Offset == 0);
5565
5566 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: Dst)
5567 .add(MO: *SrcVec)
5568 .add(MO: *Val)
5569 .addImm(Val: SubReg);
5570
5571 MI.eraseFromParent();
5572 return &MBB;
5573 }
5574
5575 // Check for a SGPR index.
5576 if (TII->getRegisterInfo().isSGPRClass(RC: IdxRC)) {
5577 MachineBasicBlock::iterator I(&MI);
5578 const DebugLoc &DL = MI.getDebugLoc();
5579
5580 if (UseGPRIdxMode) {
5581 Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
5582
5583 const MCInstrDesc &GPRIDXDesc =
5584 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
5585 BuildMI(BB&: MBB, I, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5586 .addReg(RegNo: SrcVec->getReg())
5587 .add(MO: *Val)
5588 .addReg(RegNo: Idx)
5589 .addImm(Val: SubReg);
5590 } else {
5591 setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
5592
5593 const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
5594 VecSize: TRI.getRegSizeInBits(RC: *VecRC), EltSize: 32, IsSGPR: false);
5595 BuildMI(BB&: MBB, I, MIMD: DL, MCID: MovRelDesc, DestReg: Dst)
5596 .addReg(RegNo: SrcVec->getReg())
5597 .add(MO: *Val)
5598 .addImm(Val: SubReg);
5599 }
5600 MI.eraseFromParent();
5601 return &MBB;
5602 }
5603
5604 // Control flow needs to be inserted if indexing with a VGPR.
5605 if (Val->isReg())
5606 MRI.clearKillFlags(Reg: Val->getReg());
5607
5608 const DebugLoc &DL = MI.getDebugLoc();
5609
5610 Register PhiReg = MRI.createVirtualRegister(RegClass: VecRC);
5611
5612 Register SGPRIdxReg;
5613 auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitResultReg: SrcVec->getReg(), PhiReg, Offset,
5614 UseGPRIdxMode, SGPRIdxReg);
5615 MachineBasicBlock *LoopBB = InsPt->getParent();
5616
5617 if (UseGPRIdxMode) {
5618 const MCInstrDesc &GPRIDXDesc =
5619 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
5620
5621 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5622 .addReg(RegNo: PhiReg)
5623 .add(MO: *Val)
5624 .addReg(RegNo: SGPRIdxReg)
5625 .addImm(Val: SubReg);
5626 } else {
5627 const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
5628 VecSize: TRI.getRegSizeInBits(RC: *VecRC), EltSize: 32, IsSGPR: false);
5629 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: MovRelDesc, DestReg: Dst)
5630 .addReg(RegNo: PhiReg)
5631 .add(MO: *Val)
5632 .addImm(Val: SubReg);
5633 }
5634
5635 MI.eraseFromParent();
5636 return LoopBB;
5637}
5638
5639static MachineBasicBlock *expand64BitScalarArithmetic(MachineInstr &MI,
5640 MachineBasicBlock *BB) {
5641 // For targets older than GFX12, we emit a sequence of 32-bit operations.
5642 // For GFX12, we emit s_add_u64 and s_sub_u64.
5643 MachineFunction *MF = BB->getParent();
5644 const SIInstrInfo *TII = MF->getSubtarget<GCNSubtarget>().getInstrInfo();
5645 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5646 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
5647 const DebugLoc &DL = MI.getDebugLoc();
5648 MachineOperand &Dest = MI.getOperand(i: 0);
5649 MachineOperand &Src0 = MI.getOperand(i: 1);
5650 MachineOperand &Src1 = MI.getOperand(i: 2);
5651 bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
5652 if (ST.hasScalarAddSub64()) {
5653 unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64;
5654 // clang-format off
5655 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest.getReg())
5656 .add(MO: Src0)
5657 .add(MO: Src1);
5658 // clang-format on
5659 } else {
5660 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5661 const TargetRegisterClass *BoolRC = TRI->getBoolRC();
5662
5663 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5664 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5665
5666 MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
5667 MI, MRI, SuperReg: Src0, SuperRC: BoolRC, SubIdx: AMDGPU::sub0, SubRC: &AMDGPU::SReg_32RegClass);
5668 MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
5669 MI, MRI, SuperReg: Src0, SuperRC: BoolRC, SubIdx: AMDGPU::sub1, SubRC: &AMDGPU::SReg_32RegClass);
5670
5671 MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
5672 MI, MRI, SuperReg: Src1, SuperRC: BoolRC, SubIdx: AMDGPU::sub0, SubRC: &AMDGPU::SReg_32RegClass);
5673 MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
5674 MI, MRI, SuperReg: Src1, SuperRC: BoolRC, SubIdx: AMDGPU::sub1, SubRC: &AMDGPU::SReg_32RegClass);
5675
5676 unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
5677 unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
5678 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoOpc), DestReg: DestSub0).add(MO: Src0Sub0).add(MO: Src1Sub0);
5679 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: HiOpc), DestReg: DestSub1).add(MO: Src0Sub1).add(MO: Src1Sub1);
5680 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest.getReg())
5681 .addReg(RegNo: DestSub0)
5682 .addImm(Val: AMDGPU::sub0)
5683 .addReg(RegNo: DestSub1)
5684 .addImm(Val: AMDGPU::sub1);
5685 }
5686 MI.eraseFromParent();
5687 return BB;
5688}
5689
5690static void expand64BitV_CNDMASK(MachineInstr &MI, MachineBasicBlock *BB) {
5691 MachineFunction *MF = BB->getParent();
5692 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5693 const SIInstrInfo *TII = ST.getInstrInfo();
5694 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5695 MachineRegisterInfo &MRI = MF->getRegInfo();
5696 const DebugLoc &DL = MI.getDebugLoc();
5697 Register Dst = MI.getOperand(i: 0).getReg();
5698 const MachineOperand &Src0 = MI.getOperand(i: 1);
5699 const MachineOperand &Src1 = MI.getOperand(i: 2);
5700 Register SrcCond = MI.getOperand(i: 3).getReg();
5701
5702 Register DstLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5703 Register DstHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5704 const TargetRegisterClass *CondRC = TRI->getWaveMaskRegClass();
5705 Register SrcCondCopy = MRI.createVirtualRegister(RegClass: CondRC);
5706
5707 int Src0Idx =
5708 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
5709 int Src1Idx =
5710 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src1);
5711 const TargetRegisterClass *Src0RC =
5712 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: Src0Idx));
5713 const TargetRegisterClass *Src1RC =
5714 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: Src1Idx));
5715
5716 const TargetRegisterClass *Src0SubRC =
5717 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
5718 const TargetRegisterClass *Src1SubRC =
5719 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
5720
5721 MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
5722 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
5723 MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
5724 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
5725
5726 MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
5727 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
5728 MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
5729 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
5730
5731 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: SrcCondCopy).addReg(RegNo: SrcCond);
5732 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstLo)
5733 .addImm(Val: 0)
5734 .add(MO: Src0Sub0)
5735 .addImm(Val: 0)
5736 .add(MO: Src1Sub0)
5737 .addReg(RegNo: SrcCondCopy);
5738
5739 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstHi)
5740 .addImm(Val: 0)
5741 .add(MO: Src0Sub1)
5742 .addImm(Val: 0)
5743 .add(MO: Src1Sub1)
5744 .addReg(RegNo: SrcCondCopy);
5745
5746 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
5747 .addReg(RegNo: DstLo)
5748 .addImm(Val: AMDGPU::sub0)
5749 .addReg(RegNo: DstHi)
5750 .addImm(Val: AMDGPU::sub1);
5751 MI.eraseFromParent();
5752}
5753
5754static uint64_t getIdentityValueForWaveReduction(unsigned Opc) {
5755 switch (Opc) {
5756 case AMDGPU::S_MIN_U32:
5757 return std::numeric_limits<uint32_t>::max();
5758 case AMDGPU::S_MIN_I32:
5759 return std::numeric_limits<int32_t>::max();
5760 case AMDGPU::S_MAX_U32:
5761 return std::numeric_limits<uint32_t>::min();
5762 case AMDGPU::S_MAX_I32:
5763 return std::numeric_limits<int32_t>::min();
5764 case AMDGPU::V_ADD_F32_e64: // -0.0
5765 return 0x80000000;
5766 case AMDGPU::V_SUB_F32_e64: // +0.0
5767 return 0x0;
5768 case AMDGPU::S_ADD_I32:
5769 case AMDGPU::S_SUB_I32:
5770 case AMDGPU::S_OR_B32:
5771 case AMDGPU::S_XOR_B32:
5772 return std::numeric_limits<uint32_t>::min();
5773 case AMDGPU::S_AND_B32:
5774 return std::numeric_limits<uint32_t>::max();
5775 case AMDGPU::V_MIN_F32_e64:
5776 case AMDGPU::V_MAX_F32_e64:
5777 return 0x7fc00000; // qNAN
5778 case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
5779 return std::numeric_limits<uint64_t>::max();
5780 case AMDGPU::V_CMP_LT_I64_e64: // min.i64
5781 return std::numeric_limits<int64_t>::max();
5782 case AMDGPU::V_CMP_GT_U64_e64: // umax.u64
5783 return std::numeric_limits<uint64_t>::min();
5784 case AMDGPU::V_CMP_GT_I64_e64: // max.i64
5785 return std::numeric_limits<int64_t>::min();
5786 case AMDGPU::V_MIN_F64_e64:
5787 case AMDGPU::V_MAX_F64_e64:
5788 case AMDGPU::V_MIN_NUM_F64_e64:
5789 case AMDGPU::V_MAX_NUM_F64_e64:
5790 return 0x7FF8000000000000; // qNAN
5791 case AMDGPU::S_ADD_U64_PSEUDO:
5792 case AMDGPU::S_SUB_U64_PSEUDO:
5793 case AMDGPU::S_OR_B64:
5794 case AMDGPU::S_XOR_B64:
5795 return std::numeric_limits<uint64_t>::min();
5796 case AMDGPU::S_AND_B64:
5797 return std::numeric_limits<uint64_t>::max();
5798 case AMDGPU::V_ADD_F64_e64:
5799 case AMDGPU::V_ADD_F64_pseudo_e64:
5800 return 0x8000000000000000; // -0.0
5801 default:
5802 llvm_unreachable("Unexpected opcode in getIdentityValueForWaveReduction");
5803 }
5804}
5805
5806static bool is32bitWaveReduceOperation(unsigned Opc) {
5807 return Opc == AMDGPU::S_MIN_U32 || Opc == AMDGPU::S_MIN_I32 ||
5808 Opc == AMDGPU::S_MAX_U32 || Opc == AMDGPU::S_MAX_I32 ||
5809 Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32 ||
5810 Opc == AMDGPU::S_AND_B32 || Opc == AMDGPU::S_OR_B32 ||
5811 Opc == AMDGPU::S_XOR_B32 || Opc == AMDGPU::V_MIN_F32_e64 ||
5812 Opc == AMDGPU::V_MAX_F32_e64 || Opc == AMDGPU::V_ADD_F32_e64 ||
5813 Opc == AMDGPU::V_SUB_F32_e64;
5814}
5815
5816static bool isFloatingPointWaveReduceOperation(unsigned Opc) {
5817 return Opc == AMDGPU::V_MIN_F32_e64 || Opc == AMDGPU::V_MAX_F32_e64 ||
5818 Opc == AMDGPU::V_ADD_F32_e64 || Opc == AMDGPU::V_SUB_F32_e64 ||
5819 Opc == AMDGPU::V_MIN_F64_e64 || Opc == AMDGPU::V_MAX_F64_e64 ||
5820 Opc == AMDGPU::V_MIN_NUM_F64_e64 || Opc == AMDGPU::V_MAX_NUM_F64_e64 ||
5821 Opc == AMDGPU::V_ADD_F64_e64 || Opc == AMDGPU::V_ADD_F64_pseudo_e64;
5822}
5823
5824static std::tuple<unsigned, unsigned>
5825getDPPOpcForWaveReduction(unsigned Opc, const GCNSubtarget &ST) {
5826 unsigned DPPOpc;
5827 switch (Opc) {
5828 case AMDGPU::S_MIN_U32:
5829 DPPOpc = AMDGPU::V_MIN_U32_dpp;
5830 break;
5831 case AMDGPU::S_MIN_I32:
5832 DPPOpc = AMDGPU::V_MIN_I32_dpp;
5833 break;
5834 case AMDGPU::S_MAX_U32:
5835 DPPOpc = AMDGPU::V_MAX_U32_dpp;
5836 break;
5837 case AMDGPU::S_MAX_I32:
5838 DPPOpc = AMDGPU::V_MAX_I32_dpp;
5839 break;
5840 case AMDGPU::S_ADD_I32:
5841 case AMDGPU::S_SUB_I32:
5842 DPPOpc = ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_dpp
5843 : AMDGPU::V_ADD_CO_U32_dpp;
5844 break;
5845 case AMDGPU::S_AND_B32:
5846 DPPOpc = AMDGPU::V_AND_B32_dpp;
5847 break;
5848 case AMDGPU::S_OR_B32:
5849 DPPOpc = AMDGPU::V_OR_B32_dpp;
5850 break;
5851 case AMDGPU::S_XOR_B32:
5852 DPPOpc = AMDGPU::V_XOR_B32_dpp;
5853 break;
5854 case AMDGPU::V_ADD_F32_e64:
5855 case AMDGPU::V_SUB_F32_e64:
5856 DPPOpc = AMDGPU::V_ADD_F32_dpp;
5857 break;
5858 case AMDGPU::V_MIN_F32_e64:
5859 DPPOpc = AMDGPU::V_MIN_F32_dpp;
5860 break;
5861 case AMDGPU::V_MAX_F32_e64:
5862 DPPOpc = AMDGPU::V_MAX_F32_dpp;
5863 break;
5864 case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
5865 case AMDGPU::V_CMP_LT_I64_e64: // min.i64
5866 case AMDGPU::V_CMP_GT_U64_e64: // umax.u64
5867 case AMDGPU::V_CMP_GT_I64_e64: // max.i64
5868 case AMDGPU::S_ADD_U64_PSEUDO:
5869 case AMDGPU::S_SUB_U64_PSEUDO:
5870 case AMDGPU::S_AND_B64:
5871 case AMDGPU::S_OR_B64:
5872 case AMDGPU::S_XOR_B64:
5873 case AMDGPU::V_MIN_NUM_F64_e64:
5874 case AMDGPU::V_MIN_F64_e64:
5875 case AMDGPU::V_MAX_NUM_F64_e64:
5876 case AMDGPU::V_MAX_F64_e64:
5877 case AMDGPU::V_ADD_F64_pseudo_e64:
5878 case AMDGPU::V_ADD_F64_e64:
5879 DPPOpc = AMDGPU::V_MOV_B64_DPP_PSEUDO;
5880 break;
5881 default:
5882 llvm_unreachable("unhandled lane op");
5883 }
5884 unsigned ClampOpc = Opc;
5885 if (!ST.getInstrInfo()->isVALU(Opcode: Opc, /*AllowLDSDMA=*/true)) {
5886 if (Opc == AMDGPU::S_SUB_I32)
5887 ClampOpc = AMDGPU::S_ADD_I32;
5888 if (Opc == AMDGPU::S_ADD_U64_PSEUDO || Opc == AMDGPU::S_SUB_U64_PSEUDO)
5889 ClampOpc = AMDGPU::V_ADD_CO_U32_e64;
5890 else if (Opc == AMDGPU::S_AND_B64)
5891 ClampOpc = AMDGPU::V_AND_B32_e64;
5892 else if (Opc == AMDGPU::S_OR_B64)
5893 ClampOpc = AMDGPU::V_OR_B32_e64;
5894 else if (Opc == AMDGPU::S_XOR_B64)
5895 ClampOpc = AMDGPU::V_XOR_B32_e64;
5896 else
5897 ClampOpc = ST.getInstrInfo()->getVALUOp(Opc: ClampOpc);
5898 }
5899 return {DPPOpc, ClampOpc};
5900}
5901
5902static std::pair<Register, Register>
5903ExtractSubRegs(MachineInstr &MI, MachineOperand &Op,
5904 const TargetRegisterClass *SrcRC, const GCNSubtarget &ST,
5905 MachineRegisterInfo &MRI) {
5906 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5907 const SIInstrInfo *TII = ST.getInstrInfo();
5908 const TargetRegisterClass *SrcSubRC =
5909 TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
5910 Register Op1L =
5911 TII->buildExtractSubReg(MI, MRI, SuperReg: Op, SuperRC: SrcRC, SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
5912 Register Op1H =
5913 TII->buildExtractSubReg(MI, MRI, SuperReg: Op, SuperRC: SrcRC, SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
5914 return {Op1L, Op1H};
5915}
5916
5917static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
5918 MachineBasicBlock &BB,
5919 const GCNSubtarget &ST,
5920 unsigned Opc) {
5921 MachineRegisterInfo &MRI = BB.getParent()->getRegInfo();
5922 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5923 const DebugLoc &DL = MI.getDebugLoc();
5924 const SIInstrInfo *TII = ST.getInstrInfo();
5925
5926 // Reduction operations depend on whether the input operand is SGPR or VGPR.
5927 Register SrcReg = MI.getOperand(i: 1).getReg();
5928 bool isSGPR = TRI->isSGPRClass(RC: MRI.getRegClass(Reg: SrcReg));
5929 Register DstReg = MI.getOperand(i: 0).getReg();
5930 unsigned Stratergy = static_cast<unsigned>(MI.getOperand(i: 2).getImm());
5931 enum WAVE_REDUCE_STRATEGY : unsigned { DEFAULT = 0, ITERATIVE = 1, DPP = 2 };
5932 MachineBasicBlock *RetBB = nullptr;
5933 unsigned MIOpc = MI.getOpcode();
5934 auto BuildRegSequence = [&](MachineBasicBlock &BB,
5935 MachineBasicBlock::iterator MI, Register Dst,
5936 Register Src0, Register Src1) {
5937 auto RegSequence =
5938 BuildMI(BB, I: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dst)
5939 .addReg(RegNo: Src0)
5940 .addImm(Val: AMDGPU::sub0)
5941 .addReg(RegNo: Src1)
5942 .addImm(Val: AMDGPU::sub1);
5943 return RegSequence;
5944 };
5945 if (isSGPR) {
5946 switch (Opc) {
5947 case AMDGPU::S_MIN_U32:
5948 case AMDGPU::S_MIN_I32:
5949 case AMDGPU::V_MIN_F32_e64:
5950 case AMDGPU::S_MAX_U32:
5951 case AMDGPU::S_MAX_I32:
5952 case AMDGPU::V_MAX_F32_e64:
5953 case AMDGPU::S_AND_B32:
5954 case AMDGPU::S_OR_B32: {
5955 // Idempotent operations.
5956 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstReg).addReg(RegNo: SrcReg);
5957 RetBB = &BB;
5958 break;
5959 }
5960 case AMDGPU::V_CMP_LT_U64_e64: // umin
5961 case AMDGPU::V_CMP_LT_I64_e64: // min
5962 case AMDGPU::V_CMP_GT_U64_e64: // umax
5963 case AMDGPU::V_CMP_GT_I64_e64: // max
5964 case AMDGPU::V_MIN_F64_e64:
5965 case AMDGPU::V_MIN_NUM_F64_e64:
5966 case AMDGPU::V_MAX_F64_e64:
5967 case AMDGPU::V_MAX_NUM_F64_e64:
5968 case AMDGPU::S_AND_B64:
5969 case AMDGPU::S_OR_B64: {
5970 // Idempotent operations.
5971 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B64), DestReg: DstReg).addReg(RegNo: SrcReg);
5972 RetBB = &BB;
5973 break;
5974 }
5975 case AMDGPU::S_XOR_B32:
5976 case AMDGPU::S_XOR_B64:
5977 case AMDGPU::S_ADD_I32:
5978 case AMDGPU::S_ADD_U64_PSEUDO:
5979 case AMDGPU::V_ADD_F32_e64:
5980 case AMDGPU::V_ADD_F64_e64:
5981 case AMDGPU::V_ADD_F64_pseudo_e64:
5982 case AMDGPU::S_SUB_I32:
5983 case AMDGPU::S_SUB_U64_PSEUDO:
5984 case AMDGPU::V_SUB_F32_e64: {
5985 const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
5986 const TargetRegisterClass *DstRegClass = MRI.getRegClass(Reg: DstReg);
5987 Register ExecMask = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
5988 Register NumActiveLanes =
5989 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5990
5991 bool IsWave32 = ST.isWave32();
5992 unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
5993 MCRegister ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
5994 unsigned BitCountOpc =
5995 IsWave32 ? AMDGPU::S_BCNT1_I32_B32 : AMDGPU::S_BCNT1_I32_B64;
5996
5997 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MovOpc), DestReg: ExecMask).addReg(RegNo: ExecReg);
5998
5999 auto NewAccumulator =
6000 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BitCountOpc), DestReg: NumActiveLanes)
6001 .addReg(RegNo: ExecMask);
6002
6003 switch (Opc) {
6004 case AMDGPU::S_XOR_B32:
6005 case AMDGPU::S_XOR_B64: {
6006 // Performing an XOR operation on a uniform value
6007 // depends on the parity of the number of active lanes.
6008 // For even parity, the result will be 0, for odd
6009 // parity the result will be the same as the input value.
6010 Register ParityRegister =
6011 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6012
6013 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_AND_B32), DestReg: ParityRegister)
6014 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
6015 .addImm(Val: 1)
6016 .setOperandDead(3); // Dead scc
6017 if (Opc == AMDGPU::S_XOR_B32) {
6018 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
6019 .addReg(RegNo: SrcReg)
6020 .addReg(RegNo: ParityRegister);
6021 } else {
6022 Register DestSub0 =
6023 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6024 Register DestSub1 =
6025 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6026 auto [Op1L, Op1H] = ExtractSubRegs(MI, Op&: MI.getOperand(i: 1),
6027 SrcRC: MRI.getRegClass(Reg: SrcReg), ST, MRI);
6028 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub0)
6029 .addReg(RegNo: Op1L)
6030 .addReg(RegNo: ParityRegister);
6031 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub1)
6032 .addReg(RegNo: Op1H)
6033 .addReg(RegNo: ParityRegister);
6034 BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
6035 }
6036 break;
6037 }
6038 case AMDGPU::S_SUB_I32: {
6039 Register NegatedVal = MRI.createVirtualRegister(RegClass: DstRegClass);
6040
6041 // Take the negation of the source operand.
6042 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedVal)
6043 .addImm(Val: 0)
6044 .addReg(RegNo: SrcReg);
6045 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
6046 .addReg(RegNo: NegatedVal)
6047 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg());
6048 break;
6049 }
6050 case AMDGPU::S_ADD_I32: {
6051 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
6052 .addReg(RegNo: SrcReg)
6053 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg());
6054 break;
6055 }
6056 case AMDGPU::S_ADD_U64_PSEUDO:
6057 case AMDGPU::S_SUB_U64_PSEUDO: {
6058 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6059 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6060 Register Op1H_Op0L_Reg =
6061 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6062 Register Op1L_Op0H_Reg =
6063 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6064 Register CarryReg =
6065 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6066 Register AddReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6067 Register NegatedValLo =
6068 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6069 Register NegatedValHi =
6070 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6071 auto [Op1L, Op1H] = ExtractSubRegs(MI, Op&: MI.getOperand(i: 1),
6072 SrcRC: MRI.getRegClass(Reg: SrcReg), ST, MRI);
6073 if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6074 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedValLo)
6075 .addImm(Val: 0)
6076 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
6077 .setOperandDead(3); // Dead scc
6078 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ASHR_I32), DestReg: NegatedValHi)
6079 .addReg(RegNo: NegatedValLo)
6080 .addImm(Val: 31)
6081 .setOperandDead(3); // Dead scc
6082 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: Op1L_Op0H_Reg)
6083 .addReg(RegNo: Op1L)
6084 .addReg(RegNo: NegatedValHi);
6085 }
6086 Register LowOpcode = Opc == AMDGPU::S_SUB_U64_PSEUDO
6087 ? NegatedValLo
6088 : NewAccumulator->getOperand(i: 0).getReg();
6089 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub0)
6090 .addReg(RegNo: Op1L)
6091 .addReg(RegNo: LowOpcode);
6092 if (ST.hasScalarMulHiInsts()) {
6093 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_HI_U32), DestReg: CarryReg)
6094 .addReg(RegNo: Op1L)
6095 .addReg(RegNo: LowOpcode);
6096 } else {
6097 Register VCarryReg =
6098 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6099 Register LowOpVGPR =
6100 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6101 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: LowOpVGPR)
6102 .addReg(RegNo: LowOpcode);
6103 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MUL_HI_U32_e64), DestReg: VCarryReg)
6104 .addReg(RegNo: Op1L)
6105 .addReg(RegNo: LowOpVGPR);
6106 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CarryReg)
6107 .addReg(RegNo: VCarryReg);
6108 }
6109 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: Op1H_Op0L_Reg)
6110 .addReg(RegNo: Op1H)
6111 .addReg(RegNo: LowOpcode);
6112
6113 Register HiVal = Opc == AMDGPU::S_SUB_U64_PSEUDO ? AddReg : DestSub1;
6114 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: HiVal)
6115 .addReg(RegNo: CarryReg)
6116 .addReg(RegNo: Op1H_Op0L_Reg)
6117 .setOperandDead(3); // Dead scc
6118
6119 if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6120 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: DestSub1)
6121 .addReg(RegNo: HiVal)
6122 .addReg(RegNo: Op1L_Op0H_Reg)
6123 .setOperandDead(3); // Dead scc
6124 }
6125 BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
6126 break;
6127 }
6128 case AMDGPU::V_ADD_F32_e64:
6129 case AMDGPU::V_ADD_F64_e64:
6130 case AMDGPU::V_ADD_F64_pseudo_e64:
6131 case AMDGPU::V_SUB_F32_e64: {
6132 bool is32BitOpc = is32bitWaveReduceOperation(Opc);
6133 const TargetRegisterClass *VregRC = TII->getRegClass(MCID: TII->get(Opcode: Opc), OpNum: 0);
6134 Register ActiveLanesVreg = MRI.createVirtualRegister(RegClass: VregRC);
6135 Register DstVreg = MRI.createVirtualRegister(RegClass: VregRC);
6136 // Get number of active lanes as a float val.
6137 BuildMI(BB, I&: MI, MIMD: DL,
6138 MCID: TII->get(Opcode: is32BitOpc ? AMDGPU::V_CVT_F32_I32_e64
6139 : AMDGPU::V_CVT_F64_I32_e64),
6140 DestReg: ActiveLanesVreg)
6141 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
6142 .addImm(Val: 0) // clamp
6143 .addImm(Val: 0); // output-modifier
6144
6145 // Take negation of input for SUB reduction
6146 unsigned srcMod = (MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32 ||
6147 MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64)
6148 ? SISrcMods::NEG
6149 : SISrcMods::NONE;
6150 unsigned MulOpc = is32BitOpc ? AMDGPU::V_MUL_F32_e64
6151 : ST.getGeneration() >= AMDGPUSubtarget::GFX12
6152 ? AMDGPU::V_MUL_F64_pseudo_e64
6153 : AMDGPU::V_MUL_F64_e64;
6154 auto DestVregInst = BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MulOpc),
6155 DestReg: DstVreg)
6156 .addImm(Val: srcMod) // src0 modifier
6157 .addReg(RegNo: SrcReg)
6158 .addImm(Val: SISrcMods::NONE) // src1 modifier
6159 .addReg(RegNo: ActiveLanesVreg)
6160 .addImm(Val: SISrcMods::NONE) // clamp
6161 .addImm(Val: SISrcMods::NONE); // output-mod
6162 if (is32BitOpc) {
6163 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
6164 .addReg(RegNo: DstVreg);
6165 } else {
6166 Register LaneValueLoReg =
6167 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6168 Register LaneValueHiReg =
6169 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6170 auto [Op1L, Op1H] =
6171 ExtractSubRegs(MI, Op&: DestVregInst->getOperand(i: 0), SrcRC: VregRC, ST, MRI);
6172 // lane value input should be in an sgpr
6173 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
6174 DestReg: LaneValueLoReg)
6175 .addReg(RegNo: Op1L);
6176 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
6177 DestReg: LaneValueHiReg)
6178 .addReg(RegNo: Op1H);
6179 NewAccumulator =
6180 BuildRegSequence(BB, MI, DstReg, LaneValueLoReg, LaneValueHiReg);
6181 }
6182 }
6183 }
6184 RetBB = &BB;
6185 }
6186 }
6187 } else {
6188 MachineBasicBlock::iterator I = BB.end();
6189 Register SrcReg = MI.getOperand(i: 1).getReg();
6190 bool is32BitOpc = is32bitWaveReduceOperation(Opc);
6191 bool isFPOp = isFloatingPointWaveReduceOperation(Opc);
6192 bool NeedsMovDPP = !is32BitOpc;
6193 // Create virtual registers required for lowering.
6194 const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
6195 const TargetRegisterClass *DstRegClass = MRI.getRegClass(Reg: DstReg);
6196 const TargetRegisterClass *SrcRegClass = MRI.getRegClass(Reg: SrcReg);
6197 bool IsWave32 = ST.isWave32();
6198 unsigned MovOpcForExec = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
6199 unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6200 if (Stratergy == WAVE_REDUCE_STRATEGY::ITERATIVE ||
6201 !ST.hasDPP()) { // If target doesn't support DPP operations, default to
6202 // iterative stratergy
6203
6204 // To reduce the VGPR using iterative approach, we need to iterate
6205 // over all the active lanes. Lowering consists of ComputeLoop,
6206 // which iterate over only active lanes. We use copy of EXEC register
6207 // as induction variable and every active lane modifies it using bitset0
6208 // so that we will get the next active lane for next iteration.
6209
6210 // Create Control flow for loop
6211 // Split MI's Machine Basic block into For loop
6212 auto [ComputeLoop, ComputeEnd] = splitBlockForLoop(MI, MBB&: BB, InstInLoop: true);
6213
6214 Register LoopIterator = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6215 Register IdentityValReg = MRI.createVirtualRegister(RegClass: DstRegClass);
6216 Register AccumulatorReg = MRI.createVirtualRegister(RegClass: DstRegClass);
6217 Register ActiveBitsReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6218 Register NewActiveBitsReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6219 Register FF1Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6220 Register LaneValueReg = MRI.createVirtualRegister(RegClass: DstRegClass);
6221
6222 // Create initial values of induction variable from Exec, Accumulator and
6223 // insert branch instr to newly created ComputeBlock
6224 BuildMI(BB, I, MIMD: DL, MCID: TII->get(Opcode: MovOpcForExec), DestReg: LoopIterator).addReg(RegNo: ExecReg);
6225 uint64_t IdentityValue =
6226 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
6227 ? 0x0 // +0.0 for double sub reduction
6228 : getIdentityValueForWaveReduction(Opc);
6229 BuildMI(BB, I, MIMD: DL,
6230 MCID: TII->get(Opcode: is32BitOpc ? AMDGPU::S_MOV_B32
6231 : AMDGPU::S_MOV_B64_IMM_PSEUDO),
6232 DestReg: IdentityValReg)
6233 .addImm(Val: IdentityValue);
6234 // clang-format off
6235 BuildMI(BB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_BRANCH))
6236 .addMBB(MBB: ComputeLoop);
6237 // clang-format on
6238
6239 // Start constructing ComputeLoop
6240 I = ComputeLoop->begin();
6241 auto Accumulator =
6242 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::PHI), DestReg: AccumulatorReg)
6243 .addReg(RegNo: IdentityValReg)
6244 .addMBB(MBB: &BB);
6245 auto ActiveBits =
6246 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::PHI), DestReg: ActiveBitsReg)
6247 .addReg(RegNo: LoopIterator)
6248 .addMBB(MBB: &BB);
6249
6250 I = ComputeLoop->end();
6251 MachineInstr *NewAccumulator;
6252 // Perform the computations
6253 unsigned SFFOpc =
6254 IsWave32 ? AMDGPU::S_FF1_I32_B32 : AMDGPU::S_FF1_I32_B64;
6255 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: SFFOpc), DestReg: FF1Reg)
6256 .addReg(RegNo: ActiveBitsReg);
6257 if (is32BitOpc) {
6258 Register OpDstReg = DstReg;
6259 bool hasSrc0Modifier = AMDGPU::getNamedOperandIdx(
6260 Opcode: Opc, Name: AMDGPU::OpName::src0_modifiers) != -1;
6261 bool hasSrc1Modifier = AMDGPU::getNamedOperandIdx(
6262 Opcode: Opc, Name: AMDGPU::OpName::src1_modifiers) != -1;
6263 bool hasClamp =
6264 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::clamp) != -1;
6265 bool hasOpSel =
6266 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel) != -1;
6267 bool hasOMod =
6268 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::omod) != -1;
6269 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6270 DestReg: LaneValueReg)
6271 .addReg(RegNo: SrcReg)
6272 .addReg(RegNo: FF1Reg);
6273 if (ST.getInstrInfo()->isVALU(Opcode: Opc, /*AllowLDSDMA=*/true)) {
6274 // Get the Lane Value in VGPR to avoid the Constant Bus Restriction
6275 Register LaneValVgpr = MRI.createVirtualRegister(RegClass: SrcRegClass);
6276 Register VgprResultReg = MRI.createVirtualRegister(RegClass: SrcRegClass);
6277 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: LaneValVgpr)
6278 .addReg(RegNo: LaneValueReg);
6279 OpDstReg = VgprResultReg;
6280 LaneValueReg = LaneValVgpr;
6281 }
6282 auto OpInstr = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: OpDstReg);
6283 if (hasSrc0Modifier)
6284 OpInstr.addImm(Val: SISrcMods::NONE); // src0 modifier
6285 OpInstr.addReg(RegNo: AccumulatorReg); // src0
6286 if (hasSrc1Modifier)
6287 OpInstr.addImm(Val: SISrcMods::NONE); // src1 modifier
6288 OpInstr.addReg(RegNo: LaneValueReg); // src1
6289 if (hasClamp)
6290 OpInstr.addImm(Val: 0); // clamp
6291 if (hasOpSel)
6292 OpInstr.addImm(Val: 0); // opsel
6293 if (hasOMod)
6294 OpInstr.addImm(Val: 0); // omod
6295 if (ST.getInstrInfo()->isVALU(Opcode: Opc, /*AllowLDSDMA=*/true)) {
6296 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
6297 DestReg: DstReg)
6298 .addReg(RegNo: OpDstReg);
6299 }
6300 } else {
6301 Register LaneValueLoReg =
6302 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6303 Register LaneValueHiReg =
6304 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6305 Register LaneValReg =
6306 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6307 auto [Op1L, Op1H] = ExtractSubRegs(MI, Op&: MI.getOperand(i: 1),
6308 SrcRC: MRI.getRegClass(Reg: SrcReg), ST, MRI);
6309 // lane value input should be in an sgpr
6310 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6311 DestReg: LaneValueLoReg)
6312 .addReg(RegNo: Op1L)
6313 .addReg(RegNo: FF1Reg);
6314 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6315 DestReg: LaneValueHiReg)
6316 .addReg(RegNo: Op1H)
6317 .addReg(RegNo: FF1Reg);
6318 auto LaneValue = BuildRegSequence(*ComputeLoop, I, LaneValReg,
6319 LaneValueLoReg, LaneValueHiReg);
6320 switch (Opc) {
6321 case AMDGPU::S_OR_B64:
6322 case AMDGPU::S_AND_B64:
6323 case AMDGPU::S_XOR_B64: {
6324 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstReg)
6325 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg())
6326 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6327 .setOperandDead(3); // Dead scc
6328 break;
6329 }
6330 case AMDGPU::V_CMP_GT_I64_e64:
6331 case AMDGPU::V_CMP_GT_U64_e64:
6332 case AMDGPU::V_CMP_LT_I64_e64:
6333 case AMDGPU::V_CMP_LT_U64_e64: {
6334 Register LaneMaskReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6335 Register ComparisonResultReg =
6336 MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6337 int SrcIdx =
6338 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src);
6339 const TargetRegisterClass *VregClass =
6340 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: SrcIdx));
6341 Register AccumulatorVReg = MRI.createVirtualRegister(RegClass: VregClass);
6342 auto [SrcReg0Sub0, SrcReg0Sub1] = ExtractSubRegs(
6343 MI, Op&: Accumulator->getOperand(i: 0), SrcRC: VregClass, ST, MRI);
6344 BuildRegSequence(*ComputeLoop, I, AccumulatorVReg, SrcReg0Sub0,
6345 SrcReg0Sub1);
6346 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: LaneMaskReg)
6347 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6348 .addReg(RegNo: AccumulatorVReg);
6349
6350 unsigned AndOpc = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
6351 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AndOpc), DestReg: ComparisonResultReg)
6352 .addReg(RegNo: LaneMaskReg)
6353 .addReg(RegNo: ActiveBitsReg);
6354
6355 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6356 MCID: TII->get(Opcode: AMDGPU::S_CSELECT_B64), DestReg: DstReg)
6357 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6358 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg());
6359 break;
6360 }
6361 case AMDGPU::V_MIN_F64_e64:
6362 case AMDGPU::V_MIN_NUM_F64_e64:
6363 case AMDGPU::V_MAX_F64_e64:
6364 case AMDGPU::V_MAX_NUM_F64_e64:
6365 case AMDGPU::V_ADD_F64_e64:
6366 case AMDGPU::V_ADD_F64_pseudo_e64: {
6367 int SrcIdx =
6368 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src);
6369 const TargetRegisterClass *VregRC =
6370 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: SrcIdx));
6371 Register AccumulatorVReg = MRI.createVirtualRegister(RegClass: VregRC);
6372 Register DstVreg = MRI.createVirtualRegister(RegClass: VregRC);
6373 Register LaneValLo =
6374 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6375 Register LaneValHi =
6376 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6377 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AccumulatorVReg)
6378 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg());
6379 unsigned Modifier =
6380 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
6381 ? SISrcMods::NEG
6382 : SISrcMods::NONE;
6383 auto DstVregInst =
6384 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstVreg)
6385 .addImm(Val: Modifier) // src0 modifiers
6386 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6387 .addImm(Val: SISrcMods::NONE) // src1 modifiers
6388 .addReg(RegNo: AccumulatorVReg)
6389 .addImm(Val: SISrcMods::NONE) // clamp
6390 .addImm(Val: SISrcMods::NONE); // omod
6391 auto ReadLaneLo =
6392 BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6393 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: LaneValLo);
6394 auto ReadLaneHi =
6395 BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6396 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: LaneValHi);
6397 MachineBasicBlock::iterator Iters = *ReadLaneLo;
6398 auto [Op1L, Op1H] = ExtractSubRegs(MI&: *Iters, Op&: DstVregInst->getOperand(i: 0),
6399 SrcRC: VregRC, ST, MRI);
6400 ReadLaneLo.addReg(RegNo: Op1L);
6401 ReadLaneHi.addReg(RegNo: Op1H);
6402 NewAccumulator =
6403 BuildRegSequence(*ComputeLoop, I, DstReg, LaneValLo, LaneValHi);
6404 break;
6405 }
6406 case AMDGPU::S_ADD_U64_PSEUDO:
6407 case AMDGPU::S_SUB_U64_PSEUDO: {
6408 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstReg)
6409 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg())
6410 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg());
6411 ComputeLoop =
6412 expand64BitScalarArithmetic(MI&: *NewAccumulator, BB: ComputeLoop);
6413 break;
6414 }
6415 }
6416 }
6417 // Manipulate the iterator to get the next active lane
6418 unsigned BITSETOpc =
6419 IsWave32 ? AMDGPU::S_BITSET0_B32 : AMDGPU::S_BITSET0_B64;
6420 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: BITSETOpc), DestReg: NewActiveBitsReg)
6421 .addReg(RegNo: FF1Reg)
6422 .addReg(RegNo: ActiveBitsReg);
6423
6424 // Add phi nodes
6425 Accumulator.addReg(RegNo: DstReg).addMBB(MBB: ComputeLoop);
6426 ActiveBits.addReg(RegNo: NewActiveBitsReg).addMBB(MBB: ComputeLoop);
6427
6428 // Creating branching
6429 MachineInstrBuilder SetSCCInstr;
6430 if (!ST.hasScalarCompareEq64()) {
6431 // For targets <= gfx7, use an S_OR_B32/B64 instruction to set SCC.
6432 Register LaneMaskReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6433 unsigned CMPOpc = IsWave32 ? AMDGPU::S_OR_B32 : AMDGPU::S_OR_B64;
6434 SetSCCInstr =
6435 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: CMPOpc), DestReg: LaneMaskReg);
6436 } else {
6437 unsigned CMPOpc =
6438 IsWave32 ? AMDGPU::S_CMP_LG_U32 : AMDGPU::S_CMP_LG_U64;
6439 SetSCCInstr = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: CMPOpc));
6440 }
6441 SetSCCInstr.addReg(RegNo: NewActiveBitsReg);
6442 if (ST.hasScalarCompareEq64())
6443 SetSCCInstr.addImm(Val: 0);
6444 else
6445 SetSCCInstr.addReg(RegNo: NewActiveBitsReg);
6446 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
6447 .addMBB(MBB: ComputeLoop);
6448
6449 RetBB = ComputeEnd;
6450 } else {
6451 assert(ST.hasDPP() && "Sub Target does not support DPP Operations");
6452 MachineBasicBlock *CurrBB = &BB;
6453 Register SrcWithIdentity = MRI.createVirtualRegister(RegClass: SrcRegClass);
6454 Register IdentityVGPR = MRI.createVirtualRegister(RegClass: SrcRegClass);
6455 Register IdentitySGPR = MRI.createVirtualRegister(RegClass: DstRegClass);
6456 Register DPPRowShr1 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6457 Register DPPRowShr2 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6458 Register DPPRowShr4 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6459 Register DPPRowShr8 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6460 Register RowBcast15 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6461 Register ReducedValSGPR = MRI.createVirtualRegister(RegClass: DstRegClass);
6462 Register NegatedReducedVal = MRI.createVirtualRegister(RegClass: DstRegClass);
6463 Register RowBcast31 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6464 Register UndefExec = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6465 Register FinalDPPResult;
6466 MachineInstr *SrcWithIdentityInstr;
6467 MachineInstr *LastBcastInstr;
6468 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefExec);
6469
6470 uint64_t IdentityValue = getIdentityValueForWaveReduction(Opc);
6471 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL,
6472 MCID: TII->get(Opcode: is32BitOpc ? AMDGPU::S_MOV_B32
6473 : AMDGPU::S_MOV_B64_IMM_PSEUDO),
6474 DestReg: IdentitySGPR)
6475 .addImm(Val: IdentityValue);
6476 auto IdentityCopyInstr =
6477 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: IdentityVGPR)
6478 .addReg(RegNo: IdentitySGPR);
6479 auto DPPClampOpcPair = getDPPOpcForWaveReduction(Opc, ST);
6480 unsigned DPPOpc = std::get<0>(t&: DPPClampOpcPair);
6481 unsigned ClampOpc = std::get<1>(t&: DPPClampOpcPair);
6482 auto BuildSetInactiveInstr = [&](Register Dst, Register Src0,
6483 Register Src1) {
6484 return BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_SET_INACTIVE_B32),
6485 DestReg: Dst)
6486 .addImm(Val: 0) // src0 modifiers
6487 .addReg(RegNo: Src0) // src0
6488 .addImm(Val: 0) // src1 modifiers
6489 .addReg(RegNo: Src1) // identity value for inactive lanes
6490 .addReg(RegNo: UndefExec); // bool i1
6491 };
6492 auto BuildDPPMachineInstr = [&](Register Dst, Register Src,
6493 unsigned DPPCtrl) {
6494 auto DPPInstr =
6495 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: DPPOpc), DestReg: Dst).addReg(RegNo: Src); // old
6496 if (isFPOp && !NeedsMovDPP)
6497 DPPInstr.addImm(Val: SISrcMods::NONE); // src0 modifier
6498 DPPInstr.addReg(RegNo: Src); // src0
6499 if (isFPOp && !NeedsMovDPP)
6500 DPPInstr.addImm(Val: SISrcMods::NONE); // src1 modifier
6501 if (!NeedsMovDPP)
6502 DPPInstr.addReg(RegNo: Src); // src1
6503 if (AMDGPU::getNamedOperandIdx(Opcode: DPPOpc, Name: AMDGPU::OpName::clamp) >= 0)
6504 DPPInstr.addImm(Val: 0); // clamp
6505 DPPInstr
6506 .addImm(Val: DPPCtrl) // dpp-ctrl
6507 .addImm(Val: 0xf) // row-mask
6508 .addImm(Val: 0xf) // bank-mask
6509 .addImm(Val: 0); // bound-control
6510 };
6511 auto BuildClampInstr = [&](Register Dst, Register Src0, Register Src1,
6512 bool isAddSub = false,
6513 bool needsCarryIn = false,
6514 Register CarryIn = Register()) {
6515 unsigned InstrOpc = ClampOpc;
6516 Register CarryOutReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6517 if (needsCarryIn)
6518 InstrOpc = AMDGPU::V_ADDC_U32_e64;
6519 auto ClampInstr = BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: InstrOpc), DestReg: Dst);
6520 if (isFPOp)
6521 ClampInstr.addImm(Val: SISrcMods::NONE); // src0 mod
6522 if (isAddSub) {
6523 if (needsCarryIn)
6524 ClampInstr.addReg(RegNo: CarryOutReg,
6525 Flags: RegState::Define |
6526 RegState::Dead); // killed carry-out reg
6527 else
6528 ClampInstr.addReg(RegNo: CarryOutReg, Flags: RegState::Define); // carry-out reg
6529 }
6530 ClampInstr.addReg(RegNo: Src0); // src0
6531 if (isFPOp)
6532 ClampInstr.addImm(Val: SISrcMods::NONE); // src1 mod
6533 ClampInstr.addReg(RegNo: Src1); // src1
6534 if (needsCarryIn)
6535 ClampInstr.addReg(RegNo: CarryIn, Flags: RegState::Kill); // carry-in reg
6536 if (AMDGPU::getNamedOperandIdx(Opcode: InstrOpc, Name: AMDGPU::OpName::clamp) >= 0)
6537 ClampInstr.addImm(Val: 0); // clamp
6538 if (isFPOp)
6539 ClampInstr.addImm(Val: 0); // omod
6540 LastBcastInstr = ClampInstr;
6541 return CarryOutReg;
6542 };
6543 auto BuildPostDPPInstr = [&](Register Src0, Register Src1) {
6544 bool isAddSubOpc =
6545 Opc == AMDGPU::S_ADD_U64_PSEUDO || Opc == AMDGPU::S_SUB_U64_PSEUDO;
6546 bool isBitWiseOpc = Opc == AMDGPU::S_AND_B64 ||
6547 Opc == AMDGPU::S_OR_B64 || Opc == AMDGPU::S_XOR_B64;
6548 Register ReturnReg = MRI.createVirtualRegister(RegClass: SrcRegClass);
6549 if (isAddSubOpc || isBitWiseOpc) {
6550 Register ResLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6551 Register ResHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6552 MachineOperand Src0Operand =
6553 MachineOperand::CreateReg(Reg: Src0, /*isDef=*/false);
6554 MachineOperand Src1Operand =
6555 MachineOperand::CreateReg(Reg: Src1, /*isDef=*/false);
6556 auto [Src0Lo, Src0Hi] =
6557 ExtractSubRegs(MI, Op&: Src0Operand, SrcRC: SrcRegClass, ST, MRI);
6558 auto [Src1Lo, Src1Hi] =
6559 ExtractSubRegs(MI, Op&: Src1Operand, SrcRC: SrcRegClass, ST, MRI);
6560 Register CarryReg = BuildClampInstr(
6561 ResLo, Src0Lo, Src1Lo, isAddSubOpc, /*needsCarryIn*/ false);
6562 BuildClampInstr(ResHi, Src0Hi, Src1Hi, isAddSubOpc,
6563 /*needsCarryIn*/ isAddSubOpc, CarryReg);
6564 BuildRegSequence(*CurrBB, MI, ReturnReg, ResLo, ResHi);
6565 } else {
6566 if (isFPOp) {
6567 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: ReturnReg)
6568 .addImm(Val: SISrcMods::NONE) // src0 modifiers
6569 .addReg(RegNo: Src0)
6570 .addImm(Val: SISrcMods::NONE) // src1 modifiers
6571 .addReg(RegNo: Src1)
6572 .addImm(Val: SISrcMods::NONE) // clamp
6573 .addImm(Val: SISrcMods::NONE); // omod
6574 } else {
6575 Register CmpMaskReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6576 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: CmpMaskReg)
6577 .addReg(RegNo: Src0) // src0
6578 .addReg(RegNo: Src1); // src1
6579 LastBcastInstr =
6580 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B64_PSEUDO),
6581 DestReg: ReturnReg)
6582 .addReg(RegNo: Src1) // src0
6583 .addReg(RegNo: Src0) // src1
6584 .addReg(RegNo: CmpMaskReg); // src2
6585 expand64BitV_CNDMASK(MI&: *LastBcastInstr, BB: CurrBB);
6586 }
6587 }
6588 return ReturnReg;
6589 };
6590
6591 // Set inactive lanes to the identity value.
6592 if (is32BitOpc) {
6593 SrcWithIdentityInstr =
6594 BuildSetInactiveInstr(SrcWithIdentity, SrcReg, IdentityVGPR);
6595 } else {
6596 Register SrcWithIdentitylo =
6597 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6598 Register SrcWithIdentityhi =
6599 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6600 auto [Reg0Sub0, Reg0Sub1] = ExtractSubRegs(
6601 MI, Op&: IdentityCopyInstr->getOperand(i: 0), SrcRC: SrcRegClass, ST, MRI);
6602 auto [SrcReg0Sub0, SrcReg0Sub1] =
6603 ExtractSubRegs(MI, Op&: MI.getOperand(i: 1), SrcRC: SrcRegClass, ST, MRI);
6604 MachineInstr *SetInactiveLoInstr =
6605 BuildSetInactiveInstr(SrcWithIdentitylo, SrcReg0Sub0, Reg0Sub0);
6606 MachineInstr *SetInactiveHiInstr =
6607 BuildSetInactiveInstr(SrcWithIdentityhi, SrcReg0Sub1, Reg0Sub1);
6608 SrcWithIdentityInstr =
6609 BuildRegSequence(*CurrBB, MI, SrcWithIdentity,
6610 SetInactiveLoInstr->getOperand(i: 0).getReg(),
6611 SetInactiveHiInstr->getOperand(i: 0).getReg());
6612 }
6613 // DPP reduction
6614 Register SrcWithIdentityReg =
6615 SrcWithIdentityInstr->getOperand(i: 0).getReg();
6616 BuildDPPMachineInstr(DPPRowShr1, SrcWithIdentityReg,
6617 AMDGPU::DPP::ROW_SHR_FIRST);
6618 if (NeedsMovDPP)
6619 DPPRowShr1 = BuildPostDPPInstr(SrcWithIdentityReg, DPPRowShr1);
6620
6621 BuildDPPMachineInstr(DPPRowShr2, DPPRowShr1,
6622 (AMDGPU::DPP::ROW_SHR_FIRST + 1));
6623 if (NeedsMovDPP)
6624 DPPRowShr2 = BuildPostDPPInstr(DPPRowShr1, DPPRowShr2);
6625
6626 BuildDPPMachineInstr(DPPRowShr4, DPPRowShr2,
6627 (AMDGPU::DPP::ROW_SHR_FIRST + 3));
6628 if (NeedsMovDPP)
6629 DPPRowShr4 = BuildPostDPPInstr(DPPRowShr2, DPPRowShr4);
6630
6631 BuildDPPMachineInstr(DPPRowShr8, DPPRowShr4,
6632 (AMDGPU::DPP::ROW_SHR_FIRST + 7));
6633 if (NeedsMovDPP)
6634 DPPRowShr8 = BuildPostDPPInstr(DPPRowShr4, DPPRowShr8);
6635
6636 if (ST.hasDPPBroadcasts()) {
6637 BuildDPPMachineInstr(RowBcast15, DPPRowShr8, AMDGPU::DPP::BCAST15);
6638 if (NeedsMovDPP)
6639 RowBcast15 = BuildPostDPPInstr(DPPRowShr8, RowBcast15);
6640 } else {
6641 // magic constant: 0x1E0
6642 // To Set BIT_MODE : bit 15 = 0
6643 // XOR mask : bit [14:10] = 0
6644 // OR mask : bit [9:5] = 15
6645 // AND mask : bit [4:0] = 0
6646 if (is32BitOpc) {
6647 Register SwizzledValue =
6648 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6649 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_SWIZZLE_B32),
6650 DestReg: SwizzledValue)
6651 .addReg(RegNo: DPPRowShr8) // addr
6652 .addImm(Val: 0x1E0) // swizzle offset (i16)
6653 .addImm(Val: 0x0); // gds (i1)
6654 BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue);
6655 } else {
6656 Register SwizzledValuelo =
6657 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6658 Register SwizzledValuehi =
6659 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6660 Register SwizzledValue64 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6661 MachineOperand DPPRowShr8Op =
6662 MachineOperand::CreateReg(Reg: DPPRowShr8, /*isDef=*/false);
6663 auto [Op1L, Op1H] =
6664 ExtractSubRegs(MI, Op&: DPPRowShr8Op, SrcRC: SrcRegClass, ST, MRI);
6665 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_SWIZZLE_B32),
6666 DestReg: SwizzledValuelo)
6667 .addReg(RegNo: Op1L) // addr
6668 .addImm(Val: 0x1E0) // swizzle offset (i16)
6669 .addImm(Val: 0x0); // gds (i1)
6670 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_SWIZZLE_B32),
6671 DestReg: SwizzledValuehi)
6672 .addReg(RegNo: Op1H) // addr
6673 .addImm(Val: 0x1E0) // swizzle offset (i16)
6674 .addImm(Val: 0x0); // gds (i1)
6675 BuildRegSequence(*CurrBB, MI, SwizzledValue64, SwizzledValuelo,
6676 SwizzledValuehi);
6677 if (NeedsMovDPP)
6678 RowBcast15 = BuildPostDPPInstr(DPPRowShr8, SwizzledValue64);
6679 else
6680 BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue64);
6681 }
6682 }
6683 FinalDPPResult = RowBcast15;
6684 if (!IsWave32) {
6685 if (ST.hasDPPBroadcasts()) {
6686 BuildDPPMachineInstr(RowBcast31, RowBcast15, AMDGPU::DPP::BCAST31);
6687 if (NeedsMovDPP)
6688 RowBcast31 = BuildPostDPPInstr(RowBcast15, RowBcast31);
6689 } else {
6690 Register ShiftedThreadID =
6691 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6692 Register PermuteByteOffset =
6693 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6694 Register PermutedValue = MRI.createVirtualRegister(RegClass: SrcRegClass);
6695 Register Lane32Offset =
6696 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6697 Register WordSizeConst =
6698 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6699 Register ThreadIDRegLo =
6700 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6701 Register ThreadIDReg =
6702 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6703 // Get the thread ID.
6704 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MBCNT_LO_U32_B32_e64),
6705 DestReg: ThreadIDRegLo)
6706 .addImm(Val: -1)
6707 .addImm(Val: 0);
6708 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MBCNT_HI_U32_B32_e64),
6709 DestReg: ThreadIDReg)
6710 .addImm(Val: -1)
6711 .addReg(RegNo: ThreadIDRegLo);
6712 // shift each lane over by 32 positions, so value in 31st lane is
6713 // present in 63rd lane.
6714 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: Lane32Offset)
6715 .addImm(Val: 0x20);
6716 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ADD_U32_e64),
6717 DestReg: ShiftedThreadID)
6718 .addReg(RegNo: ThreadIDReg)
6719 .addReg(RegNo: Lane32Offset)
6720 .addImm(Val: 0); // clamp
6721 // multiply by reg size.
6722 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: WordSizeConst)
6723 .addImm(Val: 0x4);
6724 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MUL_LO_U32_e64),
6725 DestReg: PermuteByteOffset)
6726 .addReg(RegNo: WordSizeConst)
6727 .addReg(RegNo: ShiftedThreadID);
6728 // Permute the lanes
6729 if (is32BitOpc) {
6730 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_PERMUTE_B32),
6731 DestReg: PermutedValue)
6732 .addReg(RegNo: PermuteByteOffset) // addr
6733 .addReg(RegNo: RowBcast15) // data
6734 .addImm(Val: 0); // offset
6735 } else {
6736 Register PermutedValuelo =
6737 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6738 Register PermutedValuehi =
6739 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6740 MachineOperand RowBcast15Op =
6741 MachineOperand::CreateReg(Reg: RowBcast15, /*isDef=*/false);
6742 auto [RowBcast15Lo, RowBcast15Hi] =
6743 ExtractSubRegs(MI, Op&: RowBcast15Op, SrcRC: SrcRegClass, ST, MRI);
6744 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_PERMUTE_B32),
6745 DestReg: PermutedValuelo)
6746 .addReg(RegNo: PermuteByteOffset) // addr
6747 .addReg(RegNo: RowBcast15Lo) // swizzle offset (i16)
6748 .addImm(Val: 0x0); // gds (i1)
6749 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_PERMUTE_B32),
6750 DestReg: PermutedValuehi)
6751 .addReg(RegNo: PermuteByteOffset) // addr
6752 .addReg(RegNo: RowBcast15Hi) // swizzle offset (i16)
6753 .addImm(Val: 0x0); // gds (i1)
6754 BuildRegSequence(*CurrBB, MI, PermutedValue, PermutedValuelo,
6755 PermutedValuehi);
6756 }
6757 if (NeedsMovDPP)
6758 RowBcast31 = BuildPostDPPInstr(RowBcast15, PermutedValue);
6759 else
6760 BuildClampInstr(RowBcast31, RowBcast15, PermutedValue);
6761 }
6762 FinalDPPResult = RowBcast31;
6763 }
6764 if (MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32 ||
6765 MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64) {
6766 Register NegatedValVGPR = MRI.createVirtualRegister(RegClass: SrcRegClass);
6767 // Opc for f32 reduction is V_SUB_F32.
6768 // For f64, there is no equivalent V_SUB_F64 opcode, so use
6769 // V_ADD_F64/V_ADD_F64_pseudo, and negate the second operand.
6770 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc),
6771 DestReg: NegatedValVGPR)
6772 .addImm(Val: SISrcMods::NONE) // src0 mods
6773 .addReg(RegNo: IdentityVGPR) // src0
6774 .addImm(Val: is32BitOpc ? SISrcMods::NONE : SISrcMods::NEG) // src1 mods
6775 .addReg(RegNo: IsWave32 ? RowBcast15 : RowBcast31) // src1
6776 .addImm(Val: SISrcMods::NONE) // clamp
6777 .addImm(Val: SISrcMods::NONE); // omod
6778 FinalDPPResult = NegatedValVGPR;
6779 }
6780 // The final reduced value is in the last lane.
6781 if (is32BitOpc) {
6782 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6783 DestReg: ReducedValSGPR)
6784 .addReg(RegNo: FinalDPPResult)
6785 .addImm(Val: ST.getWavefrontSize() - 1);
6786 } else {
6787 Register LaneValueLoReg =
6788 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6789 Register LaneValueHiReg =
6790 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6791 const TargetRegisterClass *SrcRC = MRI.getRegClass(Reg: SrcReg);
6792 MachineOperand FinalDPPResultOperand =
6793 MachineOperand::CreateReg(Reg: FinalDPPResult, /*isDef=*/false);
6794 auto [Op1L, Op1H] =
6795 ExtractSubRegs(MI, Op&: FinalDPPResultOperand, SrcRC, ST, MRI);
6796 // lane value input should be in an sgpr
6797 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6798 DestReg: LaneValueLoReg)
6799 .addReg(RegNo: Op1L)
6800 .addImm(Val: ST.getWavefrontSize() - 1);
6801 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6802 DestReg: LaneValueHiReg)
6803 .addReg(RegNo: Op1H)
6804 .addImm(Val: ST.getWavefrontSize() - 1);
6805 BuildRegSequence(*CurrBB, MI, ReducedValSGPR, LaneValueLoReg,
6806 LaneValueHiReg);
6807 }
6808 if (Opc == AMDGPU::S_SUB_I32) {
6809 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedReducedVal)
6810 .addImm(Val: 0)
6811 .addReg(RegNo: ReducedValSGPR);
6812 } else if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6813 auto NegatedValInstr =
6814 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: NegatedReducedVal)
6815 .addImm(Val: 0)
6816 .addReg(RegNo: ReducedValSGPR);
6817 CurrBB = expand64BitScalarArithmetic(MI&: *NegatedValInstr, BB: CurrBB);
6818 }
6819 // Mark the final result as a whole-wave-mode calculation.
6820 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::STRICT_WWM), DestReg: DstReg)
6821 .addReg(RegNo: Opc == AMDGPU::S_SUB_I32 || Opc == AMDGPU::S_SUB_U64_PSEUDO
6822 ? NegatedReducedVal
6823 : ReducedValSGPR);
6824 RetBB = CurrBB;
6825 }
6826 }
6827 MI.eraseFromParent();
6828 return RetBB;
6829}
6830
6831MachineBasicBlock *
6832SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
6833 MachineBasicBlock *BB) const {
6834 MachineFunction *MF = BB->getParent();
6835 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
6836 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
6837 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
6838 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
6839 MachineRegisterInfo &MRI = MF->getRegInfo();
6840 const DebugLoc &DL = MI.getDebugLoc();
6841
6842 switch (MI.getOpcode()) {
6843 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U32:
6844 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MIN_U32);
6845 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U64:
6846 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_LT_U64_e64);
6847 case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I32:
6848 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MIN_I32);
6849 case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I64:
6850 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_LT_I64_e64);
6851 case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F32:
6852 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_MIN_F32_e64);
6853 case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F64:
6854 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6855 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6856 ? AMDGPU::V_MIN_NUM_F64_e64
6857 : AMDGPU::V_MIN_F64_e64);
6858 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U32:
6859 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MAX_U32);
6860 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U64:
6861 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_GT_U64_e64);
6862 case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I32:
6863 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MAX_I32);
6864 case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I64:
6865 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_GT_I64_e64);
6866 case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F32:
6867 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_MAX_F32_e64);
6868 case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F64:
6869 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6870 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6871 ? AMDGPU::V_MAX_NUM_F64_e64
6872 : AMDGPU::V_MAX_F64_e64);
6873 case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_I32:
6874 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_ADD_I32);
6875 case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_U64:
6876 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_ADD_U64_PSEUDO);
6877 case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F32:
6878 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_ADD_F32_e64);
6879 case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F64:
6880 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6881 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6882 ? AMDGPU::V_ADD_F64_pseudo_e64
6883 : AMDGPU::V_ADD_F64_e64);
6884 case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_I32:
6885 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_SUB_I32);
6886 case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_U64:
6887 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_SUB_U64_PSEUDO);
6888 case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32:
6889 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_SUB_F32_e64);
6890 case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64:
6891 // There is no S/V_SUB_F64 opcode. Double type subtraction is expanded as
6892 // fadd + neg, by setting the NEG bit in the instruction.
6893 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6894 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6895 ? AMDGPU::V_ADD_F64_pseudo_e64
6896 : AMDGPU::V_ADD_F64_e64);
6897 case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B32:
6898 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_AND_B32);
6899 case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B64:
6900 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_AND_B64);
6901 case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B32:
6902 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_OR_B32);
6903 case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B64:
6904 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_OR_B64);
6905 case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B32:
6906 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_XOR_B32);
6907 case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B64:
6908 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_XOR_B64);
6909 case AMDGPU::S_UADDO_PSEUDO:
6910 case AMDGPU::S_USUBO_PSEUDO: {
6911 MachineOperand &Dest0 = MI.getOperand(i: 0);
6912 MachineOperand &Dest1 = MI.getOperand(i: 1);
6913 MachineOperand &Src0 = MI.getOperand(i: 2);
6914 MachineOperand &Src1 = MI.getOperand(i: 3);
6915
6916 unsigned Opc = (MI.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
6917 ? AMDGPU::S_ADD_U32
6918 : AMDGPU::S_SUB_U32;
6919 // clang-format off
6920 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest0.getReg())
6921 .add(MO: Src0)
6922 .add(MO: Src1);
6923 // clang-format on
6924
6925 unsigned SelOpc =
6926 Subtarget->isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
6927 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SelOpc), DestReg: Dest1.getReg()).addImm(Val: -1).addImm(Val: 0);
6928
6929 MI.eraseFromParent();
6930 return BB;
6931 }
6932 case AMDGPU::S_ADD_U64_PSEUDO:
6933 case AMDGPU::S_SUB_U64_PSEUDO: {
6934 return expand64BitScalarArithmetic(MI, BB);
6935 }
6936 case AMDGPU::V_ADD_U64_PSEUDO:
6937 case AMDGPU::V_SUB_U64_PSEUDO: {
6938 bool IsAdd = (MI.getOpcode() == AMDGPU::V_ADD_U64_PSEUDO);
6939
6940 MachineOperand &Dest = MI.getOperand(i: 0);
6941 MachineOperand &Src0 = MI.getOperand(i: 1);
6942 MachineOperand &Src1 = MI.getOperand(i: 2);
6943
6944 if (ST.hasAddSubU64Insts()) {
6945 auto I = BuildMI(BB&: *BB, I&: MI, MIMD: DL,
6946 MCID: TII->get(Opcode: IsAdd ? AMDGPU::V_ADD_U64_e64
6947 : AMDGPU::V_SUB_U64_e64),
6948 DestReg: Dest.getReg())
6949 .add(MO: Src0)
6950 .add(MO: Src1)
6951 .addImm(Val: 0); // clamp
6952 TII->legalizeOperands(MI&: *I);
6953 MI.eraseFromParent();
6954 return BB;
6955 }
6956
6957 if (IsAdd && ST.hasLshlAddU64Inst()) {
6958 auto Add = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_LSHL_ADD_U64_e64),
6959 DestReg: Dest.getReg())
6960 .add(MO: Src0)
6961 .addImm(Val: 0)
6962 .add(MO: Src1);
6963 TII->legalizeOperands(MI&: *Add);
6964 MI.eraseFromParent();
6965 return BB;
6966 }
6967
6968 const auto *CarryRC = TRI->getWaveMaskRegClass();
6969
6970 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6971 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6972
6973 Register CarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
6974 Register DeadCarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
6975
6976 const TargetRegisterClass *Src0RC = Src0.isReg()
6977 ? MRI.getRegClass(Reg: Src0.getReg())
6978 : &AMDGPU::VReg_64RegClass;
6979 const TargetRegisterClass *Src1RC = Src1.isReg()
6980 ? MRI.getRegClass(Reg: Src1.getReg())
6981 : &AMDGPU::VReg_64RegClass;
6982
6983 const TargetRegisterClass *Src0SubRC =
6984 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
6985 const TargetRegisterClass *Src1SubRC =
6986 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
6987
6988 MachineOperand SrcReg0Sub0 = TII->buildExtractSubRegOrImm(
6989 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
6990 MachineOperand SrcReg1Sub0 = TII->buildExtractSubRegOrImm(
6991 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
6992
6993 MachineOperand SrcReg0Sub1 = TII->buildExtractSubRegOrImm(
6994 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
6995 MachineOperand SrcReg1Sub1 = TII->buildExtractSubRegOrImm(
6996 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
6997
6998 unsigned LoOpc =
6999 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
7000 MachineInstr *LoHalf = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoOpc), DestReg: DestSub0)
7001 .addReg(RegNo: CarryReg, Flags: RegState::Define)
7002 .add(MO: SrcReg0Sub0)
7003 .add(MO: SrcReg1Sub0)
7004 .addImm(Val: 0); // clamp bit
7005
7006 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
7007 MachineInstr *HiHalf =
7008 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: HiOpc), DestReg: DestSub1)
7009 .addReg(RegNo: DeadCarryReg, Flags: RegState::Define | RegState::Dead)
7010 .add(MO: SrcReg0Sub1)
7011 .add(MO: SrcReg1Sub1)
7012 .addReg(RegNo: CarryReg, Flags: RegState::Kill)
7013 .addImm(Val: 0); // clamp bit
7014
7015 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest.getReg())
7016 .addReg(RegNo: DestSub0)
7017 .addImm(Val: AMDGPU::sub0)
7018 .addReg(RegNo: DestSub1)
7019 .addImm(Val: AMDGPU::sub1);
7020 TII->legalizeOperands(MI&: *LoHalf);
7021 TII->legalizeOperands(MI&: *HiHalf);
7022 MI.eraseFromParent();
7023 return BB;
7024 }
7025 case AMDGPU::S_ADD_CO_PSEUDO:
7026 case AMDGPU::S_SUB_CO_PSEUDO: {
7027 // This pseudo has a chance to be selected
7028 // only from uniform add/subcarry node. All the VGPR operands
7029 // therefore assumed to be splat vectors.
7030 MachineBasicBlock::iterator MII = MI;
7031 MachineOperand &Dest = MI.getOperand(i: 0);
7032 MachineOperand &CarryDest = MI.getOperand(i: 1);
7033 MachineOperand &Src0 = MI.getOperand(i: 2);
7034 MachineOperand &Src1 = MI.getOperand(i: 3);
7035 MachineOperand &Src2 = MI.getOperand(i: 4);
7036 if (Src0.isReg() && TRI->isVectorRegister(MRI, Reg: Src0.getReg())) {
7037 Register RegOp0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7038 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp0)
7039 .addReg(RegNo: Src0.getReg());
7040 Src0.setReg(RegOp0);
7041 }
7042 if (Src1.isReg() && TRI->isVectorRegister(MRI, Reg: Src1.getReg())) {
7043 Register RegOp1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7044 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp1)
7045 .addReg(RegNo: Src1.getReg());
7046 Src1.setReg(RegOp1);
7047 }
7048 Register RegOp2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7049 if (TRI->isVectorRegister(MRI, Reg: Src2.getReg())) {
7050 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp2)
7051 .addReg(RegNo: Src2.getReg());
7052 Src2.setReg(RegOp2);
7053 }
7054
7055 if (ST.isWave64()) {
7056 if (ST.hasScalarCompareEq64()) {
7057 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U64))
7058 .addReg(RegNo: Src2.getReg())
7059 .addImm(Val: 0);
7060 } else {
7061 const TargetRegisterClass *Src2RC = MRI.getRegClass(Reg: Src2.getReg());
7062 const TargetRegisterClass *SubRC =
7063 TRI->getSubRegisterClass(Src2RC, AMDGPU::sub0);
7064 MachineOperand Src2Sub0 = TII->buildExtractSubRegOrImm(
7065 MI: MII, MRI, SuperReg: Src2, SuperRC: Src2RC, SubIdx: AMDGPU::sub0, SubRC);
7066 MachineOperand Src2Sub1 = TII->buildExtractSubRegOrImm(
7067 MI: MII, MRI, SuperReg: Src2, SuperRC: Src2RC, SubIdx: AMDGPU::sub1, SubRC);
7068 Register Src2_32 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7069
7070 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_OR_B32), DestReg: Src2_32)
7071 .add(MO: Src2Sub0)
7072 .add(MO: Src2Sub1);
7073
7074 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
7075 .addReg(RegNo: Src2_32, Flags: RegState::Kill)
7076 .addImm(Val: 0);
7077 }
7078 } else {
7079 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
7080 .addReg(RegNo: Src2.getReg())
7081 .addImm(Val: 0);
7082 }
7083
7084 unsigned Opc = MI.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO
7085 ? AMDGPU::S_ADDC_U32
7086 : AMDGPU::S_SUBB_U32;
7087
7088 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest.getReg()).add(MO: Src0).add(MO: Src1);
7089
7090 unsigned SelOpc =
7091 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
7092
7093 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: SelOpc), DestReg: CarryDest.getReg())
7094 .addImm(Val: -1)
7095 .addImm(Val: 0);
7096
7097 MI.eraseFromParent();
7098 return BB;
7099 }
7100 case AMDGPU::SI_INIT_M0: {
7101 MachineOperand &M0Init = MI.getOperand(i: 0);
7102 BuildMI(BB&: *BB, I: MI.getIterator(), MIMD: MI.getDebugLoc(),
7103 MCID: TII->get(Opcode: M0Init.isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32),
7104 DestReg: AMDGPU::M0)
7105 .add(MO: M0Init);
7106 MI.eraseFromParent();
7107 return BB;
7108 }
7109 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM: {
7110 // Set SCC to true, in case the barrier instruction gets converted to a NOP.
7111 BuildMI(BB&: *BB, I: MI.getIterator(), MIMD: MI.getDebugLoc(),
7112 MCID: TII->get(Opcode: AMDGPU::S_CMP_EQ_U32))
7113 .addImm(Val: 0)
7114 .addImm(Val: 0);
7115 return BB;
7116 }
7117 case AMDGPU::GET_GROUPSTATICSIZE: {
7118 assert(getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA ||
7119 getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL);
7120 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32))
7121 .add(MO: MI.getOperand(i: 0))
7122 .addImm(Val: MFI->getLDSSize());
7123 MI.eraseFromParent();
7124 return BB;
7125 }
7126 case AMDGPU::GET_SHADERCYCLESHILO: {
7127 assert(MF->getSubtarget<GCNSubtarget>().hasShaderCyclesHiLoRegisters());
7128 // The algorithm is:
7129 //
7130 // hi1 = getreg(SHADER_CYCLES_HI)
7131 // lo1 = getreg(SHADER_CYCLES_LO)
7132 // hi2 = getreg(SHADER_CYCLES_HI)
7133 //
7134 // If hi1 == hi2 then there was no overflow and the result is hi2:lo1.
7135 // Otherwise there was overflow and the result is hi2:0. In both cases the
7136 // result should represent the actual time at some point during the sequence
7137 // of three getregs.
7138 using namespace AMDGPU::Hwreg;
7139 Register RegHi1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7140 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegHi1)
7141 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES_HI, Values: 0, Values: 32));
7142 Register RegLo1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7143 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegLo1)
7144 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES, Values: 0, Values: 32));
7145 Register RegHi2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7146 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegHi2)
7147 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES_HI, Values: 0, Values: 32));
7148 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_EQ_U32))
7149 .addReg(RegNo: RegHi1)
7150 .addReg(RegNo: RegHi2);
7151 Register RegLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7152 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: RegLo)
7153 .addReg(RegNo: RegLo1)
7154 .addImm(Val: 0);
7155 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE))
7156 .add(MO: MI.getOperand(i: 0))
7157 .addReg(RegNo: RegLo)
7158 .addImm(Val: AMDGPU::sub0)
7159 .addReg(RegNo: RegHi2)
7160 .addImm(Val: AMDGPU::sub1);
7161 MI.eraseFromParent();
7162 return BB;
7163 }
7164 case AMDGPU::SI_INDIRECT_SRC_V1:
7165 case AMDGPU::SI_INDIRECT_SRC_V2:
7166 case AMDGPU::SI_INDIRECT_SRC_V3:
7167 case AMDGPU::SI_INDIRECT_SRC_V4:
7168 case AMDGPU::SI_INDIRECT_SRC_V5:
7169 case AMDGPU::SI_INDIRECT_SRC_V6:
7170 case AMDGPU::SI_INDIRECT_SRC_V7:
7171 case AMDGPU::SI_INDIRECT_SRC_V8:
7172 case AMDGPU::SI_INDIRECT_SRC_V9:
7173 case AMDGPU::SI_INDIRECT_SRC_V10:
7174 case AMDGPU::SI_INDIRECT_SRC_V11:
7175 case AMDGPU::SI_INDIRECT_SRC_V12:
7176 case AMDGPU::SI_INDIRECT_SRC_V16:
7177 case AMDGPU::SI_INDIRECT_SRC_V32:
7178 return emitIndirectSrc(MI, MBB&: *BB, ST: *getSubtarget());
7179 case AMDGPU::SI_INDIRECT_DST_V1:
7180 case AMDGPU::SI_INDIRECT_DST_V2:
7181 case AMDGPU::SI_INDIRECT_DST_V3:
7182 case AMDGPU::SI_INDIRECT_DST_V4:
7183 case AMDGPU::SI_INDIRECT_DST_V5:
7184 case AMDGPU::SI_INDIRECT_DST_V6:
7185 case AMDGPU::SI_INDIRECT_DST_V7:
7186 case AMDGPU::SI_INDIRECT_DST_V8:
7187 case AMDGPU::SI_INDIRECT_DST_V9:
7188 case AMDGPU::SI_INDIRECT_DST_V10:
7189 case AMDGPU::SI_INDIRECT_DST_V11:
7190 case AMDGPU::SI_INDIRECT_DST_V12:
7191 case AMDGPU::SI_INDIRECT_DST_V16:
7192 case AMDGPU::SI_INDIRECT_DST_V32:
7193 return emitIndirectDst(MI, MBB&: *BB, ST: *getSubtarget());
7194 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
7195 case AMDGPU::SI_KILL_I1_PSEUDO:
7196 return splitKillBlock(MI, BB);
7197 case AMDGPU::V_CNDMASK_B64_PSEUDO: {
7198 expand64BitV_CNDMASK(MI, BB);
7199 return BB;
7200 }
7201 case AMDGPU::SI_BR_UNDEF: {
7202 MachineInstr *Br = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
7203 .add(MO: MI.getOperand(i: 0));
7204 Br->getOperand(i: 1).setIsUndef(); // read undef SCC
7205 MI.eraseFromParent();
7206 return BB;
7207 }
7208 case AMDGPU::ADJCALLSTACKUP:
7209 case AMDGPU::ADJCALLSTACKDOWN: {
7210 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
7211 MachineInstrBuilder MIB(*MF, &MI);
7212 MIB.addReg(RegNo: Info->getStackPtrOffsetReg(), Flags: RegState::ImplicitDefine)
7213 .addReg(RegNo: Info->getStackPtrOffsetReg(), Flags: RegState::Implicit);
7214 return BB;
7215 }
7216 case AMDGPU::SI_CALL_ISEL: {
7217 unsigned ReturnAddrReg = TII->getRegisterInfo().getReturnAddressReg(MF: *MF);
7218
7219 MachineInstrBuilder MIB;
7220 MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_CALL), DestReg: ReturnAddrReg);
7221
7222 for (const MachineOperand &MO : MI.operands())
7223 MIB.add(MO);
7224
7225 MIB.cloneMemRefs(OtherMI: MI);
7226 MI.eraseFromParent();
7227 return BB;
7228 }
7229 case AMDGPU::V_ADD_CO_U32_e32:
7230 case AMDGPU::V_SUB_CO_U32_e32:
7231 case AMDGPU::V_SUBREV_CO_U32_e32: {
7232 // TODO: Define distinct V_*_I32_Pseudo instructions instead.
7233 unsigned Opc = MI.getOpcode();
7234
7235 bool NeedClampOperand = false;
7236 if (TII->pseudoToMCOpcode(Opcode: Opc) == -1) {
7237 Opc = AMDGPU::getVOPe64(Opcode: Opc);
7238 NeedClampOperand = true;
7239 }
7240
7241 auto I = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: MI.getOperand(i: 0).getReg());
7242 if (TII->isVOP3(MI: *I)) {
7243 I.addReg(RegNo: TRI->getVCC(), Flags: RegState::Define);
7244 }
7245 I.add(MO: MI.getOperand(i: 1)).add(MO: MI.getOperand(i: 2));
7246 if (NeedClampOperand)
7247 I.addImm(Val: 0); // clamp bit for e64 encoding
7248
7249 TII->legalizeOperands(MI&: *I);
7250
7251 MI.eraseFromParent();
7252 return BB;
7253 }
7254 case AMDGPU::V_ADDC_U32_e32:
7255 case AMDGPU::V_SUBB_U32_e32:
7256 case AMDGPU::V_SUBBREV_U32_e32:
7257 // These instructions have an implicit use of vcc which counts towards the
7258 // constant bus limit.
7259 TII->legalizeOperands(MI);
7260 return BB;
7261 case AMDGPU::DS_GWS_INIT:
7262 case AMDGPU::DS_GWS_SEMA_BR:
7263 case AMDGPU::DS_GWS_BARRIER:
7264 case AMDGPU::DS_GWS_SEMA_V:
7265 case AMDGPU::DS_GWS_SEMA_P:
7266 case AMDGPU::DS_GWS_SEMA_RELEASE_ALL:
7267 // A s_waitcnt 0 is required to be the instruction immediately following.
7268 if (getSubtarget()->hasGWSAutoReplay()) {
7269 bundleInstWithWaitcnt(MI);
7270 return BB;
7271 }
7272
7273 return emitGWSMemViolTestLoop(MI, BB);
7274 case AMDGPU::S_SETREG_B32: {
7275 // Try to optimize cases that only set the denormal mode or rounding mode.
7276 //
7277 // If the s_setreg_b32 fully sets all of the bits in the rounding mode or
7278 // denormal mode to a constant, we can use s_round_mode or s_denorm_mode
7279 // instead.
7280 //
7281 // FIXME: This could be predicates on the immediate, but tablegen doesn't
7282 // allow you to have a no side effect instruction in the output of a
7283 // sideeffecting pattern.
7284 auto [ID, Offset, Width] =
7285 AMDGPU::Hwreg::HwregEncoding::decode(Encoded: MI.getOperand(i: 1).getImm());
7286 if (ID != AMDGPU::Hwreg::ID_MODE)
7287 return BB;
7288
7289 const unsigned WidthMask = maskTrailingOnes<unsigned>(N: Width);
7290 const unsigned SetMask = WidthMask << Offset;
7291
7292 if (getSubtarget()->hasDenormModeInst()) {
7293 unsigned SetDenormOp = 0;
7294 unsigned SetRoundOp = 0;
7295
7296 // The dedicated instructions can only set the whole denorm or round mode
7297 // at once, not a subset of bits in either.
7298 if (SetMask ==
7299 (AMDGPU::Hwreg::FP_ROUND_MASK | AMDGPU::Hwreg::FP_DENORM_MASK)) {
7300 // If this fully sets both the round and denorm mode, emit the two
7301 // dedicated instructions for these.
7302 SetRoundOp = AMDGPU::S_ROUND_MODE;
7303 SetDenormOp = AMDGPU::S_DENORM_MODE;
7304 } else if (SetMask == AMDGPU::Hwreg::FP_ROUND_MASK) {
7305 SetRoundOp = AMDGPU::S_ROUND_MODE;
7306 } else if (SetMask == AMDGPU::Hwreg::FP_DENORM_MASK) {
7307 SetDenormOp = AMDGPU::S_DENORM_MODE;
7308 }
7309
7310 if (SetRoundOp || SetDenormOp) {
7311 MachineInstr *Def = MRI.getVRegDef(Reg: MI.getOperand(i: 0).getReg());
7312 if (Def && Def->isMoveImmediate() && Def->getOperand(i: 1).isImm()) {
7313 unsigned ImmVal = Def->getOperand(i: 1).getImm();
7314 if (SetRoundOp) {
7315 BuildMI(BB&: *BB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SetRoundOp))
7316 .addImm(Val: ImmVal & 0xf);
7317
7318 // If we also have the denorm mode, get just the denorm mode bits.
7319 ImmVal >>= 4;
7320 }
7321
7322 if (SetDenormOp) {
7323 BuildMI(BB&: *BB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SetDenormOp))
7324 .addImm(Val: ImmVal & 0xf);
7325 }
7326
7327 MI.eraseFromParent();
7328 return BB;
7329 }
7330 }
7331 }
7332
7333 // If only FP bits are touched, used the no side effects pseudo.
7334 if ((SetMask & (AMDGPU::Hwreg::FP_ROUND_MASK |
7335 AMDGPU::Hwreg::FP_DENORM_MASK)) == SetMask)
7336 MI.setDesc(TII->get(Opcode: AMDGPU::S_SETREG_B32_mode));
7337
7338 return BB;
7339 }
7340 case AMDGPU::S_INVERSE_BALLOT_U32:
7341 case AMDGPU::S_INVERSE_BALLOT_U64:
7342 // These opcodes only exist to let SIFixSGPRCopies insert a readfirstlane if
7343 // necessary. After that they are equivalent to a COPY.
7344 MI.setDesc(TII->get(Opcode: AMDGPU::COPY));
7345 return BB;
7346 case AMDGPU::ENDPGM_TRAP: {
7347 if (BB->succ_empty() && std::next(x: MI.getIterator()) == BB->end()) {
7348 MI.setDesc(TII->get(Opcode: AMDGPU::S_ENDPGM));
7349 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
7350 return BB;
7351 }
7352
7353 // We need a block split to make the real endpgm a terminator. We also don't
7354 // want to break phis in successor blocks, so we can't just delete to the
7355 // end of the block.
7356
7357 MachineBasicBlock *SplitBB = BB->splitAt(SplitInst&: MI, UpdateLiveIns: false /*UpdateLiveIns*/);
7358 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
7359 MF->push_back(MBB: TrapBB);
7360 // clang-format off
7361 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ENDPGM))
7362 .addImm(Val: 0);
7363 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_EXECNZ))
7364 .addMBB(MBB: TrapBB);
7365 // clang-format on
7366
7367 BB->addSuccessor(Succ: TrapBB);
7368 MI.eraseFromParent();
7369 return SplitBB;
7370 }
7371 case AMDGPU::SIMULATED_TRAP: {
7372 assert(Subtarget->hasPrivEnabledTrap2NopBug());
7373 MachineBasicBlock *SplitBB =
7374 TII->insertSimulatedTrap(MRI, MBB&: *BB, MI, DL: MI.getDebugLoc());
7375 MI.eraseFromParent();
7376 return SplitBB;
7377 }
7378 case AMDGPU::SI_TCRETURN_GFX_WholeWave:
7379 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN: {
7380 assert(MFI->isWholeWaveFunction());
7381
7382 // During ISel, it's difficult to propagate the original EXEC mask to use as
7383 // an input to SI_WHOLE_WAVE_FUNC_RETURN. Set it up here instead.
7384 MachineInstr *Setup = TII->getWholeWaveFunctionSetup(MF&: *BB->getParent());
7385 assert(Setup && "Couldn't find SI_SETUP_WHOLE_WAVE_FUNC");
7386 Register OriginalExec = Setup->getOperand(i: 0).getReg();
7387 MF->getRegInfo().clearKillFlags(Reg: OriginalExec);
7388 MI.getOperand(i: 0).setReg(OriginalExec);
7389 return BB;
7390 }
7391 case AMDGPU::V_DOT2_F32_F16:
7392 case AMDGPU::V_DOT2_F32_BF16: {
7393 // Hint RA to assign dst and src2 the same physical register.
7394 // For targets without VOP2, but with VOPD, variant of the instruction this
7395 // is one of the conditions to attempt converting VOP3P to VOPD.
7396 MRI.setSimpleHint(VReg: MI.getOperand(i: 0).getReg(), PrefReg: MI.getOperand(i: 6).getReg());
7397 return BB;
7398 }
7399 default:
7400 if (TII->isImage(MI) || TII->isMUBUF(MI)) {
7401 if (!MI.mayStore())
7402 AddMemOpInit(MI);
7403 return BB;
7404 }
7405 return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, MBB: BB);
7406 }
7407}
7408
7409bool SITargetLowering::enableAggressiveFMAFusion(EVT VT) const {
7410 // This currently forces unfolding various combinations of fsub into fma with
7411 // free fneg'd operands. As long as we have fast FMA (controlled by
7412 // isFMAFasterThanFMulAndFAdd), we should perform these.
7413
7414 // When fma is quarter rate, for f64 where add / sub are at best half rate,
7415 // most of these combines appear to be cycle neutral but save on instruction
7416 // count / code size.
7417 return true;
7418}
7419
7420bool SITargetLowering::enableAggressiveFMAFusion(LLT Ty) const { return true; }
7421
7422EVT SITargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx,
7423 EVT VT) const {
7424 if (!VT.isVector()) {
7425 return MVT::i1;
7426 }
7427 return EVT::getVectorVT(Context&: Ctx, VT: MVT::i1, NumElements: VT.getVectorNumElements());
7428}
7429
7430MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT VT) const {
7431 // TODO: Should i16 be used always if legal? For now it would force VALU
7432 // shifts.
7433 return (VT == MVT::i16) ? MVT::i16 : MVT::i32;
7434}
7435
7436LLT SITargetLowering::getPreferredShiftAmountTy(LLT Ty) const {
7437 return (Ty.getScalarSizeInBits() <= 16 && Subtarget->has16BitInsts())
7438 ? Ty.changeElementSize(NewEltSize: 16)
7439 : Ty.changeElementSize(NewEltSize: 32);
7440}
7441
7442// Answering this is somewhat tricky and depends on the specific device which
7443// have different rates for fma or all f64 operations.
7444//
7445// v_fma_f64 and v_mul_f64 always take the same number of cycles as each other
7446// regardless of which device (although the number of cycles differs between
7447// devices), so it is always profitable for f64.
7448//
7449// v_fma_f32 takes 4 or 16 cycles depending on the device, so it is profitable
7450// only on full rate devices. Normally, we should prefer selecting v_mad_f32
7451// which we can always do even without fused FP ops since it returns the same
7452// result as the separate operations and since it is always full
7453// rate. Therefore, we lie and report that it is not faster for f32. v_mad_f32
7454// however does not support denormals, so we do report fma as faster if we have
7455// a fast fma device and require denormals.
7456//
7457bool SITargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT,
7458 DenormalFPEnv FPEnv) const {
7459 VT = VT.getScalarType();
7460 if (!VT.isSimple())
7461 return false;
7462
7463 switch (VT.getSimpleVT().SimpleTy) {
7464 case MVT::f32: {
7465 // If mad is not available this depends only on if f32 fma is full rate.
7466 if (!Subtarget->hasMadMacF32Insts())
7467 return Subtarget->hasFastFMAF32();
7468
7469 // Otherwise f32 mad is always full rate and returns the same result as
7470 // the separate operations so should be preferred over fma.
7471 // However does not support denormals.
7472 if (FPEnv.F32Mode != DenormalMode::getPreserveSign())
7473 return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts();
7474
7475 // If the subtarget has v_fmac_f32, that's just as good as v_mac_f32.
7476 return Subtarget->hasFastFMAF32() && Subtarget->hasDLInsts();
7477 }
7478 case MVT::f64:
7479 return true;
7480 case MVT::f16:
7481 case MVT::bf16:
7482 return Subtarget->has16BitInsts() &&
7483 FPEnv.DefaultMode != DenormalMode::getPreserveSign();
7484 default:
7485 break;
7486 }
7487
7488 return false;
7489}
7490
7491bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
7492 EVT VT) const {
7493 return isFMAFasterThanFMulAndFAdd(VT, FPEnv: getDenormalFPEnv(MF));
7494}
7495
7496bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
7497 Type *Ty) const {
7498 return isFMAFasterThanFMulAndFAdd(
7499 VT: getValueType(DL: F.getDataLayout(), Ty, /*AllowUnknown=*/true),
7500 FPEnv: F.getDenormalFPEnv());
7501}
7502
7503bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
7504 LLT Ty) const {
7505 switch (Ty.getScalarSizeInBits()) {
7506 case 16:
7507 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f16);
7508 case 32:
7509 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f32);
7510 case 64:
7511 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f64);
7512 default:
7513 break;
7514 }
7515
7516 return false;
7517}
7518
7519bool SITargetLowering::isFMADLegal(EVT VT, DenormalFPEnv FPEnv) const {
7520 // TODO: Check future ftz flag
7521 // v_mad_f32/v_mac_f32 do not support denormals.
7522 if (VT == MVT::f32)
7523 return Subtarget->hasMadMacF32Insts() &&
7524 FPEnv.F32Mode == DenormalMode::getPreserveSign();
7525 if (VT == MVT::f16)
7526 return Subtarget->hasMadF16() &&
7527 FPEnv.DefaultMode == DenormalMode::getPreserveSign();
7528
7529 return false;
7530}
7531
7532bool SITargetLowering::isFMADLegal(const MachineInstr &MI, LLT Ty) const {
7533 if (!Ty.isScalar())
7534 return false;
7535
7536 DenormalFPEnv FPEnv = getDenormalFPEnv(MF: *MI.getMF());
7537 if (Ty.getScalarSizeInBits() == 16)
7538 return isFMADLegal(VT: MVT::f16, FPEnv);
7539 if (Ty.getScalarSizeInBits() == 32)
7540 return isFMADLegal(VT: MVT::f32, FPEnv);
7541
7542 return false;
7543}
7544
7545bool SITargetLowering::isFMADLegal(const SelectionDAG &DAG,
7546 const SDNode *N) const {
7547 return isFMADLegal(VT: N->getValueType(ResNo: 0),
7548 FPEnv: getDenormalFPEnv(MF: DAG.getMachineFunction()));
7549}
7550
7551//===----------------------------------------------------------------------===//
7552// Custom DAG Lowering Operations
7553//===----------------------------------------------------------------------===//
7554
7555// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
7556// wider vector type is legal.
7557SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op,
7558 SelectionDAG &DAG) const {
7559 unsigned Opc = Op.getOpcode();
7560 EVT VT = Op.getValueType();
7561 assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
7562
7563 auto [Lo, Hi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
7564 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
7565
7566 SDLoc SL(Op);
7567
7568 // Forward any trailing scalar operands unchanged to both halves.
7569 SmallVector<SDValue, 2> LoOps = {Lo};
7570 SmallVector<SDValue, 2> HiOps = {Hi};
7571 auto TrailingOps = drop_begin(RangeOrContainer: Op->ops());
7572 LoOps.append(in_start: TrailingOps.begin(), in_end: TrailingOps.end());
7573 HiOps.append(in_start: TrailingOps.begin(), in_end: TrailingOps.end());
7574
7575 SDValue OpLo = DAG.getNode(Opcode: Opc, DL: SL, VT: LoVT, Ops: LoOps, Flags: Op->getFlags());
7576 SDValue OpHi = DAG.getNode(Opcode: Opc, DL: SL, VT: HiVT, Ops: HiOps, Flags: Op->getFlags());
7577
7578 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7579}
7580
7581// Enable lowering of ROTR for vxi32 types. This is a workaround for a
7582// regression whereby extra unnecessary instructions were added to codegen
7583// for rotr operations, casued by legalising v2i32 or. This resulted in extra
7584// instructions to extract the result from the vector.
7585SDValue SITargetLowering::lowerROTR(SDValue Op, SelectionDAG &DAG) const {
7586 [[maybe_unused]] EVT VT = Op.getValueType();
7587
7588 assert((VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v8i32 ||
7589 VT == MVT::v16i32) &&
7590 "Unexpected ValueType.");
7591
7592 return DAG.UnrollVectorOp(N: Op.getNode());
7593}
7594
7595// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
7596// wider vector type is legal.
7597SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op,
7598 SelectionDAG &DAG) const {
7599 unsigned Opc = Op.getOpcode();
7600 EVT VT = Op.getValueType();
7601 assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
7602
7603 auto [Lo0, Hi0] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
7604 auto [Lo1, Hi1] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 1);
7605
7606 SDLoc SL(Op);
7607
7608 SDValue OpLo =
7609 DAG.getNode(Opcode: Opc, DL: SL, VT: Lo0.getValueType(), N1: Lo0, N2: Lo1, Flags: Op->getFlags());
7610 SDValue OpHi =
7611 DAG.getNode(Opcode: Opc, DL: SL, VT: Hi0.getValueType(), N1: Hi0, N2: Hi1, Flags: Op->getFlags());
7612
7613 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7614}
7615
7616SDValue SITargetLowering::splitTernaryVectorOp(SDValue Op,
7617 SelectionDAG &DAG) const {
7618 unsigned Opc = Op.getOpcode();
7619 EVT VT = Op.getValueType();
7620 assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
7621
7622 SDValue Op0 = Op.getOperand(i: 0);
7623 auto [Lo0, Hi0] = Op0.getValueType().isVector()
7624 ? DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0)
7625 : std::pair(Op0, Op0);
7626
7627 auto [Lo1, Hi1] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 1);
7628 auto [Lo2, Hi2] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 2);
7629
7630 SDLoc SL(Op);
7631 auto ResVT = DAG.GetSplitDestVTs(VT);
7632
7633 SDValue OpLo =
7634 DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT.first, N1: Lo0, N2: Lo1, N3: Lo2, Flags: Op->getFlags());
7635 SDValue OpHi =
7636 DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT.second, N1: Hi0, N2: Hi1, N3: Hi2, Flags: Op->getFlags());
7637
7638 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7639}
7640
7641SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
7642 switch (Op.getOpcode()) {
7643 default:
7644 return AMDGPUTargetLowering::LowerOperation(Op, DAG);
7645 case ISD::BRCOND:
7646 return LowerBRCOND(Op, DAG);
7647 case ISD::RETURNADDR:
7648 return LowerRETURNADDR(Op, DAG);
7649 case ISD::SPONENTRY:
7650 return LowerSPONENTRY(Op, DAG);
7651 case ISD::LOAD: {
7652 SDValue Result = LowerLOAD(Op, DAG);
7653 assert((!Result.getNode() || Result.getNode()->getNumValues() == 2) &&
7654 "Load should return a value and a chain");
7655 return Result;
7656 }
7657 case ISD::FSQRT: {
7658 EVT VT = Op.getValueType();
7659 if (VT == MVT::f32)
7660 return lowerFSQRTF32(Op, DAG);
7661 if (VT == MVT::f64)
7662 return lowerFSQRTF64(Op, DAG);
7663 return SDValue();
7664 }
7665 case ISD::FSIN:
7666 case ISD::FCOS:
7667 return LowerTrig(Op, DAG);
7668 case ISD::SELECT:
7669 return LowerSELECT(Op, DAG);
7670 case ISD::FDIV:
7671 return LowerFDIV(Op, DAG);
7672 case ISD::FFREXP:
7673 return LowerFFREXP(Op, DAG);
7674 case ISD::ATOMIC_CMP_SWAP:
7675 return LowerATOMIC_CMP_SWAP(Op, DAG);
7676 case ISD::STORE:
7677 return LowerSTORE(Op, DAG);
7678 case ISD::GlobalAddress: {
7679 MachineFunction &MF = DAG.getMachineFunction();
7680 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
7681 return LowerGlobalAddress(MFI, Op, DAG);
7682 }
7683 case ISD::BlockAddress:
7684 return LowerBlockAddress(Op, DAG);
7685 case ISD::ExternalSymbol:
7686 return LowerExternalSymbol(Op, DAG);
7687 case ISD::INTRINSIC_WO_CHAIN:
7688 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
7689 case ISD::CONVERT_FROM_ARBITRARY_FP:
7690 return LowerCONVERT_FROM_ARBITRARY_FP(Op, DAG);
7691 case ISD::CONVERT_TO_ARBITRARY_FP:
7692 return LowerCONVERT_TO_ARBITRARY_FP(Op, DAG);
7693 case ISD::INTRINSIC_W_CHAIN:
7694 return LowerINTRINSIC_W_CHAIN(Op, DAG);
7695 case ISD::INTRINSIC_VOID:
7696 return LowerINTRINSIC_VOID(Op, DAG);
7697 case ISD::ADDRSPACECAST:
7698 return lowerADDRSPACECAST(Op, DAG);
7699 case ISD::INSERT_SUBVECTOR:
7700 return lowerINSERT_SUBVECTOR(Op, DAG);
7701 case ISD::INSERT_VECTOR_ELT:
7702 return lowerINSERT_VECTOR_ELT(Op, DAG);
7703 case ISD::EXTRACT_VECTOR_ELT:
7704 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
7705 case ISD::VECTOR_SHUFFLE:
7706 return lowerVECTOR_SHUFFLE(Op, DAG);
7707 case ISD::SCALAR_TO_VECTOR:
7708 return lowerSCALAR_TO_VECTOR(Op, DAG);
7709 case ISD::BUILD_VECTOR:
7710 return lowerBUILD_VECTOR(Op, DAG);
7711 case ISD::FP_ROUND:
7712 case ISD::STRICT_FP_ROUND:
7713 return lowerFP_ROUND(Op, DAG);
7714 case ISD::TRAP:
7715 return lowerTRAP(Op, DAG);
7716 case ISD::DEBUGTRAP:
7717 return lowerDEBUGTRAP(Op, DAG);
7718 case ISD::ABS:
7719 case ISD::FABS:
7720 case ISD::FNEG:
7721 case ISD::FCANONICALIZE:
7722 case ISD::BSWAP:
7723 return splitUnaryVectorOp(Op, DAG);
7724 case ISD::FP_TO_SINT_SAT:
7725 case ISD::FP_TO_UINT_SAT:
7726 if (Op.getValueType().isVector() && Op.getValueType() != MVT::v2i16 &&
7727 Op.getOperand(i: 0).getValueType().getScalarType() == MVT::f32)
7728 return splitUnaryVectorOp(Op, DAG);
7729 return LowerFP_TO_INT_SAT(Op, DAG);
7730 case ISD::FSUB:
7731 if (Op.getValueType() == MVT::bf16) {
7732 // Custom expansion:
7733 // fsub bf16 %a, %b -> fadd v2bf16(widen %a), fneg v2bf16(widen %b)
7734 // Then extract back to bf16.
7735 //
7736 // We create fneg on v2bf16 (not bf16) so the instruction selector can
7737 // fold the negation into the packed add's neg_lo/neg_hi modifiers,
7738 // generating a single v_pk_add_bf16 instruction. If we negate bf16 first,
7739 // it becomes a separate v_xor instruction before widening.
7740 SDLoc DL(Op);
7741 SDValue Op0 = Op.getOperand(i: 0);
7742 SDValue Op1 = Op.getOperand(i: 1);
7743
7744 // Widen both operands to v2bf16
7745 SDValue Vec0 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2bf16, Operand: Op0);
7746 SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2bf16, Operand: Op1);
7747
7748 // Create FNEG v2bf16 for the second operand
7749 SDValue NegVec1 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::v2bf16, Operand: Vec1);
7750
7751 // Perform FADD v2bf16
7752 SDValue Result = DAG.getNode(Opcode: ISD::FADD, DL, VT: MVT::v2bf16, N1: Vec0, N2: NegVec1);
7753
7754 // Extract element 0 back to bf16
7755 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::bf16, N1: Result,
7756 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
7757 }
7758 return SDValue();
7759 case ISD::FMINNUM:
7760 case ISD::FMAXNUM:
7761 return lowerFMINNUM_FMAXNUM(Op, DAG);
7762 case ISD::FMINIMUMNUM:
7763 case ISD::FMAXIMUMNUM:
7764 return lowerFMINIMUMNUM_FMAXIMUMNUM(Op, DAG);
7765 case ISD::FLDEXP:
7766 case ISD::STRICT_FLDEXP:
7767 return lowerFLDEXP(Op, DAG);
7768 case ISD::FMA:
7769 return splitTernaryVectorOp(Op, DAG);
7770 case ISD::FP_TO_SINT:
7771 case ISD::FP_TO_UINT:
7772 if (Subtarget->hasVCvtPkIU16F32() && Op.getValueType() == MVT::i16 &&
7773 Op.getOperand(i: 0).getValueType() == MVT::f32) {
7774 // Make f32->i16 legal so we can select V_CVT_PK_[IU]16_F32.
7775 return Op;
7776 }
7777 return LowerFP_TO_INT(Op, DAG);
7778 case ISD::SHL:
7779 case ISD::SRA:
7780 case ISD::SRL:
7781 case ISD::ADD:
7782 case ISD::SUB:
7783 case ISD::SMIN:
7784 case ISD::SMAX:
7785 case ISD::UMIN:
7786 case ISD::UMAX:
7787 case ISD::FMINNUM_IEEE:
7788 case ISD::FMAXNUM_IEEE:
7789 case ISD::FMINIMUM:
7790 case ISD::FMAXIMUM:
7791 case ISD::UADDSAT:
7792 case ISD::USUBSAT:
7793 case ISD::SADDSAT:
7794 case ISD::SSUBSAT:
7795 case ISD::FADD:
7796 case ISD::FMUL:
7797 return splitBinaryVectorOp(Op, DAG);
7798 case ISD::FCOPYSIGN:
7799 return lowerFCOPYSIGN(Op, DAG);
7800 case ISD::MUL:
7801 return lowerMUL(Op, DAG);
7802 case ISD::SMULO:
7803 case ISD::UMULO:
7804 return lowerXMULO(Op, DAG);
7805 case ISD::SMUL_LOHI:
7806 case ISD::UMUL_LOHI:
7807 return lowerXMUL_LOHI(Op, DAG);
7808 case ISD::DYNAMIC_STACKALLOC:
7809 return LowerDYNAMIC_STACKALLOC(Op, DAG);
7810 case ISD::STACKSAVE:
7811 return LowerSTACKSAVE(Op, DAG);
7812 case ISD::GET_ROUNDING:
7813 return lowerGET_ROUNDING(Op, DAG);
7814 case ISD::SET_ROUNDING:
7815 return lowerSET_ROUNDING(Op, DAG);
7816 case ISD::PREFETCH:
7817 return lowerPREFETCH(Op, DAG);
7818 case ISD::FP_EXTEND:
7819 case ISD::STRICT_FP_EXTEND:
7820 return lowerFP_EXTEND(Op, DAG);
7821 case ISD::GET_FPENV:
7822 return lowerGET_FPENV(Op, DAG);
7823 case ISD::SET_FPENV:
7824 return lowerSET_FPENV(Op, DAG);
7825 case ISD::ROTR:
7826 return lowerROTR(Op, DAG);
7827 case ISD::INLINEASM:
7828 return LowerINLINEASM(Op, DAG);
7829 }
7830 return SDValue();
7831}
7832
7833// Used for D16: Casts the result of an instruction into the right vector,
7834// packs values if loads return unpacked values.
7835static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
7836 const SDLoc &DL, SelectionDAG &DAG,
7837 bool Unpacked) {
7838 if (!LoadVT.isVector())
7839 return Result;
7840
7841 // Cast back to the original packed type or to a larger type that is a
7842 // multiple of 32 bit for D16. Widening the return type is a required for
7843 // legalization.
7844 EVT FittingLoadVT = LoadVT;
7845 if ((LoadVT.getVectorNumElements() % 2) == 1) {
7846 FittingLoadVT =
7847 EVT::getVectorVT(Context&: *DAG.getContext(), VT: LoadVT.getVectorElementType(),
7848 NumElements: LoadVT.getVectorNumElements() + 1);
7849 }
7850
7851 if (Unpacked) { // From v2i32/v4i32 back to v2f16/v4f16.
7852 // Truncate to v2i16/v4i16.
7853 EVT IntLoadVT = FittingLoadVT.changeTypeToInteger();
7854
7855 // Workaround legalizer not scalarizing truncate after vector op
7856 // legalization but not creating intermediate vector trunc.
7857 SmallVector<SDValue, 4> Elts;
7858 DAG.ExtractVectorElements(Op: Result, Args&: Elts);
7859 for (SDValue &Elt : Elts)
7860 Elt = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Elt);
7861
7862 // Pad illegal v1i16/v3fi6 to v4i16
7863 if ((LoadVT.getVectorNumElements() % 2) == 1)
7864 Elts.push_back(Elt: DAG.getPOISON(VT: MVT::i16));
7865
7866 Result = DAG.getBuildVector(VT: IntLoadVT, DL, Ops: Elts);
7867
7868 // Bitcast to original type (v2f16/v4f16).
7869 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FittingLoadVT, Operand: Result);
7870 }
7871
7872 // Cast back to the original packed type.
7873 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FittingLoadVT, Operand: Result);
7874}
7875
7876SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, MemSDNode *M,
7877 SelectionDAG &DAG,
7878 ArrayRef<SDValue> Ops,
7879 bool IsIntrinsic) const {
7880 SDLoc DL(M);
7881
7882 bool Unpacked = Subtarget->hasUnpackedD16VMem();
7883 EVT LoadVT = M->getValueType(ResNo: 0);
7884
7885 EVT EquivLoadVT = LoadVT;
7886 if (LoadVT.isVector()) {
7887 if (Unpacked) {
7888 EquivLoadVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
7889 NumElements: LoadVT.getVectorNumElements());
7890 } else if ((LoadVT.getVectorNumElements() % 2) == 1) {
7891 // Widen v3f16 to legal type
7892 EquivLoadVT =
7893 EVT::getVectorVT(Context&: *DAG.getContext(), VT: LoadVT.getVectorElementType(),
7894 NumElements: LoadVT.getVectorNumElements() + 1);
7895 }
7896 }
7897
7898 // Change from v4f16/v2f16 to EquivLoadVT.
7899 SDVTList VTList = DAG.getVTList(VT1: EquivLoadVT, VT2: MVT::Other);
7900
7901 SDValue Load = DAG.getMemIntrinsicNode(
7902 Opcode: IsIntrinsic ? (unsigned)ISD::INTRINSIC_W_CHAIN : Opcode, dl: DL, VTList, Ops,
7903 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
7904
7905 SDValue Adjusted = adjustLoadValueTypeImpl(Result: Load, LoadVT, DL, DAG, Unpacked);
7906
7907 return DAG.getMergeValues(Ops: {Adjusted, Load.getValue(R: 1)}, dl: DL);
7908}
7909
7910SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
7911 SelectionDAG &DAG,
7912 ArrayRef<SDValue> Ops) const {
7913 SDLoc DL(M);
7914 EVT LoadVT = M->getValueType(ResNo: 0);
7915 EVT EltType = LoadVT.getScalarType();
7916 EVT IntVT = LoadVT.changeTypeToInteger();
7917
7918 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
7919
7920 if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
7921 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
7922 DAG.getMachineFunction().getFunction(),
7923 "unsupported sub-dword format buffer load", DL.getDebugLoc()));
7924 return DAG.getMergeValues(Ops: {DAG.getPOISON(VT: LoadVT), M->getOperand(Num: 0)}, dl: DL);
7925 }
7926
7927 assert(M->getNumValues() == 2 || M->getNumValues() == 3);
7928 bool IsTFE = M->getNumValues() == 3;
7929
7930 unsigned Opc = IsFormat ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_TFE
7931 : AMDGPUISD::BUFFER_LOAD_FORMAT)
7932 : IsTFE ? AMDGPUISD::BUFFER_LOAD_TFE
7933 : AMDGPUISD::BUFFER_LOAD;
7934
7935 if (IsD16) {
7936 return adjustLoadValueType(Opcode: AMDGPUISD::BUFFER_LOAD_FORMAT_D16, M, DAG, Ops);
7937 }
7938
7939 // Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics
7940 if (!IsD16 && !LoadVT.isVector() && EltType.getSizeInBits() < 32)
7941 return handleByteShortBufferLoads(DAG, LoadVT, DL, Ops, MMO: M->getMemOperand(),
7942 IsTFE);
7943
7944 if (isTypeLegal(VT: LoadVT)) {
7945 return getMemIntrinsicNode(Opcode: Opc, DL, VTList: M->getVTList(), Ops, MemVT: IntVT,
7946 MMO: M->getMemOperand(), DAG);
7947 }
7948
7949 EVT CastVT = getEquivalentMemType(Context&: *DAG.getContext(), VT: LoadVT);
7950 SDVTList VTList = DAG.getVTList(VT1: CastVT, VT2: MVT::Other);
7951 SDValue MemNode = getMemIntrinsicNode(Opcode: Opc, DL, VTList, Ops, MemVT: CastVT,
7952 MMO: M->getMemOperand(), DAG);
7953 return DAG.getMergeValues(
7954 Ops: {DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: MemNode), MemNode.getValue(R: 1)},
7955 dl: DL);
7956}
7957
7958static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N,
7959 SelectionDAG &DAG) {
7960 EVT VT = N->getValueType(ResNo: 0);
7961 SDValue Src = N->getOperand(Num: 1);
7962 SDLoc SL(N);
7963
7964 if (Src.getOpcode() == ISD::SETCC) {
7965 SDValue Op0 = Src.getOperand(i: 0);
7966 SDValue Op1 = Src.getOperand(i: 1);
7967 // Need to expand bfloat to float for comparison (setcc).
7968 if (Op0.getValueType() == MVT::bf16) {
7969 Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op0);
7970 Op1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op1);
7971 }
7972 // (ballot (ISD::SETCC ...)) -> (AMDGPUISD::SETCC ...)
7973 return DAG.getNode(Opcode: AMDGPUISD::SETCC, DL: SL, VT, N1: Op0, N2: Op1, N3: Src.getOperand(i: 2));
7974 }
7975 if (const ConstantSDNode *Arg = dyn_cast<ConstantSDNode>(Val&: Src)) {
7976 // (ballot 0) -> 0
7977 if (Arg->isZero())
7978 return DAG.getConstant(Val: 0, DL: SL, VT);
7979
7980 // (ballot 1) -> EXEC/EXEC_LO
7981 if (Arg->isOne()) {
7982 Register Exec;
7983 if (VT.getScalarSizeInBits() == 32)
7984 Exec = AMDGPU::EXEC_LO;
7985 else if (VT.getScalarSizeInBits() == 64)
7986 Exec = AMDGPU::EXEC;
7987 else
7988 return SDValue();
7989
7990 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: Exec, VT);
7991 }
7992 }
7993
7994 // (ballot (i1 $src)) -> (AMDGPUISD::SETCC (i32 (zext $src)) (i32 0)
7995 // ISD::SETNE)
7996 return DAG.getNode(
7997 Opcode: AMDGPUISD::SETCC, DL: SL, VT, N1: DAG.getZExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32),
7998 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), N3: DAG.getCondCode(Cond: ISD::SETNE));
7999}
8000
8001static SDValue lowerBFEIntrinsic(SDValue Op, SelectionDAG &DAG,
8002 Intrinsic::ID IntrinsicID) {
8003 bool Signed = IntrinsicID == Intrinsic::amdgcn_sbfe;
8004 SDLoc DL(Op);
8005 EVT VT = Op.getValueType();
8006 SDValue Src = Op.getOperand(i: 1);
8007 SDValue Offset = Op.getOperand(i: 2);
8008 SDValue Width = Op.getOperand(i: 3);
8009
8010 if (VT != MVT::i32) {
8011 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
8012 DAG.getMachineFunction().getFunction(),
8013 Twine(Intrinsic::getBaseName(id: IntrinsicID)) + " only supports i32",
8014 DL.getDebugLoc()));
8015 return DAG.getPOISON(VT);
8016 }
8017
8018 return DAG.getNode(Opcode: Signed ? AMDGPUISD::BFE_I32 : AMDGPUISD::BFE_U32, DL, VT,
8019 N1: Src, N2: Offset, N3: Width);
8020}
8021
8022static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
8023 EVT VT);
8024
8025static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N,
8026 SelectionDAG &DAG) {
8027 EVT VT = N->getValueType(ResNo: 0);
8028 unsigned ValSize = VT.getSizeInBits();
8029 unsigned IID = N->getConstantOperandVal(Num: 0);
8030 bool IsPermLane16 = IID == Intrinsic::amdgcn_permlane16 ||
8031 IID == Intrinsic::amdgcn_permlanex16;
8032 bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
8033 IID == Intrinsic::amdgcn_set_inactive_chain_arg;
8034 bool IsPermlaneShuffle = IID == Intrinsic::amdgcn_permlane_bcast ||
8035 IID == Intrinsic::amdgcn_permlane_up ||
8036 IID == Intrinsic::amdgcn_permlane_down ||
8037 IID == Intrinsic::amdgcn_permlane_xor;
8038 SDLoc SL(N);
8039 MVT IntVT = MVT::getIntegerVT(BitWidth: ValSize);
8040 const GCNSubtarget *ST = TLI.getSubtarget();
8041
8042 unsigned SplitSize = 32;
8043 if (IID == Intrinsic::amdgcn_update_dpp && (ValSize % 64 == 0) &&
8044 ST->hasDPALU_DPP() &&
8045 AMDGPU::isLegalDPALU_DPPControl(ST: *ST, DC: N->getConstantOperandVal(Num: 3)))
8046 SplitSize = 64;
8047
8048 auto createLaneOp = [&DAG, &SL, N, IID](SDValue Src0, SDValue Src1,
8049 SDValue Src2, MVT ValT) -> SDValue {
8050 SmallVector<SDValue, 8> Operands;
8051 switch (IID) {
8052 case Intrinsic::amdgcn_permlane16:
8053 case Intrinsic::amdgcn_permlanex16:
8054 case Intrinsic::amdgcn_update_dpp:
8055 Operands.push_back(Elt: N->getOperand(Num: 6));
8056 Operands.push_back(Elt: N->getOperand(Num: 5));
8057 Operands.push_back(Elt: N->getOperand(Num: 4));
8058 [[fallthrough]];
8059 case Intrinsic::amdgcn_writelane:
8060 case Intrinsic::amdgcn_permlane_bcast:
8061 case Intrinsic::amdgcn_permlane_up:
8062 case Intrinsic::amdgcn_permlane_down:
8063 case Intrinsic::amdgcn_permlane_xor:
8064 Operands.push_back(Elt: Src2);
8065 [[fallthrough]];
8066 case Intrinsic::amdgcn_readlane:
8067 case Intrinsic::amdgcn_set_inactive:
8068 case Intrinsic::amdgcn_set_inactive_chain_arg:
8069 case Intrinsic::amdgcn_mov_dpp8:
8070 Operands.push_back(Elt: Src1);
8071 [[fallthrough]];
8072 case Intrinsic::amdgcn_readfirstlane:
8073 case Intrinsic::amdgcn_permlane64:
8074 Operands.push_back(Elt: Src0);
8075 break;
8076 default:
8077 llvm_unreachable("unhandled lane op");
8078 }
8079
8080 Operands.push_back(Elt: DAG.getTargetConstant(Val: IID, DL: SL, VT: MVT::i32));
8081 std::reverse(first: Operands.begin(), last: Operands.end());
8082
8083 if (SDNode *GL = N->getGluedNode()) {
8084 assert(GL->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
8085 GL = GL->getOperand(Num: 0).getNode();
8086 Operands.push_back(Elt: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL: SL, VT: MVT::Glue,
8087 Operand: SDValue(GL, 0)));
8088 }
8089
8090 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: ValT, Ops: Operands);
8091 };
8092
8093 SDValue Src0 = N->getOperand(Num: 1);
8094 SDValue Src1, Src2;
8095 if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
8096 IID == Intrinsic::amdgcn_mov_dpp8 ||
8097 IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16 ||
8098 IsPermlaneShuffle) {
8099 Src1 = N->getOperand(Num: 2);
8100 if (IID == Intrinsic::amdgcn_writelane ||
8101 IID == Intrinsic::amdgcn_update_dpp || IsPermLane16 ||
8102 IsPermlaneShuffle)
8103 Src2 = N->getOperand(Num: 3);
8104 }
8105
8106 if (ValSize == SplitSize) {
8107 // Already legal
8108 return SDValue();
8109 }
8110
8111 if (ValSize < 32) {
8112 bool IsFloat = VT.isFloatingPoint();
8113 Src0 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src0) : Src0,
8114 DL: SL, VT: MVT::i32);
8115
8116 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) {
8117 Src1 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src1) : Src1,
8118 DL: SL, VT: MVT::i32);
8119 }
8120
8121 if (IID == Intrinsic::amdgcn_writelane) {
8122 Src2 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src2) : Src2,
8123 DL: SL, VT: MVT::i32);
8124 }
8125
8126 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, MVT::i32);
8127 SDValue Trunc = DAG.getAnyExtOrTrunc(Op: LaneOp, DL: SL, VT: IntVT);
8128 return IsFloat ? DAG.getBitcast(VT, V: Trunc) : Trunc;
8129 }
8130
8131 if (ValSize % SplitSize != 0)
8132 return SDValue();
8133
8134 auto unrollLaneOp = [&DAG, &SL](SDNode *N) -> SDValue {
8135 EVT VT = N->getValueType(ResNo: 0);
8136 unsigned NE = VT.getVectorNumElements();
8137 EVT EltVT = VT.getVectorElementType();
8138 SmallVector<SDValue, 8> Scalars;
8139 unsigned NumOperands = N->getNumOperands();
8140 SmallVector<SDValue, 4> Operands(NumOperands);
8141 SDNode *GL = N->getGluedNode();
8142
8143 // only handle convergencectrl_glue
8144 assert(!GL || GL->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
8145
8146 for (unsigned i = 0; i != NE; ++i) {
8147 for (unsigned j = 0, e = GL ? NumOperands - 1 : NumOperands; j != e;
8148 ++j) {
8149 SDValue Operand = N->getOperand(Num: j);
8150 EVT OperandVT = Operand.getValueType();
8151 if (OperandVT.isVector()) {
8152 // A vector operand; extract a single element.
8153 EVT OperandEltVT = OperandVT.getVectorElementType();
8154 Operands[j] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: OperandEltVT,
8155 N1: Operand, N2: DAG.getVectorIdxConstant(Val: i, DL: SL));
8156 } else {
8157 // A scalar operand; just use it as is.
8158 Operands[j] = Operand;
8159 }
8160 }
8161
8162 if (GL)
8163 Operands[NumOperands - 1] =
8164 DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL: SL, VT: MVT::Glue,
8165 Operand: SDValue(GL->getOperand(Num: 0).getNode(), 0));
8166
8167 Scalars.push_back(Elt: DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: EltVT, Ops: Operands));
8168 }
8169
8170 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NE);
8171 return DAG.getBuildVector(VT: VecVT, DL: SL, Ops: Scalars);
8172 };
8173
8174 if (VT.isVector()) {
8175 switch (MVT::SimpleValueType EltTy =
8176 VT.getVectorElementType().getSimpleVT().SimpleTy) {
8177 case MVT::i32:
8178 case MVT::f32:
8179 if (SplitSize == 32) {
8180 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VT.getSimpleVT());
8181 return unrollLaneOp(LaneOp.getNode());
8182 }
8183 [[fallthrough]];
8184 case MVT::i16:
8185 case MVT::f16:
8186 case MVT::bf16: {
8187 unsigned SubVecNumElt =
8188 SplitSize / VT.getVectorElementType().getSizeInBits();
8189 MVT SubVecVT = MVT::getVectorVT(VT: EltTy, NumElements: SubVecNumElt);
8190 SmallVector<SDValue, 4> Pieces;
8191 SDValue Src0SubVec, Src1SubVec, Src2SubVec;
8192 for (unsigned i = 0, EltIdx = 0; i < ValSize / SplitSize; i++) {
8193 Src0SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src0,
8194 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
8195
8196 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive ||
8197 IsPermLane16) {
8198 Src1SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src1,
8199 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
8200
8201 Pieces.push_back(
8202 Elt: createLaneOp(Src0SubVec, Src1SubVec, Src2, SubVecVT));
8203 } else if (IID == Intrinsic::amdgcn_writelane) {
8204 Src2SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src2,
8205 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
8206 Pieces.push_back(
8207 Elt: createLaneOp(Src0SubVec, Src1, Src2SubVec, SubVecVT));
8208 } else {
8209 Pieces.push_back(Elt: createLaneOp(Src0SubVec, Src1, Src2, SubVecVT));
8210 }
8211
8212 EltIdx += SubVecNumElt;
8213 }
8214 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT, Ops: Pieces);
8215 }
8216 default:
8217 // Handle all other cases by bitcasting to i32 vectors
8218 break;
8219 }
8220 }
8221
8222 MVT VecVT =
8223 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplitSize), NumElements: ValSize / SplitSize);
8224 Src0 = DAG.getBitcast(VT: VecVT, V: Src0);
8225
8226 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
8227 Src1 = DAG.getBitcast(VT: VecVT, V: Src1);
8228
8229 if (IID == Intrinsic::amdgcn_writelane)
8230 Src2 = DAG.getBitcast(VT: VecVT, V: Src2);
8231
8232 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VecVT);
8233 SDValue UnrolledLaneOp = unrollLaneOp(LaneOp.getNode());
8234 return DAG.getBitcast(VT, V: UnrolledLaneOp);
8235}
8236
8237static SDValue lowerWaveShuffle(const SITargetLowering &TLI, SDNode *N,
8238 SelectionDAG &DAG) {
8239 EVT VT = N->getValueType(ResNo: 0);
8240
8241 if (VT.getSizeInBits() != 32)
8242 return SDValue();
8243
8244 SDLoc SL(N);
8245
8246 SDValue Value = N->getOperand(Num: 1);
8247 SDValue Index = N->getOperand(Num: 2);
8248
8249 // ds_bpermute requires index to be multiplied by 4
8250 SDValue ShiftAmount = DAG.getShiftAmountConstant(Val: 2, VT: MVT::i32, DL: SL);
8251 SDValue ShiftedIndex =
8252 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: Index.getValueType(), N1: Index, N2: ShiftAmount);
8253
8254 // Intrinsics will require i32 to operate on
8255 SDValue ValueI32 = DAG.getBitcast(VT: MVT::i32, V: Value);
8256
8257 auto MakeIntrinsic = [&DAG, &SL](unsigned IID, MVT RetVT,
8258 SmallVector<SDValue> IntrinArgs) -> SDValue {
8259 SmallVector<SDValue> Operands(1);
8260 Operands[0] = DAG.getTargetConstant(Val: IID, DL: SL, VT: MVT::i32);
8261 Operands.append(RHS: IntrinArgs);
8262 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: RetVT, Ops: Operands);
8263 };
8264
8265 // If we can bpermute across the whole wave, then just do that
8266 if (TLI.getSubtarget()->supportsWaveWideBPermute()) {
8267 SDValue BPermute = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
8268 {ShiftedIndex, ValueI32});
8269 return DAG.getBitcast(VT, V: BPermute);
8270 }
8271
8272 assert(TLI.getSubtarget()->isWave64());
8273
8274 // Otherwise, we need to make use of whole wave mode
8275 SDValue PoisonVal = DAG.getPOISON(VT: ValueI32->getValueType(ResNo: 0));
8276
8277 // Set inactive lanes to poison
8278 SDValue WWMValue = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
8279 {ValueI32, PoisonVal});
8280 SDValue WWMIndex = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
8281 {ShiftedIndex, PoisonVal});
8282
8283 SDValue Swapped =
8284 MakeIntrinsic(Intrinsic::amdgcn_permlane64, MVT::i32, {WWMValue});
8285
8286 // Get permutation of each half, then we'll select which one to use
8287 SDValue BPermSameHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
8288 {WWMIndex, WWMValue});
8289 SDValue BPermOtherHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute,
8290 MVT::i32, {WWMIndex, Swapped});
8291 SDValue BPermOtherHalfWWM =
8292 MakeIntrinsic(Intrinsic::amdgcn_wwm, MVT::i32, {BPermOtherHalf});
8293
8294 // Select which side to take the permute from
8295 SDValue ThreadIDMask = DAG.getAllOnesConstant(DL: SL, VT: MVT::i32);
8296 // We can get away with only using mbcnt_lo here since we're only
8297 // trying to detect which side of 32 each lane is on, and mbcnt_lo
8298 // returns 32 for lanes 32-63.
8299 SDValue ThreadID =
8300 MakeIntrinsic(Intrinsic::amdgcn_mbcnt_lo, MVT::i32,
8301 {ThreadIDMask, DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32)});
8302
8303 SDValue SameOrOtherHalf =
8304 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32,
8305 N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: ThreadID, N2: Index),
8306 N2: DAG.getTargetConstant(Val: 32, DL: SL, VT: MVT::i32));
8307 SDValue UseSameHalf =
8308 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: SameOrOtherHalf,
8309 RHS: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond: ISD::SETEQ);
8310 SDValue Result = DAG.getSelect(DL: SL, VT: MVT::i32, Cond: UseSameHalf, LHS: BPermSameHalf,
8311 RHS: BPermOtherHalfWWM);
8312 return DAG.getBitcast(VT, V: Result);
8313}
8314
8315void SITargetLowering::ReplaceNodeResults(SDNode *N,
8316 SmallVectorImpl<SDValue> &Results,
8317 SelectionDAG &DAG) const {
8318 switch (N->getOpcode()) {
8319 case ISD::INSERT_VECTOR_ELT: {
8320 if (SDValue Res = lowerINSERT_VECTOR_ELT(Op: SDValue(N, 0), DAG))
8321 Results.push_back(Elt: Res);
8322 return;
8323 }
8324 case ISD::EXTRACT_VECTOR_ELT: {
8325 if (SDValue Res = lowerEXTRACT_VECTOR_ELT(Op: SDValue(N, 0), DAG))
8326 Results.push_back(Elt: Res);
8327 return;
8328 }
8329 case ISD::CONVERT_TO_ARBITRARY_FP: {
8330 if (SDValue Res = LowerCONVERT_TO_ARBITRARY_FP(Op: SDValue(N, 0), DAG))
8331 Results.push_back(Elt: Res);
8332 return;
8333 }
8334 case ISD::INTRINSIC_WO_CHAIN: {
8335 unsigned IID = N->getConstantOperandVal(Num: 0);
8336 switch (IID) {
8337 case Intrinsic::amdgcn_wave_reduce_min:
8338 case Intrinsic::amdgcn_wave_reduce_umin:
8339 case Intrinsic::amdgcn_wave_reduce_max:
8340 case Intrinsic::amdgcn_wave_reduce_umax:
8341 case Intrinsic::amdgcn_wave_reduce_add:
8342 case Intrinsic::amdgcn_wave_reduce_sub:
8343 case Intrinsic::amdgcn_wave_reduce_and:
8344 case Intrinsic::amdgcn_wave_reduce_or:
8345 case Intrinsic::amdgcn_wave_reduce_xor: {
8346 EVT VT = N->getValueType(ResNo: 0);
8347 if (isTypeLegal(VT))
8348 return;
8349 SDLoc SL(N);
8350 bool NeedsSignExt = IID == Intrinsic::amdgcn_wave_reduce_min ||
8351 IID == Intrinsic::amdgcn_wave_reduce_max ||
8352 IID == Intrinsic::amdgcn_wave_reduce_add ||
8353 IID == Intrinsic::amdgcn_wave_reduce_sub;
8354 unsigned ExtOpc = NeedsSignExt ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
8355 SDValue ExtSrc = DAG.getNode(Opcode: ExtOpc, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 1));
8356 SDValue Result = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
8357 N1: N->getOperand(Num: 0), N2: ExtSrc, N3: N->getOperand(Num: 2));
8358 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Result));
8359 return;
8360 }
8361 case Intrinsic::amdgcn_make_buffer_rsrc:
8362 Results.push_back(Elt: lowerPointerAsRsrcIntrin(Op: N, DAG));
8363 return;
8364 case Intrinsic::amdgcn_cvt_pkrtz: {
8365 SDValue Src0 = N->getOperand(Num: 1);
8366 SDValue Src1 = N->getOperand(Num: 2);
8367 SDLoc SL(N);
8368 SDValue Cvt =
8369 DAG.getNode(Opcode: AMDGPUISD::CVT_PKRTZ_F16_F32, DL: SL, VT: MVT::i32, N1: Src0, N2: Src1);
8370 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Cvt));
8371 return;
8372 }
8373 case Intrinsic::amdgcn_cvt_pknorm_i16:
8374 case Intrinsic::amdgcn_cvt_pknorm_u16:
8375 case Intrinsic::amdgcn_cvt_pk_i16:
8376 case Intrinsic::amdgcn_cvt_pk_u16: {
8377 SDValue Src0 = N->getOperand(Num: 1);
8378 SDValue Src1 = N->getOperand(Num: 2);
8379 SDLoc SL(N);
8380 unsigned Opcode;
8381
8382 if (IID == Intrinsic::amdgcn_cvt_pknorm_i16)
8383 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
8384 else if (IID == Intrinsic::amdgcn_cvt_pknorm_u16)
8385 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
8386 else if (IID == Intrinsic::amdgcn_cvt_pk_i16)
8387 Opcode = AMDGPUISD::CVT_PK_I16_I32;
8388 else
8389 Opcode = AMDGPUISD::CVT_PK_U16_U32;
8390
8391 EVT VT = N->getValueType(ResNo: 0);
8392 if (isTypeLegal(VT))
8393 Results.push_back(Elt: DAG.getNode(Opcode, DL: SL, VT, N1: Src0, N2: Src1));
8394 else {
8395 SDValue Cvt = DAG.getNode(Opcode, DL: SL, VT: MVT::i32, N1: Src0, N2: Src1);
8396 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: Cvt));
8397 }
8398 return;
8399 }
8400 case Intrinsic::amdgcn_s_buffer_load: {
8401 // Lower llvm.amdgcn.s.buffer.load.(i8, u8) intrinsics. First, we generate
8402 // s_buffer_load_u8 for signed and unsigned load instructions. Next, DAG
8403 // combiner tries to merge the s_buffer_load_u8 with a sext instruction
8404 // (performSignExtendInRegCombine()) and it replaces s_buffer_load_u8 with
8405 // s_buffer_load_i8.
8406 if (!Subtarget->hasScalarSubwordLoads())
8407 return;
8408 SDValue Op = SDValue(N, 0);
8409 SDValue Rsrc = Op.getOperand(i: 1);
8410 SDValue Offset = Op.getOperand(i: 2);
8411 SDValue CachePolicy = Op.getOperand(i: 3);
8412 EVT VT = Op.getValueType();
8413 assert(VT == MVT::i8 && "Expected 8-bit s_buffer_load intrinsics.\n");
8414 SDLoc DL(Op);
8415 MachineFunction &MF = DAG.getMachineFunction();
8416 const DataLayout &DataLayout = DAG.getDataLayout();
8417 Align Alignment =
8418 DataLayout.getABITypeAlign(Ty: VT.getTypeForEVT(Context&: *DAG.getContext()));
8419 MachineMemOperand *MMO = MF.getMachineMemOperand(
8420 PtrInfo: MachinePointerInfo(),
8421 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
8422 MachineMemOperand::MOInvariant,
8423 Size: VT.getStoreSize(), BaseAlignment: Alignment);
8424 SDValue LoadVal;
8425 if (!Offset->isDivergent()) {
8426 SDValue Ops[] = {DAG.getEntryNode(), // Chain
8427 Rsrc, // source register
8428 Offset, CachePolicy};
8429 SDValue BufferLoad = DAG.getMemIntrinsicNode(
8430 Opcode: AMDGPUISD::SBUFFER_LOAD_UBYTE, dl: DL,
8431 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops, MemVT: VT, MMO);
8432 LoadVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: BufferLoad);
8433 } else {
8434 SDValue Ops[] = {
8435 DAG.getEntryNode(), // Chain
8436 Rsrc, // rsrc
8437 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
8438 {}, // voffset
8439 {}, // soffset
8440 {}, // offset
8441 CachePolicy, // cachepolicy
8442 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
8443 };
8444 setBufferOffsets(CombinedOffset: Offset, DAG, Offsets: &Ops[3], Alignment: Align(4));
8445 LoadVal = handleByteShortBufferLoads(DAG, LoadVT: VT, DL, Ops, MMO);
8446 }
8447 Results.push_back(Elt: LoadVal);
8448 return;
8449 }
8450 case Intrinsic::amdgcn_dead: {
8451 for (unsigned I = 0, E = N->getNumValues(); I < E; ++I)
8452 Results.push_back(Elt: DAG.getPOISON(VT: N->getValueType(ResNo: I)));
8453 return;
8454 }
8455 }
8456 break;
8457 }
8458 case ISD::INTRINSIC_W_CHAIN: {
8459 if (SDValue Res = LowerINTRINSIC_W_CHAIN(Op: SDValue(N, 0), DAG)) {
8460 if (Res.getOpcode() == ISD::MERGE_VALUES) {
8461 // FIXME: Hacky
8462 for (unsigned I = 0; I < Res.getNumOperands(); I++) {
8463 Results.push_back(Elt: Res.getOperand(i: I));
8464 }
8465 } else {
8466 Results.push_back(Elt: Res);
8467 Results.push_back(Elt: Res.getValue(R: 1));
8468 }
8469 return;
8470 }
8471
8472 break;
8473 }
8474 case ISD::SELECT: {
8475 SDLoc SL(N);
8476 EVT VT = N->getValueType(ResNo: 0);
8477 EVT NewVT = getEquivalentMemType(Context&: *DAG.getContext(), VT);
8478 SDValue LHS = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: N->getOperand(Num: 1));
8479 SDValue RHS = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: N->getOperand(Num: 2));
8480
8481 EVT SelectVT = NewVT;
8482 if (NewVT.bitsLT(VT: MVT::i32)) {
8483 LHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: LHS);
8484 RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: RHS);
8485 SelectVT = MVT::i32;
8486 }
8487
8488 SDValue NewSelect =
8489 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: SelectVT, N1: N->getOperand(Num: 0), N2: LHS, N3: RHS);
8490
8491 if (NewVT != SelectVT)
8492 NewSelect = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: NewVT, Operand: NewSelect);
8493 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: NewSelect));
8494 return;
8495 }
8496 case ISD::FNEG: {
8497 if (N->getValueType(ResNo: 0) != MVT::v2f16)
8498 break;
8499
8500 SDLoc SL(N);
8501 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 0));
8502
8503 SDValue Op = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: BC,
8504 N2: DAG.getConstant(Val: 0x80008000, DL: SL, VT: MVT::i32));
8505 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Op));
8506 return;
8507 }
8508 case ISD::FABS: {
8509 if (N->getValueType(ResNo: 0) != MVT::v2f16)
8510 break;
8511
8512 SDLoc SL(N);
8513 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 0));
8514
8515 SDValue Op = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: BC,
8516 N2: DAG.getConstant(Val: 0x7fff7fff, DL: SL, VT: MVT::i32));
8517 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Op));
8518 return;
8519 }
8520 case ISD::FSQRT: {
8521 if (N->getValueType(ResNo: 0) != MVT::f16)
8522 break;
8523 Results.push_back(Elt: lowerFSQRTF16(Op: SDValue(N, 0), DAG));
8524 break;
8525 }
8526 default:
8527 AMDGPUTargetLowering::ReplaceNodeResults(N, Results, DAG);
8528 break;
8529 }
8530}
8531
8532/// Helper function for LowerBRCOND
8533static SDNode *findUser(SDValue Value, unsigned Opcode) {
8534
8535 for (SDUse &U : Value->uses()) {
8536 if (U.get() != Value)
8537 continue;
8538
8539 if (U.getUser()->getOpcode() == Opcode)
8540 return U.getUser();
8541 }
8542 return nullptr;
8543}
8544
8545unsigned SITargetLowering::isCFIntrinsic(const SDNode *Intr) const {
8546 if (Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
8547 switch (Intr->getConstantOperandVal(Num: 1)) {
8548 case Intrinsic::amdgcn_if:
8549 return AMDGPUISD::IF;
8550 case Intrinsic::amdgcn_else:
8551 return AMDGPUISD::ELSE;
8552 case Intrinsic::amdgcn_loop:
8553 return AMDGPUISD::LOOP;
8554 case Intrinsic::amdgcn_end_cf:
8555 llvm_unreachable("should not occur");
8556 default:
8557 return 0;
8558 }
8559 }
8560
8561 // break, if_break, else_break are all only used as inputs to loop, not
8562 // directly as branch conditions.
8563 return 0;
8564}
8565
8566bool SITargetLowering::shouldEmitFixup(const GlobalValue *GV) const {
8567 const Triple &TT = getTargetMachine().getTargetTriple();
8568 return (GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
8569 GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
8570 AMDGPU::shouldEmitConstantsToTextSection(TT);
8571}
8572
8573bool SITargetLowering::shouldEmitGOTReloc(const GlobalValue *GV) const {
8574 if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS())
8575 return false;
8576
8577 // FIXME: Either avoid relying on address space here or change the default
8578 // address space for functions to avoid the explicit check.
8579 return (GV->getValueType()->isFunctionTy() ||
8580 !isNonGlobalAddrSpace(AS: GV->getAddressSpace())) &&
8581 !shouldEmitFixup(GV) && !getTargetMachine().shouldAssumeDSOLocal(GV);
8582}
8583
8584bool SITargetLowering::shouldEmitPCReloc(const GlobalValue *GV) const {
8585 return !shouldEmitFixup(GV) && !shouldEmitGOTReloc(GV);
8586}
8587
8588bool SITargetLowering::shouldUseLDSConstAddress(const GlobalValue *GV) const {
8589 if (!GV->hasExternalLinkage())
8590 return true;
8591
8592 // With object linking, external LDS declarations need relocations so the
8593 // linker can assign their offsets.
8594 if (AMDGPUTargetMachine::EnableObjectLinking) {
8595 if (const auto *GVar = dyn_cast<GlobalVariable>(Val: GV)) {
8596 if (GVar->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
8597 assert(GVar->isDeclaration() && "AS3 GVs should be declaration here "
8598 "when object linking is enabled");
8599 return false;
8600 }
8601 }
8602 }
8603
8604 const auto OS = getTargetMachine().getTargetTriple().getOS();
8605 return OS == Triple::AMDHSA || OS == Triple::AMDPAL;
8606}
8607
8608/// This transforms the control flow intrinsics to get the branch destination as
8609/// last parameter, also switches branch target with BR if the need arise
8610SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND, SelectionDAG &DAG) const {
8611 SDLoc DL(BRCOND);
8612
8613 SDNode *Intr = BRCOND.getOperand(i: 1).getNode();
8614 SDValue Target = BRCOND.getOperand(i: 2);
8615 SDNode *BR = nullptr;
8616 SDNode *SetCC = nullptr;
8617
8618 switch (Intr->getOpcode()) {
8619 case ISD::SETCC: {
8620 // As long as we negate the condition everything is fine
8621 SetCC = Intr;
8622 Intr = SetCC->getOperand(Num: 0).getNode();
8623 break;
8624 }
8625 case ISD::XOR: {
8626 // Similar to SETCC, if we have (xor c, -1), we will be fine.
8627 SDValue LHS = Intr->getOperand(Num: 0);
8628 SDValue RHS = Intr->getOperand(Num: 1);
8629 if (auto *C = dyn_cast<ConstantSDNode>(Val&: RHS); C && C->getZExtValue()) {
8630 Intr = LHS.getNode();
8631 break;
8632 }
8633 [[fallthrough]];
8634 }
8635 default: {
8636 // Get the target from BR if we don't negate the condition
8637 BR = findUser(Value: BRCOND, Opcode: ISD::BR);
8638 assert(BR && "brcond missing unconditional branch user");
8639 Target = BR->getOperand(Num: 1);
8640 }
8641 }
8642
8643 unsigned CFNode = isCFIntrinsic(Intr);
8644 if (CFNode == 0) {
8645 // This is a uniform branch so we don't need to legalize.
8646 return BRCOND;
8647 }
8648
8649 bool HaveChain = Intr->getOpcode() == ISD::INTRINSIC_VOID ||
8650 Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN;
8651
8652 assert(!SetCC ||
8653 (SetCC->getConstantOperandVal(1) == 1 &&
8654 cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() ==
8655 ISD::SETNE));
8656
8657 // operands of the new intrinsic call
8658 SmallVector<SDValue, 4> Ops;
8659 if (HaveChain)
8660 Ops.push_back(Elt: BRCOND.getOperand(i: 0));
8661
8662 Ops.append(in_start: Intr->op_begin() + (HaveChain ? 2 : 1), in_end: Intr->op_end());
8663 Ops.push_back(Elt: Target);
8664
8665 ArrayRef<EVT> Res(Intr->value_begin() + 1, Intr->value_end());
8666
8667 // build the new intrinsic call
8668 SDNode *Result = DAG.getNode(Opcode: CFNode, DL, VTList: DAG.getVTList(VTs: Res), Ops).getNode();
8669
8670 if (!HaveChain) {
8671 SDValue Ops[] = {SDValue(Result, 0), BRCOND.getOperand(i: 0)};
8672
8673 Result = DAG.getMergeValues(Ops, dl: DL).getNode();
8674 }
8675
8676 if (BR) {
8677 // Give the branch instruction our target
8678 SDValue Ops[] = {BR->getOperand(Num: 0), BRCOND.getOperand(i: 2)};
8679 SDValue NewBR = DAG.getNode(Opcode: ISD::BR, DL, VTList: BR->getVTList(), Ops);
8680 DAG.ReplaceAllUsesWith(From: BR, To: NewBR.getNode());
8681 }
8682
8683 SDValue Chain = SDValue(Result, Result->getNumValues() - 1);
8684
8685 // Copy the intrinsic results to registers
8686 for (unsigned i = 1, e = Intr->getNumValues() - 1; i != e; ++i) {
8687 SDNode *CopyToReg = findUser(Value: SDValue(Intr, i), Opcode: ISD::CopyToReg);
8688 if (!CopyToReg)
8689 continue;
8690
8691 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: CopyToReg->getOperand(Num: 1),
8692 N: SDValue(Result, i - 1), Glue: SDValue());
8693
8694 DAG.ReplaceAllUsesWith(From: SDValue(CopyToReg, 0), To: CopyToReg->getOperand(Num: 0));
8695 }
8696
8697 // Remove the old intrinsic from the chain
8698 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Intr, Intr->getNumValues() - 1),
8699 To: Intr->getOperand(Num: 0));
8700
8701 return Chain;
8702}
8703
8704SDValue SITargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
8705 MVT VT = Op.getSimpleValueType();
8706 SDLoc DL(Op);
8707 // Checking the depth
8708 if (Op.getConstantOperandVal(i: 0) != 0)
8709 return DAG.getConstant(Val: 0, DL, VT);
8710
8711 MachineFunction &MF = DAG.getMachineFunction();
8712 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8713 // Check for kernel and shader functions
8714 if (Info->isEntryFunction())
8715 return DAG.getConstant(Val: 0, DL, VT);
8716
8717 MachineFrameInfo &MFI = MF.getFrameInfo();
8718 // There is a call to @llvm.returnaddress in this function
8719 MFI.setReturnAddressIsTaken(true);
8720
8721 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
8722 // Get the return address reg and mark it as an implicit live-in
8723 Register Reg = MF.addLiveIn(PReg: TRI->getReturnAddressReg(MF),
8724 RC: getRegClassFor(VT, isDivergent: Op.getNode()->isDivergent()));
8725
8726 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg, VT);
8727}
8728
8729SDValue SITargetLowering::LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const {
8730 MachineFunction &MF = DAG.getMachineFunction();
8731 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
8732
8733 // For functions that set up their own stack, select the GET_STACK_BASE
8734 // pseudo.
8735 if (MFI->isBottomOfStack())
8736 return Op;
8737
8738 // For everything else, create a dummy stack object.
8739 int FI = MF.getFrameInfo().CreateFixedObject(Size: 1, SPOffset: 0, /*IsImmutable=*/false);
8740 return DAG.getFrameIndex(FI, VT: Op.getValueType());
8741}
8742
8743SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG, SDValue Op,
8744 const SDLoc &DL, EVT VT) const {
8745 return Op.getValueType().bitsLE(VT)
8746 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
8747 : DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
8748 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
8749}
8750
8751SDValue SITargetLowering::splitFP_ROUNDVectorOp(SDValue Op,
8752 SelectionDAG &DAG) const {
8753 EVT DstVT = Op.getValueType();
8754 unsigned NumElts = DstVT.getVectorNumElements();
8755 assert(NumElts > 2 && isPowerOf2_32(NumElts));
8756
8757 auto [Lo, Hi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
8758
8759 SDLoc DL(Op);
8760 unsigned Opc = Op.getOpcode();
8761 SDValue Flags = Op.getOperand(i: 1);
8762 EVT HalfDstVT =
8763 EVT::getVectorVT(Context&: *DAG.getContext(), VT: DstVT.getScalarType(), NumElements: NumElts / 2);
8764 SDValue OpLo = DAG.getNode(Opcode: Opc, DL, VT: HalfDstVT, N1: Lo, N2: Flags);
8765 SDValue OpHi = DAG.getNode(Opcode: Opc, DL, VT: HalfDstVT, N1: Hi, N2: Flags);
8766
8767 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: DstVT, N1: OpLo, N2: OpHi);
8768}
8769
8770SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
8771 bool IsStrict = Op->isStrictFPOpcode();
8772 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8773 EVT SrcVT = Src.getValueType();
8774 EVT DstVT = Op.getValueType();
8775
8776 if (DstVT.isVectorOf(EltVT: MVT::f16)) {
8777 assert(Subtarget->hasCvtPkF16F32Inst() && "support v_cvt_pk_f16_f32");
8778 if (SrcVT.getScalarType() != MVT::f32)
8779 return SDValue();
8780 return SrcVT == MVT::v2f32 ? Op : splitFP_ROUNDVectorOp(Op, DAG);
8781 }
8782
8783 if (SrcVT.getScalarType() != MVT::f64)
8784 return Op;
8785
8786 SDLoc DL(Op);
8787 if (DstVT == MVT::f16) {
8788 // TODO: Handle strictfp
8789 if (Op.getOpcode() != ISD::FP_ROUND)
8790 return Op;
8791
8792 if (!Subtarget->has16BitInsts()) {
8793 SDValue FpToFp16 = DAG.getNode(Opcode: ISD::FP_TO_FP16, DL, VT: MVT::i32, Operand: Src);
8794 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToFp16);
8795 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f16, Operand: Trunc);
8796 }
8797 if (Op->getFlags().hasApproximateFuncs()) {
8798 SDValue Flags = Op.getOperand(i: 1);
8799 SDValue Src32 = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: MVT::f32, N1: Src, N2: Flags);
8800 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: MVT::f16, N1: Src32, N2: Flags);
8801 }
8802 SDValue FpToFp16 = LowerF64ToF16Safe(Src, DL, DAG);
8803 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToFp16);
8804 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f16, Operand: Trunc);
8805 }
8806
8807 assert(DstVT.getScalarType() == MVT::bf16 &&
8808 "custom lower FP_ROUND for f16 or bf16");
8809 assert(Subtarget->hasBF16ConversionInsts() && "f32 -> bf16 is legal");
8810
8811 // Round-inexact-to-odd f64 to f32, then do the final rounding using the
8812 // hardware f32 -> bf16 instruction.
8813 EVT F32VT = SrcVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
8814 SDValue Rod = expandRoundInexactToOdd(ResultVT: F32VT, Op: Src, DL, DAG);
8815 if (IsStrict) {
8816 return DAG.getNode(
8817 Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {DstVT, MVT::Other},
8818 Ops: {Op.getOperand(i: 0), Rod, DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32)});
8819 }
8820 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: DstVT, N1: Rod,
8821 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
8822}
8823
8824SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
8825 SelectionDAG &DAG) const {
8826 EVT VT = Op.getValueType();
8827 const MachineFunction &MF = DAG.getMachineFunction();
8828 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8829 bool IsIEEEMode = Info->getMode().IEEE;
8830
8831 // FIXME: Assert during selection that this is only selected for
8832 // ieee_mode. Currently a combine can produce the ieee version for non-ieee
8833 // mode functions, but this happens to be OK since it's only done in cases
8834 // where there is known no sNaN.
8835 if (IsIEEEMode)
8836 return expandFMINNUM_FMAXNUM(N: Op.getNode(), DAG);
8837
8838 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
8839 VT == MVT::v16bf16)
8840 return splitBinaryVectorOp(Op, DAG);
8841 return Op;
8842}
8843
8844SDValue
8845SITargetLowering::lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op,
8846 SelectionDAG &DAG) const {
8847 EVT VT = Op.getValueType();
8848 const MachineFunction &MF = DAG.getMachineFunction();
8849 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8850 bool IsIEEEMode = Info->getMode().IEEE;
8851
8852 if (IsIEEEMode)
8853 return expandFMINIMUMNUM_FMAXIMUMNUM(N: Op.getNode(), DAG);
8854
8855 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
8856 VT == MVT::v32f16)
8857 return splitBinaryVectorOp(Op, DAG);
8858 return Op;
8859}
8860
8861SDValue SITargetLowering::lowerFLDEXP(SDValue Op, SelectionDAG &DAG) const {
8862 bool IsStrict = Op.getOpcode() == ISD::STRICT_FLDEXP;
8863 EVT VT = Op.getValueType();
8864 assert(VT == MVT::f16);
8865
8866 SDValue Exp = Op.getOperand(i: IsStrict ? 2 : 1);
8867 EVT ExpVT = Exp.getValueType();
8868 if (ExpVT == MVT::i16)
8869 return Op;
8870
8871 SDLoc DL(Op);
8872
8873 // Correct the exponent type for f16 to i16.
8874 // Clamp the range of the exponent to the instruction's range.
8875
8876 // TODO: This should be a generic narrowing legalization, and can easily be
8877 // for GlobalISel.
8878
8879 SDValue MinExp = DAG.getSignedConstant(Val: minIntN(N: 16), DL, VT: ExpVT);
8880 SDValue ClampMin = DAG.getNode(Opcode: ISD::SMAX, DL, VT: ExpVT, N1: Exp, N2: MinExp);
8881
8882 SDValue MaxExp = DAG.getSignedConstant(Val: maxIntN(N: 16), DL, VT: ExpVT);
8883 SDValue Clamp = DAG.getNode(Opcode: ISD::SMIN, DL, VT: ExpVT, N1: ClampMin, N2: MaxExp);
8884
8885 SDValue TruncExp = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Clamp);
8886
8887 if (IsStrict) {
8888 return DAG.getNode(Opcode: ISD::STRICT_FLDEXP, DL, ResultTys: {VT, MVT::Other},
8889 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), TruncExp});
8890 }
8891
8892 return DAG.getNode(Opcode: ISD::FLDEXP, DL, VT, N1: Op.getOperand(i: 0), N2: TruncExp);
8893}
8894
8895static unsigned getExtOpcodeForPromotedOp(SDValue Op) {
8896 switch (Op->getOpcode()) {
8897 case ISD::ABS:
8898 case ISD::SRA:
8899 case ISD::SMIN:
8900 case ISD::SMAX:
8901 return ISD::SIGN_EXTEND;
8902 case ISD::SRL:
8903 case ISD::UMIN:
8904 case ISD::UMAX:
8905 case ISD::USUBSAT:
8906 return ISD::ZERO_EXTEND;
8907 case ISD::ADD:
8908 case ISD::SUB:
8909 case ISD::AND:
8910 case ISD::OR:
8911 case ISD::XOR:
8912 case ISD::SHL:
8913 case ISD::SELECT:
8914 case ISD::MUL:
8915 // operation result won't be influenced by garbage high bits.
8916 // TODO: are all of those cases correct, and are there more?
8917 return ISD::ANY_EXTEND;
8918 case ISD::SETCC: {
8919 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
8920 return ISD::isSignedIntSetCC(Code: CC) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
8921 }
8922 default:
8923 llvm_unreachable("unexpected opcode!");
8924 }
8925}
8926
8927SDValue
8928SITargetLowering::promoteUniformUnaryOpToI32(SDValue Op,
8929 DAGCombinerInfo &DCI) const {
8930 EVT OpTy = Op.getValueType();
8931 SelectionDAG &DAG = DCI.DAG;
8932 EVT ExtTy = OpTy.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
8933
8934 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: ExtTy, DestVT: OpTy))
8935 return SDValue();
8936
8937 SDLoc DL(Op);
8938 SDValue Input = Op.getOperand(i: 0);
8939 const unsigned ExtOp = getExtOpcodeForPromotedOp(Op);
8940 Input = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: Input);
8941
8942 SDValue NewVal = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: ExtTy, Operand: Input);
8943
8944 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OpTy, Operand: NewVal);
8945}
8946
8947SDValue SITargetLowering::promoteUniformOpToI32(SDValue Op,
8948 DAGCombinerInfo &DCI) const {
8949 const unsigned Opc = Op.getOpcode();
8950 assert(Opc == ISD::ADD || Opc == ISD::SUB || Opc == ISD::SHL ||
8951 Opc == ISD::SRL || Opc == ISD::SRA || Opc == ISD::AND ||
8952 Opc == ISD::OR || Opc == ISD::XOR || Opc == ISD::MUL ||
8953 Opc == ISD::SETCC || Opc == ISD::SELECT || Opc == ISD::SMIN ||
8954 Opc == ISD::SMAX || Opc == ISD::UMIN || Opc == ISD::UMAX ||
8955 Opc == ISD::USUBSAT);
8956
8957 EVT OpTy = (Opc != ISD::SETCC) ? Op.getValueType()
8958 : Op->getOperand(Num: 0).getValueType();
8959 auto &DAG = DCI.DAG;
8960 auto ExtTy = OpTy.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
8961
8962 if (DCI.isBeforeLegalizeOps() ||
8963 isNarrowingProfitable(N: Op.getNode(), SrcVT: ExtTy, DestVT: OpTy))
8964 return SDValue();
8965
8966 SDLoc DL(Op);
8967 SDValue LHS;
8968 SDValue RHS;
8969 if (Opc == ISD::SELECT) {
8970 LHS = Op->getOperand(Num: 1);
8971 RHS = Op->getOperand(Num: 2);
8972 } else {
8973 LHS = Op->getOperand(Num: 0);
8974 RHS = Op->getOperand(Num: 1);
8975 }
8976
8977 const unsigned ExtOp = getExtOpcodeForPromotedOp(Op);
8978 LHS = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: {LHS});
8979
8980 // Special case: for shifts, the RHS always needs a zext.
8981 if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA)
8982 RHS = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtTy, Operand: {RHS});
8983 else
8984 RHS = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: {RHS});
8985
8986 // setcc always return i1/i1 vec so no need to truncate after.
8987 if (Opc == ISD::SETCC) {
8988 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
8989 return DAG.getSetCC(DL, VT: Op.getValueType(), LHS, RHS, Cond: CC);
8990 }
8991
8992 // For other ops, we extend the operation's return type as well so we need to
8993 // truncate back to the original type.
8994 SDValue NewVal;
8995 if (Opc == ISD::SELECT)
8996 NewVal = DAG.getNode(Opcode: ISD::SELECT, DL, VT: ExtTy, Ops: {Op->getOperand(Num: 0), LHS, RHS});
8997 else
8998 NewVal = DAG.getNode(Opcode: Opc, DL, VT: ExtTy, Ops: {LHS, RHS});
8999
9000 return DAG.getZExtOrTrunc(Op: NewVal, DL, VT: OpTy);
9001}
9002
9003SDValue SITargetLowering::lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
9004 SDValue Mag = Op.getOperand(i: 0);
9005 EVT MagVT = Mag.getValueType();
9006
9007 if (MagVT.getVectorNumElements() > 2)
9008 return splitBinaryVectorOp(Op, DAG);
9009
9010 SDValue Sign = Op.getOperand(i: 1);
9011 EVT SignVT = Sign.getValueType();
9012
9013 if (MagVT == SignVT)
9014 return Op;
9015
9016 // fcopysign v2f16:mag, v2f32:sign ->
9017 // fcopysign v2f16:mag,
9018 // bitcast (trunc (srl (bitcast sign to v2i32), 16) to v2i16)
9019
9020 SDLoc SL(Op);
9021 SDValue SignAsInt32 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Sign);
9022 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: 16, VT: MVT::v2i32, DL: SL);
9023 SDValue SignShifted =
9024 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::v2i32, N1: SignAsInt32, N2: ShiftAmt);
9025 SDValue SignAsInt16 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::v2i16, Operand: SignShifted);
9026
9027 SDValue SignAsHalf16 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MagVT, Operand: SignAsInt16);
9028
9029 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT: MagVT, N1: Mag, N2: SignAsHalf16);
9030}
9031
9032// Custom lowering for vector multiplications and s_mul_u64.
9033SDValue SITargetLowering::lowerMUL(SDValue Op, SelectionDAG &DAG) const {
9034 EVT VT = Op.getValueType();
9035
9036 // Split vector operands.
9037 if (VT.isVector())
9038 return splitBinaryVectorOp(Op, DAG);
9039
9040 assert(VT == MVT::i64 && "The following code is a special for s_mul_u64");
9041
9042 // There are four ways to lower s_mul_u64:
9043 //
9044 // 1. If all the operands are uniform, then we lower it as it is.
9045 //
9046 // 2. If the operands are divergent, then we have to split s_mul_u64 in 32-bit
9047 // multiplications because there is not a vector equivalent of s_mul_u64.
9048 //
9049 // 3. If the cost model decides that it is more efficient to use vector
9050 // registers, then we have to split s_mul_u64 in 32-bit multiplications.
9051 // This happens in splitScalarSMULU64() in SIInstrInfo.cpp .
9052 //
9053 // 4. If the cost model decides to use vector registers and both of the
9054 // operands are zero-extended/sign-extended from 32-bits, then we split the
9055 // s_mul_u64 in two 32-bit multiplications. The problem is that it is not
9056 // possible to check if the operands are zero-extended or sign-extended in
9057 // SIInstrInfo.cpp. For this reason, here, we replace s_mul_u64 with
9058 // s_mul_u64_u32_pseudo if both operands are zero-extended and we replace
9059 // s_mul_u64 with s_mul_i64_i32_pseudo if both operands are sign-extended.
9060 // If the cost model decides that we have to use vector registers, then
9061 // splitScalarSMulPseudo() (in SIInstrInfo.cpp) split s_mul_u64_u32/
9062 // s_mul_i64_i32_pseudo in two vector multiplications. If the cost model
9063 // decides that we should use scalar registers, then s_mul_u64_u32_pseudo/
9064 // s_mul_i64_i32_pseudo is lowered as s_mul_u64 in expandPostRAPseudo() in
9065 // SIInstrInfo.cpp .
9066
9067 if (Op->isDivergent())
9068 return SDValue();
9069
9070 SDValue Op0 = Op.getOperand(i: 0);
9071 SDValue Op1 = Op.getOperand(i: 1);
9072 // If all the operands are zero-enteted to 32-bits, then we replace s_mul_u64
9073 // with s_mul_u64_u32_pseudo. If all the operands are sign-extended to
9074 // 32-bits, then we replace s_mul_u64 with s_mul_i64_i32_pseudo.
9075 KnownBits Op0KnownBits = DAG.computeKnownBits(Op: Op0);
9076 unsigned Op0LeadingZeros = Op0KnownBits.countMinLeadingZeros();
9077 KnownBits Op1KnownBits = DAG.computeKnownBits(Op: Op1);
9078 unsigned Op1LeadingZeros = Op1KnownBits.countMinLeadingZeros();
9079 SDLoc SL(Op);
9080 if (Op0LeadingZeros >= 32 && Op1LeadingZeros >= 32)
9081 return SDValue(
9082 DAG.getMachineNode(Opcode: AMDGPU::S_MUL_U64_U32_PSEUDO, dl: SL, VT, Op1: Op0, Op2: Op1), 0);
9083 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op: Op0);
9084 unsigned Op1SignBits = DAG.ComputeNumSignBits(Op: Op1);
9085 if (Op0SignBits >= 33 && Op1SignBits >= 33)
9086 return SDValue(
9087 DAG.getMachineNode(Opcode: AMDGPU::S_MUL_I64_I32_PSEUDO, dl: SL, VT, Op1: Op0, Op2: Op1), 0);
9088 // If all the operands are uniform, then we lower s_mul_u64 as it is.
9089 return Op;
9090}
9091
9092SDValue SITargetLowering::lowerXMULO(SDValue Op, SelectionDAG &DAG) const {
9093 EVT VT = Op.getValueType();
9094 SDLoc SL(Op);
9095 SDValue LHS = Op.getOperand(i: 0);
9096 SDValue RHS = Op.getOperand(i: 1);
9097 bool isSigned = Op.getOpcode() == ISD::SMULO;
9098
9099 if (ConstantSDNode *RHSC = isConstOrConstSplat(N: RHS)) {
9100 const APInt &C = RHSC->getAPIntValue();
9101 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
9102 if (C.isPowerOf2()) {
9103 // smulo(x, signed_min) is same as umulo(x, signed_min).
9104 bool UseArithShift = isSigned && !C.isMinSignedValue();
9105 SDValue ShiftAmt = DAG.getConstant(Val: C.logBase2(), DL: SL, VT: MVT::i32);
9106 SDValue Result = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT, N1: LHS, N2: ShiftAmt);
9107 SDValue Overflow =
9108 DAG.getSetCC(DL: SL, VT: MVT::i1,
9109 LHS: DAG.getNode(Opcode: UseArithShift ? ISD::SRA : ISD::SRL, DL: SL, VT,
9110 N1: Result, N2: ShiftAmt),
9111 RHS: LHS, Cond: ISD::SETNE);
9112 return DAG.getMergeValues(Ops: {Result, Overflow}, dl: SL);
9113 }
9114 }
9115
9116 SDValue Result = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT, N1: LHS, N2: RHS);
9117 SDValue Top =
9118 DAG.getNode(Opcode: isSigned ? ISD::MULHS : ISD::MULHU, DL: SL, VT, N1: LHS, N2: RHS);
9119
9120 SDValue Sign = isSigned
9121 ? DAG.getNode(Opcode: ISD::SRA, DL: SL, VT, N1: Result,
9122 N2: DAG.getConstant(Val: VT.getScalarSizeInBits() - 1,
9123 DL: SL, VT: MVT::i32))
9124 : DAG.getConstant(Val: 0, DL: SL, VT);
9125 SDValue Overflow = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Top, RHS: Sign, Cond: ISD::SETNE);
9126
9127 return DAG.getMergeValues(Ops: {Result, Overflow}, dl: SL);
9128}
9129
9130SDValue SITargetLowering::lowerXMUL_LOHI(SDValue Op, SelectionDAG &DAG) const {
9131 if (Op->isDivergent()) {
9132 // Select to V_MAD_[IU]64_[IU]32.
9133 return Op;
9134 }
9135 if (Subtarget->hasSMulHi()) {
9136 // Expand to S_MUL_I32 + S_MUL_HI_[IU]32.
9137 return SDValue();
9138 }
9139 // The multiply is uniform but we would have to use V_MUL_HI_[IU]32 to
9140 // calculate the high part, so we might as well do the whole thing with
9141 // V_MAD_[IU]64_[IU]32.
9142 return Op;
9143}
9144
9145SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const {
9146 if (!Subtarget->hasTrapHandler() ||
9147 Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
9148 return lowerTrapEndpgm(Op, DAG);
9149
9150 return Subtarget->supportsGetDoorbellID() ? lowerTrapHsa(Op, DAG)
9151 : lowerTrapHsaQueuePtr(Op, DAG);
9152}
9153
9154SDValue SITargetLowering::lowerTrapEndpgm(SDValue Op, SelectionDAG &DAG) const {
9155 SDLoc SL(Op);
9156 SDValue Chain = Op.getOperand(i: 0);
9157 return DAG.getNode(Opcode: AMDGPUISD::ENDPGM_TRAP, DL: SL, VT: MVT::Other, Operand: Chain);
9158}
9159
9160SDValue
9161SITargetLowering::loadImplicitKernelArgument(SelectionDAG &DAG, MVT VT,
9162 const SDLoc &DL, Align Alignment,
9163 ImplicitParameter Param) const {
9164 MachineFunction &MF = DAG.getMachineFunction();
9165 uint64_t Offset = getImplicitParameterOffset(MF, Param);
9166 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL: DL, Chain: DAG.getEntryNode(), Offset);
9167 MachinePointerInfo PtrInfo =
9168 getKernargSegmentPtrInfo(MF&: DAG.getMachineFunction());
9169 return DAG.getLoad(
9170 VT, dl: DL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: PtrInfo.getWithOffset(O: Offset), Alignment,
9171 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
9172}
9173
9174SDValue SITargetLowering::lowerTrapHsaQueuePtr(SDValue Op,
9175 SelectionDAG &DAG) const {
9176 SDLoc SL(Op);
9177 SDValue Chain = Op.getOperand(i: 0);
9178
9179 SDValue QueuePtr;
9180 // For code object version 5, QueuePtr is passed through implicit kernarg.
9181 const Module *M = DAG.getMachineFunction().getFunction().getParent();
9182 if (AMDGPU::getAMDHSACodeObjectVersion(M: *M) >= AMDGPU::AMDHSA_COV5) {
9183 QueuePtr =
9184 loadImplicitKernelArgument(DAG, VT: MVT::i64, DL: SL, Alignment: Align(8), Param: QUEUE_PTR);
9185 } else {
9186 MachineFunction &MF = DAG.getMachineFunction();
9187 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
9188 Register UserSGPR = Info->getQueuePtrUserSGPR();
9189
9190 if (UserSGPR == AMDGPU::NoRegister) {
9191 // We probably are in a function incorrectly marked with
9192 // amdgpu-no-queue-ptr. This is undefined. We don't want to delete the
9193 // trap, so just use a null pointer.
9194 QueuePtr = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i64);
9195 } else {
9196 QueuePtr = CreateLiveInRegister(DAG, RC: &AMDGPU::SReg_64RegClass, Reg: UserSGPR,
9197 VT: MVT::i64);
9198 }
9199 }
9200
9201 SDValue SGPR01 = DAG.getRegister(Reg: AMDGPU::SGPR0_SGPR1, VT: MVT::i64);
9202 SDValue ToReg = DAG.getCopyToReg(Chain, dl: SL, Reg: SGPR01, N: QueuePtr, Glue: SDValue());
9203
9204 uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap);
9205 SDValue Ops[] = {ToReg, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16), SGPR01,
9206 ToReg.getValue(R: 1)};
9207 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
9208}
9209
9210SDValue SITargetLowering::lowerTrapHsa(SDValue Op, SelectionDAG &DAG) const {
9211 SDLoc SL(Op);
9212 SDValue Chain = Op.getOperand(i: 0);
9213
9214 // We need to simulate the 's_trap 2' instruction on targets that run in
9215 // PRIV=1 (where it is treated as a nop).
9216 if (Subtarget->hasPrivEnabledTrap2NopBug())
9217 return DAG.getNode(Opcode: AMDGPUISD::SIMULATED_TRAP, DL: SL, VT: MVT::Other, Operand: Chain);
9218
9219 uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap);
9220 SDValue Ops[] = {Chain, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16)};
9221 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
9222}
9223
9224SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const {
9225 SDLoc SL(Op);
9226 SDValue Chain = Op.getOperand(i: 0);
9227 MachineFunction &MF = DAG.getMachineFunction();
9228
9229 if (!Subtarget->hasTrapHandler() ||
9230 Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) {
9231 LLVMContext &Ctx = MF.getFunction().getContext();
9232 Ctx.diagnose(DI: DiagnosticInfoUnsupported(MF.getFunction(),
9233 "debugtrap handler not supported",
9234 Op.getDebugLoc(), DS_Warning));
9235 return Chain;
9236 }
9237
9238 uint64_t TrapID =
9239 static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap);
9240 SDValue Ops[] = {Chain, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16)};
9241 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
9242}
9243
9244/// When a divergent value (in VGPR) is passed to an inline asm with an SGPR
9245/// constraint ('s'), we need to insert v_readfirstlane to move the value from
9246/// VGPR to SGPR. This is done by modifying the CopyToReg nodes in the glue
9247/// chain that feed into the INLINEASM node.
9248SDValue SITargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
9249 unsigned NumOps = Op.getNumOperands();
9250
9251 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
9252 SmallSet<Register, 8> SGPRInputRegs;
9253
9254 unsigned NumVals = 0;
9255 for (unsigned I = InlineAsm::Op_FirstOperand; I < NumOps - 1;
9256 I += 1 + NumVals) {
9257 const InlineAsm::Flag Flags(Op.getConstantOperandVal(i: I));
9258 NumVals = Flags.getNumOperandRegisters();
9259
9260 unsigned RCID;
9261 bool IsSGPRInput = Flags.getKind() == InlineAsm::Kind::RegUse &&
9262 NumVals > 0 && Flags.hasRegClassConstraint(RC&: RCID) &&
9263 TRI->isSGPRClass(RC: TRI->getRegClass(i: RCID));
9264
9265 for (unsigned J = 0; J < NumVals; ++J) {
9266 SDValue Val = Op.getOperand(i: I + 1 + J);
9267 if (const RegisterSDNode *RegNode =
9268 dyn_cast<RegisterSDNode>(Val: Val.getNode())) {
9269 Register Reg = RegNode->getReg();
9270 if (IsSGPRInput || (Reg.isPhysical() && TRI->isSGPRPhysReg(Reg)))
9271 SGPRInputRegs.insert(V: Reg);
9272 }
9273 }
9274 }
9275
9276 if (SGPRInputRegs.empty())
9277 return Op;
9278
9279 // Walk the glue chain and insert readfirstlane for divergent SGPR inputs.
9280 SDLoc DL(Op);
9281 SDNode *N = Op.getOperand(i: NumOps - 1).getNode();
9282
9283 while (N && N->getOpcode() == ISD::CopyToReg) {
9284 Register Reg = cast<RegisterSDNode>(Val: N->getOperand(Num: 1))->getReg();
9285 SDValue SrcVal = N->getOperand(Num: 2);
9286
9287 // Insert readfirstlane if copying a divergent value to an SGPR input.
9288 if (SrcVal->isDivergent() && SGPRInputRegs.count(V: Reg)) {
9289 SDValue ReadFirstLaneID =
9290 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
9291 SDValue ReadFirstLane =
9292 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: SrcVal.getValueType(),
9293 N1: ReadFirstLaneID, N2: SrcVal);
9294
9295 SmallVector<SDValue, 4> Ops = {N->getOperand(Num: 0), N->getOperand(Num: 1),
9296 ReadFirstLane};
9297 if (N->getNumOperands() > 3)
9298 Ops.push_back(Elt: N->getOperand(Num: 3)); // Glue input
9299
9300 DAG.UpdateNodeOperands(N, Ops);
9301 }
9302
9303 // Follow glue chain to next CopyToReg.
9304 SDNode *Next = nullptr;
9305 for (unsigned I = 0, E = N->getNumOperands(); I != E; ++I) {
9306 if (N->getOperand(Num: I).getValueType() == MVT::Glue) {
9307 Next = N->getOperand(Num: I).getNode();
9308 break;
9309 }
9310 }
9311 N = Next;
9312 }
9313
9314 return Op;
9315}
9316
9317SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL,
9318 SelectionDAG &DAG) const {
9319 if (Subtarget->hasApertureRegs()) {
9320 const unsigned ApertureRegNo = (AS == AMDGPUAS::LOCAL_ADDRESS)
9321 ? AMDGPU::SRC_SHARED_BASE
9322 : AMDGPU::SRC_PRIVATE_BASE;
9323 assert((ApertureRegNo != AMDGPU::SRC_PRIVATE_BASE ||
9324 !Subtarget->hasGloballyAddressableScratch()) &&
9325 "Cannot use src_private_base with globally addressable scratch!");
9326 // Note: this feature (register) is broken. When used as a 32-bit operand,
9327 // it returns a wrong value (all zeroes?). The real value is in the upper 32
9328 // bits.
9329 //
9330 // To work around the issue, emit a 64 bit copy from this register
9331 // then extract the high bits. Note that this shouldn't even result in a
9332 // shift being emitted and simply become a pair of registers (e.g.):
9333 // s_mov_b64 s[6:7], src_shared_base
9334 // v_mov_b32_e32 v1, s7
9335 SDValue Copy =
9336 DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: ApertureRegNo, VT: MVT::v2i32);
9337 return DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: Copy, Idx: 1);
9338 }
9339
9340 // For code object version 5, private_base and shared_base are passed through
9341 // implicit kernargs.
9342 const Module *M = DAG.getMachineFunction().getFunction().getParent();
9343 if (AMDGPU::getAMDHSACodeObjectVersion(M: *M) >= AMDGPU::AMDHSA_COV5) {
9344 ImplicitParameter Param =
9345 (AS == AMDGPUAS::LOCAL_ADDRESS) ? SHARED_BASE : PRIVATE_BASE;
9346 return loadImplicitKernelArgument(DAG, VT: MVT::i32, DL, Alignment: Align(4), Param);
9347 }
9348
9349 MachineFunction &MF = DAG.getMachineFunction();
9350 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
9351 Register UserSGPR = Info->getQueuePtrUserSGPR();
9352 if (UserSGPR == AMDGPU::NoRegister) {
9353 // We probably are in a function incorrectly marked with
9354 // amdgpu-no-queue-ptr. This is undefined.
9355 return DAG.getPOISON(VT: MVT::i32);
9356 }
9357
9358 SDValue QueuePtr =
9359 CreateLiveInRegister(DAG, RC: &AMDGPU::SReg_64RegClass, Reg: UserSGPR, VT: MVT::i64);
9360
9361 // Offset into amd_queue_t for group_segment_aperture_base_hi /
9362 // private_segment_aperture_base_hi.
9363 uint32_t StructOffset = (AS == AMDGPUAS::LOCAL_ADDRESS) ? 0x40 : 0x44;
9364
9365 SDValue Ptr =
9366 DAG.getObjectPtrOffset(SL: DL, Ptr: QueuePtr, Offset: TypeSize::getFixed(ExactSize: StructOffset));
9367
9368 // TODO: Use custom target PseudoSourceValue.
9369 // TODO: We should use the value from the IR intrinsic call, but it might not
9370 // be available and how do we get it?
9371 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
9372 return DAG.getLoad(VT: MVT::i32, dl: DL, Chain: QueuePtr.getValue(R: 1), Ptr, PtrInfo,
9373 Alignment: commonAlignment(A: Align(64), Offset: StructOffset),
9374 MMOFlags: MachineMemOperand::MODereferenceable |
9375 MachineMemOperand::MOInvariant);
9376}
9377
9378/// Return true if the value is a known valid address, such that a null check is
9379/// not necessary.
9380static bool isKnownNonNull(SDValue Val, SelectionDAG &DAG,
9381 const AMDGPUTargetMachine &TM, unsigned AddrSpace) {
9382 if (isa<FrameIndexSDNode, GlobalAddressSDNode, BasicBlockSDNode>(Val))
9383 return true;
9384
9385 if (auto *ConstVal = dyn_cast<ConstantSDNode>(Val))
9386 return ConstVal->getSExtValue() != AMDGPU::getNullPointerValue(AS: AddrSpace);
9387
9388 // TODO: Search through arithmetic, handle arguments and loads
9389 // marked nonnull.
9390 return false;
9391}
9392
9393SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op,
9394 SelectionDAG &DAG) const {
9395 SDLoc SL(Op);
9396
9397 const AMDGPUTargetMachine &TM =
9398 static_cast<const AMDGPUTargetMachine &>(getTargetMachine());
9399
9400 unsigned DestAS, SrcAS;
9401 SDValue Src;
9402 bool IsNonNull = false;
9403 if (const auto *ASC = dyn_cast<AddrSpaceCastSDNode>(Val&: Op)) {
9404 SrcAS = ASC->getSrcAddressSpace();
9405 Src = ASC->getOperand(Num: 0);
9406 DestAS = ASC->getDestAddressSpace();
9407 } else {
9408 assert(Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
9409 Op.getConstantOperandVal(0) ==
9410 Intrinsic::amdgcn_addrspacecast_nonnull);
9411 Src = Op->getOperand(Num: 1);
9412 SrcAS = Op->getConstantOperandVal(Num: 2);
9413 DestAS = Op->getConstantOperandVal(Num: 3);
9414 IsNonNull = true;
9415 }
9416
9417 SDValue FlatNullPtr = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i64);
9418
9419 // flat -> local/private
9420 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
9421 if (DestAS == AMDGPUAS::LOCAL_ADDRESS ||
9422 DestAS == AMDGPUAS::PRIVATE_ADDRESS) {
9423 SDValue Ptr = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Src);
9424
9425 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
9426 Subtarget->hasGloballyAddressableScratch()) {
9427 // flat -> private with globally addressable scratch: subtract
9428 // src_flat_scratch_base_lo.
9429 SDValue FlatScratchBaseLo(
9430 DAG.getMachineNode(
9431 Opcode: AMDGPU::S_MOV_B32, dl: SL, VT: MVT::i32,
9432 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO, VT: MVT::i32)),
9433 0);
9434 Ptr = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: Ptr, N2: FlatScratchBaseLo);
9435 }
9436
9437 if (IsNonNull || isKnownNonNull(Val: Op, DAG, TM, AddrSpace: SrcAS))
9438 return Ptr;
9439
9440 unsigned NullVal = AMDGPU::getNullPointerValue(AS: DestAS);
9441 SDValue SegmentNullPtr = DAG.getConstant(Val: NullVal, DL: SL, VT: MVT::i32);
9442 SDValue NonNull = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Src, RHS: FlatNullPtr, Cond: ISD::SETNE);
9443
9444 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i32, N1: NonNull, N2: Ptr,
9445 N3: SegmentNullPtr);
9446 }
9447 }
9448
9449 // local/private -> flat
9450 if (DestAS == AMDGPUAS::FLAT_ADDRESS) {
9451 if (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
9452 SrcAS == AMDGPUAS::PRIVATE_ADDRESS) {
9453 SDValue CvtPtr;
9454 if (SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
9455 Subtarget->hasGloballyAddressableScratch()) {
9456 // For wave32: Addr = (TID[4:0] << 52) + FLAT_SCRATCH_BASE + privateAddr
9457 // For wave64: Addr = (TID[5:0] << 51) + FLAT_SCRATCH_BASE + privateAddr
9458 SDValue AllOnes = DAG.getSignedTargetConstant(Val: -1, DL: SL, VT: MVT::i32);
9459 SDValue ThreadID = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
9460 ThreadID = DAG.getNode(
9461 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
9462 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_mbcnt_lo, DL: SL, VT: MVT::i32),
9463 N2: AllOnes, N3: ThreadID);
9464 if (Subtarget->isWave64())
9465 ThreadID = DAG.getNode(
9466 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
9467 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_mbcnt_hi, DL: SL, VT: MVT::i32),
9468 N2: AllOnes, N3: ThreadID);
9469 SDValue ShAmt = DAG.getShiftAmountConstant(
9470 Val: 57 - 32 - Subtarget->getWavefrontSizeLog2(), VT: MVT::i32, DL: SL);
9471 SDValue SrcHi = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: ThreadID, N2: ShAmt);
9472 CvtPtr = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: SrcHi);
9473 CvtPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
9474 // Accessing src_flat_scratch_base_lo as a 64-bit operand gives the full
9475 // 64-bit hi:lo value.
9476 SDValue FlatScratchBase = {
9477 DAG.getMachineNode(
9478 Opcode: AMDGPU::S_MOV_B64, dl: SL, VT: MVT::i64,
9479 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE, VT: MVT::i64)),
9480 0};
9481 CvtPtr = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i64, N1: CvtPtr, N2: FlatScratchBase);
9482 } else {
9483 SDValue Aperture = getSegmentAperture(AS: SrcAS, DL: SL, DAG);
9484 CvtPtr = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: Aperture);
9485 CvtPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
9486 }
9487
9488 if (IsNonNull || isKnownNonNull(Val: Op, DAG, TM, AddrSpace: SrcAS))
9489 return CvtPtr;
9490
9491 unsigned NullVal = AMDGPU::getNullPointerValue(AS: SrcAS);
9492 SDValue SegmentNullPtr = DAG.getConstant(Val: NullVal, DL: SL, VT: MVT::i32);
9493
9494 SDValue NonNull =
9495 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Src, RHS: SegmentNullPtr, Cond: ISD::SETNE);
9496
9497 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: NonNull, N2: CvtPtr,
9498 N3: FlatNullPtr);
9499 }
9500 }
9501
9502 if (SrcAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
9503 Op.getValueType() == MVT::i64) {
9504 const SIMachineFunctionInfo *Info =
9505 DAG.getMachineFunction().getInfo<SIMachineFunctionInfo>();
9506 if (Info->get32BitAddressHighBits() == 0)
9507 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i64, Operand: Src);
9508
9509 SDValue Hi = DAG.getConstant(Val: Info->get32BitAddressHighBits(), DL: SL, VT: MVT::i32);
9510 SDValue Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: Hi);
9511 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
9512 }
9513
9514 if (DestAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
9515 Src.getValueType() == MVT::i64)
9516 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Src);
9517
9518 // global <-> flat are no-ops and never emitted.
9519
9520 // Invalid casts are poison.
9521 return DAG.getPOISON(VT: Op->getValueType(ResNo: 0));
9522}
9523
9524// This lowers an INSERT_SUBVECTOR by extracting the individual elements from
9525// the small vector and inserting them into the big vector. That is better than
9526// the default expansion of doing it via a stack slot. Even though the use of
9527// the stack slot would be optimized away afterwards, the stack slot itself
9528// remains.
9529SDValue SITargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
9530 SelectionDAG &DAG) const {
9531 SDValue Vec = Op.getOperand(i: 0);
9532 SDValue Ins = Op.getOperand(i: 1);
9533 SDValue Idx = Op.getOperand(i: 2);
9534 EVT VecVT = Vec.getValueType();
9535 EVT InsVT = Ins.getValueType();
9536 EVT EltVT = VecVT.getVectorElementType();
9537 unsigned InsNumElts = InsVT.getVectorNumElements();
9538 unsigned IdxVal = Idx->getAsZExtVal();
9539 SDLoc SL(Op);
9540
9541 if (EltVT.getScalarSizeInBits() == 16 && IdxVal % 2 == 0) {
9542 // Insert 32-bit registers at a time.
9543 assert(InsNumElts % 2 == 0 && "expect legal vector types");
9544
9545 unsigned VecNumElts = VecVT.getVectorNumElements();
9546 EVT NewVecVT =
9547 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: VecNumElts / 2);
9548 EVT NewInsVT = InsNumElts == 2 ? MVT::i32
9549 : EVT::getVectorVT(Context&: *DAG.getContext(),
9550 VT: MVT::i32, NumElements: InsNumElts / 2);
9551
9552 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVecVT, Operand: Vec);
9553 Ins = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewInsVT, Operand: Ins);
9554
9555 for (unsigned I = 0; I != InsNumElts / 2; ++I) {
9556 SDValue Elt;
9557 if (InsNumElts == 2) {
9558 Elt = Ins;
9559 } else {
9560 Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Ins,
9561 N2: DAG.getConstant(Val: I, DL: SL, VT: MVT::i32));
9562 }
9563 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NewVecVT, N1: Vec, N2: Elt,
9564 N3: DAG.getConstant(Val: IdxVal / 2 + I, DL: SL, VT: MVT::i32));
9565 }
9566
9567 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: Vec);
9568 }
9569
9570 for (unsigned I = 0; I != InsNumElts; ++I) {
9571 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Ins,
9572 N2: DAG.getConstant(Val: I, DL: SL, VT: MVT::i32));
9573 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: VecVT, N1: Vec, N2: Elt,
9574 N3: DAG.getConstant(Val: IdxVal + I, DL: SL, VT: MVT::i32));
9575 }
9576 return Vec;
9577}
9578
9579SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
9580 SelectionDAG &DAG) const {
9581 SDValue Vec = Op.getOperand(i: 0);
9582 SDValue InsVal = Op.getOperand(i: 1);
9583 SDValue Idx = Op.getOperand(i: 2);
9584 EVT VecVT = Vec.getValueType();
9585 EVT EltVT = VecVT.getVectorElementType();
9586 unsigned VecSize = VecVT.getSizeInBits();
9587 unsigned EltSize = EltVT.getSizeInBits();
9588 SDLoc SL(Op);
9589
9590 // Specially handle the case of v4i16 with static indexing.
9591 unsigned NumElts = VecVT.getVectorNumElements();
9592 auto *KIdx = dyn_cast<ConstantSDNode>(Val&: Idx);
9593 if (NumElts == 4 && EltSize == 16 && KIdx) {
9594 SDValue BCVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Vec);
9595
9596 SDValue LoHalf = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: BCVec,
9597 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
9598 SDValue HiHalf = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: BCVec,
9599 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
9600
9601 SDValue LoVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: LoHalf);
9602 SDValue HiVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: HiHalf);
9603
9604 unsigned Idx = KIdx->getZExtValue();
9605 bool InsertLo = Idx < 2;
9606 SDValue InsHalf = DAG.getNode(
9607 Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: MVT::v2i16, N1: InsertLo ? LoVec : HiVec,
9608 N2: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: InsVal),
9609 N3: DAG.getConstant(Val: InsertLo ? Idx : (Idx - 2), DL: SL, VT: MVT::i32));
9610
9611 InsHalf = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: InsHalf);
9612
9613 SDValue Concat =
9614 InsertLo ? DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {InsHalf, HiHalf})
9615 : DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {LoHalf, InsHalf});
9616
9617 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: Concat);
9618 }
9619
9620 // Static indexing does not lower to stack access, and hence there is no need
9621 // for special custom lowering to avoid stack access.
9622 if (isa<ConstantSDNode>(Val: Idx))
9623 return SDValue();
9624
9625 // Avoid stack access for dynamic indexing by custom lowering to
9626 // v_bfi_b32 (v_bfm_b32 16, (shl idx, 16)), val, vec
9627
9628 assert(VecSize <= 64 && "Expected target vector size to be <= 64 bits");
9629
9630 MVT IntVT = MVT::getIntegerVT(BitWidth: VecSize);
9631
9632 // Convert vector index to bit-index and get the required bit mask.
9633 assert(isPowerOf2_32(EltSize));
9634 const auto EltMask = maskTrailingOnes<uint64_t>(N: EltSize);
9635 SDValue ScaleFactor = DAG.getConstant(Val: Log2_32(Value: EltSize), DL: SL, VT: MVT::i32);
9636 SDValue ScaledIdx = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Idx, N2: ScaleFactor);
9637 SDValue BFM = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: IntVT,
9638 N1: DAG.getConstant(Val: EltMask, DL: SL, VT: IntVT), N2: ScaledIdx);
9639
9640 // 1. Create a congruent vector with the target value in each element.
9641 SDValue ExtVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT,
9642 Operand: DAG.getSplatBuildVector(VT: VecVT, DL: SL, Op: InsVal));
9643
9644 // 2. Mask off all other indices except the required index within (1).
9645 SDValue LHS = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IntVT, N1: BFM, N2: ExtVal);
9646
9647 // 3. Mask off the required index within the target vector.
9648 SDValue BCVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT, Operand: Vec);
9649 SDValue RHS =
9650 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IntVT, N1: DAG.getNOT(DL: SL, Val: BFM, VT: IntVT), N2: BCVec);
9651
9652 // 4. Get (2) and (3) ORed into the target vector.
9653 SDValue BFI =
9654 DAG.getNode(Opcode: ISD::OR, DL: SL, VT: IntVT, N1: LHS, N2: RHS, Flags: SDNodeFlags::Disjoint);
9655
9656 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: BFI);
9657}
9658
9659SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
9660 SelectionDAG &DAG) const {
9661 SDLoc SL(Op);
9662
9663 EVT ResultVT = Op.getValueType();
9664 SDValue Vec = Op.getOperand(i: 0);
9665 SDValue Idx = Op.getOperand(i: 1);
9666 EVT VecVT = Vec.getValueType();
9667 unsigned VecSize = VecVT.getSizeInBits();
9668 EVT EltVT = VecVT.getVectorElementType();
9669
9670 DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr);
9671
9672 // Make sure we do any optimizations that will make it easier to fold
9673 // source modifiers before obscuring it with bit operations.
9674
9675 // XXX - Why doesn't this get called when vector_shuffle is expanded?
9676 if (SDValue Combined = performExtractVectorEltCombine(N: Op.getNode(), DCI))
9677 return Combined;
9678
9679 if (VecSize == 128 || VecSize == 256 || VecSize == 512) {
9680 SDValue Lo, Hi;
9681 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
9682
9683 if (VecSize == 128) {
9684 SDValue V2 = DAG.getBitcast(VT: MVT::v2i64, V: Vec);
9685 Lo = DAG.getBitcast(VT: LoVT,
9686 V: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9687 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32)));
9688 Hi = DAG.getBitcast(VT: HiVT,
9689 V: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9690 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32)));
9691 } else if (VecSize == 256) {
9692 SDValue V2 = DAG.getBitcast(VT: MVT::v4i64, V: Vec);
9693 SDValue Parts[4];
9694 for (unsigned P = 0; P < 4; ++P) {
9695 Parts[P] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9696 N2: DAG.getConstant(Val: P, DL: SL, VT: MVT::i32));
9697 }
9698
9699 Lo = DAG.getBitcast(VT: LoVT, V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i64,
9700 N1: Parts[0], N2: Parts[1]));
9701 Hi = DAG.getBitcast(VT: HiVT, V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i64,
9702 N1: Parts[2], N2: Parts[3]));
9703 } else {
9704 assert(VecSize == 512);
9705
9706 SDValue V2 = DAG.getBitcast(VT: MVT::v8i64, V: Vec);
9707 SDValue Parts[8];
9708 for (unsigned P = 0; P < 8; ++P) {
9709 Parts[P] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9710 N2: DAG.getConstant(Val: P, DL: SL, VT: MVT::i32));
9711 }
9712
9713 Lo = DAG.getBitcast(VT: LoVT,
9714 V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v4i64,
9715 N1: Parts[0], N2: Parts[1], N3: Parts[2], N4: Parts[3]));
9716 Hi = DAG.getBitcast(VT: HiVT,
9717 V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v4i64,
9718 N1: Parts[4], N2: Parts[5], N3: Parts[6], N4: Parts[7]));
9719 }
9720
9721 EVT IdxVT = Idx.getValueType();
9722 unsigned NElem = VecVT.getVectorNumElements();
9723 assert(isPowerOf2_32(NElem));
9724 SDValue IdxMask = DAG.getConstant(Val: NElem / 2 - 1, DL: SL, VT: IdxVT);
9725 SDValue NewIdx = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IdxVT, N1: Idx, N2: IdxMask);
9726 SDValue Half = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IdxMask, True: Hi, False: Lo, Cond: ISD::SETUGT);
9727 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Half, N2: NewIdx);
9728 }
9729
9730 assert(VecSize <= 64);
9731
9732 MVT IntVT = MVT::getIntegerVT(BitWidth: VecSize);
9733
9734 // If Vec is just a SCALAR_TO_VECTOR, then use the scalar integer directly.
9735 SDValue VecBC = peekThroughBitcasts(V: Vec);
9736 if (VecBC.getOpcode() == ISD::SCALAR_TO_VECTOR) {
9737 SDValue Src = VecBC.getOperand(i: 0);
9738 Src = DAG.getBitcast(VT: Src.getValueType().changeTypeToInteger(), V: Src);
9739 Vec = DAG.getAnyExtOrTrunc(Op: Src, DL: SL, VT: IntVT);
9740 }
9741
9742 unsigned EltSize = EltVT.getSizeInBits();
9743 assert(isPowerOf2_32(EltSize));
9744
9745 SDValue ScaleFactor = DAG.getConstant(Val: Log2_32(Value: EltSize), DL: SL, VT: MVT::i32);
9746
9747 // Convert vector index to bit-index (* EltSize)
9748 SDValue ScaledIdx = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Idx, N2: ScaleFactor);
9749
9750 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT, Operand: Vec);
9751 SDValue Elt = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: IntVT, N1: BC, N2: ScaledIdx);
9752
9753 if (ResultVT == MVT::f16 || ResultVT == MVT::bf16) {
9754 SDValue Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i16, Operand: Elt);
9755 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ResultVT, Operand: Result);
9756 }
9757
9758 return DAG.getAnyExtOrTrunc(Op: Elt, DL: SL, VT: ResultVT);
9759}
9760
9761static bool elementPairIsContiguous(ArrayRef<int> Mask, int Elt) {
9762 assert(Elt % 2 == 0);
9763 return Mask[Elt + 1] == Mask[Elt] + 1 && (Mask[Elt] % 2 == 0);
9764}
9765
9766static bool elementPairIsOddToEven(ArrayRef<int> Mask, int Elt) {
9767 assert(Elt % 2 == 0);
9768 return Mask[Elt] >= 0 && Mask[Elt + 1] >= 0 && (Mask[Elt] & 1) &&
9769 !(Mask[Elt + 1] & 1);
9770}
9771
9772SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
9773 SelectionDAG &DAG) const {
9774 SDLoc SL(Op);
9775 EVT ResultVT = Op.getValueType();
9776 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
9777 MVT EltVT = ResultVT.getVectorElementType().getSimpleVT();
9778 const int NewSrcNumElts = 2;
9779 MVT PackVT = MVT::getVectorVT(VT: EltVT, NumElements: NewSrcNumElts);
9780 int SrcNumElts = Op.getOperand(i: 0).getValueType().getVectorNumElements();
9781
9782 // Break up the shuffle into registers sized pieces.
9783 //
9784 // We're trying to form sub-shuffles that the register allocation pipeline
9785 // won't be able to figure out, like how to use v_pk_mov_b32 to do a register
9786 // blend or 16-bit op_sel. It should be able to figure out how to reassemble a
9787 // pair of copies into a consecutive register copy, so use the ordinary
9788 // extract_vector_elt lowering unless we can use the shuffle.
9789 //
9790 // TODO: This is a bit of hack, and we should probably always use
9791 // extract_subvector for the largest possible subvector we can (or at least
9792 // use it for PackVT aligned pieces). However we have worse support for
9793 // combines on them don't directly treat extract_subvector / insert_subvector
9794 // as legal. The DAG scheduler also ends up doing a worse job with the
9795 // extract_subvectors.
9796 const bool ShouldUseConsecutiveExtract = EltVT.getSizeInBits() == 16;
9797
9798 // vector_shuffle <0,1,6,7> lhs, rhs
9799 // -> concat_vectors (extract_subvector lhs, 0), (extract_subvector rhs, 2)
9800 //
9801 // vector_shuffle <6,7,2,3> lhs, rhs
9802 // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 2)
9803 //
9804 // vector_shuffle <6,7,0,1> lhs, rhs
9805 // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 0)
9806
9807 // Avoid scalarizing when both halves are reading from consecutive elements.
9808
9809 // If we're treating 2 element shuffles as legal, also create odd-to-even
9810 // shuffles of neighboring pairs.
9811 //
9812 // vector_shuffle <3,2,7,6> lhs, rhs
9813 // -> concat_vectors vector_shuffle <1, 0> (extract_subvector lhs, 0)
9814 // vector_shuffle <1, 0> (extract_subvector rhs, 2)
9815
9816 SmallVector<SDValue, 16> Pieces;
9817 for (int I = 0, N = ResultVT.getVectorNumElements(); I != N; I += 2) {
9818 if (ShouldUseConsecutiveExtract &&
9819 elementPairIsContiguous(Mask: SVN->getMask(), Elt: I)) {
9820 const int Idx = SVN->getMaskElt(Idx: I);
9821 int VecIdx = Idx < SrcNumElts ? 0 : 1;
9822 int EltIdx = Idx < SrcNumElts ? Idx : Idx - SrcNumElts;
9823 SDValue SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT,
9824 N1: SVN->getOperand(Num: VecIdx),
9825 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
9826 Pieces.push_back(Elt: SubVec);
9827 } else if (elementPairIsOddToEven(Mask: SVN->getMask(), Elt: I) &&
9828 isOperationLegal(Op: ISD::VECTOR_SHUFFLE, VT: PackVT)) {
9829 int Idx0 = SVN->getMaskElt(Idx: I);
9830 int Idx1 = SVN->getMaskElt(Idx: I + 1);
9831
9832 SDValue SrcOp0 = SVN->getOperand(Num: 0);
9833 SDValue SrcOp1 = SrcOp0;
9834 if (Idx0 >= SrcNumElts) {
9835 SrcOp0 = SVN->getOperand(Num: 1);
9836 Idx0 -= SrcNumElts;
9837 }
9838
9839 if (Idx1 >= SrcNumElts) {
9840 SrcOp1 = SVN->getOperand(Num: 1);
9841 Idx1 -= SrcNumElts;
9842 }
9843
9844 int AlignedIdx0 = Idx0 & ~(NewSrcNumElts - 1);
9845 int AlignedIdx1 = Idx1 & ~(NewSrcNumElts - 1);
9846
9847 // Extract nearest even aligned piece.
9848 SDValue SubVec0 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT, N1: SrcOp0,
9849 N2: DAG.getConstant(Val: AlignedIdx0, DL: SL, VT: MVT::i32));
9850 SDValue SubVec1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT, N1: SrcOp1,
9851 N2: DAG.getConstant(Val: AlignedIdx1, DL: SL, VT: MVT::i32));
9852
9853 int NewMaskIdx0 = Idx0 - AlignedIdx0;
9854 int NewMaskIdx1 = Idx1 - AlignedIdx1;
9855
9856 SDValue Result0 = SubVec0;
9857 SDValue Result1 = SubVec0;
9858
9859 if (SubVec0 != SubVec1) {
9860 NewMaskIdx1 += NewSrcNumElts;
9861 Result1 = SubVec1;
9862 } else {
9863 Result1 = DAG.getPOISON(VT: PackVT);
9864 }
9865
9866 SDValue Shuf = DAG.getVectorShuffle(VT: PackVT, dl: SL, N1: Result0, N2: Result1,
9867 Mask: {NewMaskIdx0, NewMaskIdx1});
9868 Pieces.push_back(Elt: Shuf);
9869 } else {
9870 const int Idx0 = SVN->getMaskElt(Idx: I);
9871 const int Idx1 = SVN->getMaskElt(Idx: I + 1);
9872 int VecIdx0 = Idx0 < SrcNumElts ? 0 : 1;
9873 int VecIdx1 = Idx1 < SrcNumElts ? 0 : 1;
9874 int EltIdx0 = Idx0 < SrcNumElts ? Idx0 : Idx0 - SrcNumElts;
9875 int EltIdx1 = Idx1 < SrcNumElts ? Idx1 : Idx1 - SrcNumElts;
9876
9877 SDValue Vec0 = SVN->getOperand(Num: VecIdx0);
9878 SDValue Elt0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec0,
9879 N2: DAG.getSignedConstant(Val: EltIdx0, DL: SL, VT: MVT::i32));
9880
9881 SDValue Vec1 = SVN->getOperand(Num: VecIdx1);
9882 SDValue Elt1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec1,
9883 N2: DAG.getSignedConstant(Val: EltIdx1, DL: SL, VT: MVT::i32));
9884 Pieces.push_back(Elt: DAG.getBuildVector(VT: PackVT, DL: SL, Ops: {Elt0, Elt1}));
9885 }
9886 }
9887
9888 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: ResultVT, Ops: Pieces);
9889}
9890
9891SDValue SITargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
9892 SelectionDAG &DAG) const {
9893 SDValue SVal = Op.getOperand(i: 0);
9894 EVT ResultVT = Op.getValueType();
9895 EVT SValVT = SVal.getValueType();
9896 SDValue UndefVal = DAG.getPOISON(VT: SValVT);
9897 SDLoc SL(Op);
9898
9899 SmallVector<SDValue, 8> VElts;
9900 VElts.push_back(Elt: SVal);
9901 for (int I = 1, E = ResultVT.getVectorNumElements(); I < E; ++I)
9902 VElts.push_back(Elt: UndefVal);
9903
9904 return DAG.getBuildVector(VT: ResultVT, DL: SL, Ops: VElts);
9905}
9906
9907SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op,
9908 SelectionDAG &DAG) const {
9909 SDLoc SL(Op);
9910 EVT VT = Op.getValueType();
9911
9912 if (VT == MVT::v2f16 || VT == MVT::v2i16 || VT == MVT::v2bf16) {
9913 assert(!Subtarget->hasVOP3PInsts() && "this should be legal");
9914
9915 SDValue Lo = Op.getOperand(i: 0);
9916 SDValue Hi = Op.getOperand(i: 1);
9917
9918 // Avoid adding defined bits with the zero_extend.
9919 if (Hi.isUndef()) {
9920 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Lo);
9921 SDValue ExtLo = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Lo);
9922 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: ExtLo);
9923 }
9924
9925 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Hi);
9926 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Hi);
9927
9928 SDValue ShlHi = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Hi,
9929 N2: DAG.getConstant(Val: 16, DL: SL, VT: MVT::i32));
9930 if (Lo.isUndef())
9931 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: ShlHi);
9932
9933 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Lo);
9934 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Lo);
9935
9936 SDValue Or =
9937 DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Lo, N2: ShlHi, Flags: SDNodeFlags::Disjoint);
9938 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Or);
9939 }
9940
9941 // Split into 2-element chunks.
9942 const unsigned NumParts = VT.getVectorNumElements() / 2;
9943 EVT PartVT = MVT::getVectorVT(VT: VT.getVectorElementType().getSimpleVT(), NumElements: 2);
9944 MVT PartIntVT = MVT::getIntegerVT(BitWidth: PartVT.getSizeInBits());
9945
9946 SmallVector<SDValue> Casts;
9947 for (unsigned P = 0; P < NumParts; ++P) {
9948 SDValue Vec = DAG.getBuildVector(
9949 VT: PartVT, DL: SL, Ops: {Op.getOperand(i: P * 2), Op.getOperand(i: P * 2 + 1)});
9950 Casts.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: PartIntVT, Operand: Vec));
9951 }
9952
9953 SDValue Blend =
9954 DAG.getBuildVector(VT: MVT::getVectorVT(VT: PartIntVT, NumElements: NumParts), DL: SL, Ops: Casts);
9955 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Blend);
9956}
9957
9958bool SITargetLowering::isOffsetFoldingLegal(
9959 const GlobalAddressSDNode *GA) const {
9960 // Named barriers have fixed, non-relocated LDS addresses, so a constant
9961 // offset into an array of them can be folded into the address.
9962 if (GA->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
9963 const auto *GV = dyn_cast<GlobalVariable>(Val: GA->getGlobal());
9964 return GV && AMDGPU::isNamedBarrier(GV: *GV);
9965 }
9966
9967 // OSes that use ELF REL relocations (instead of RELA) can only store a
9968 // 32-bit addend in the instruction, so it is not safe to allow offset folding
9969 // which can create arbitrary 64-bit addends. (This is only a problem for
9970 // R_AMDGPU_*32_HI relocations since other relocation types are unaffected by
9971 // the high 32 bits of the addend.)
9972 //
9973 // This should be kept in sync with how HasRelocationAddend is initialized in
9974 // the constructor of ELFAMDGPUAsmBackend.
9975 if (!Subtarget->isAmdHsaOS())
9976 return false;
9977
9978 // We can fold offsets for anything that doesn't require a GOT relocation.
9979 return (GA->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS ||
9980 GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
9981 GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
9982 !shouldEmitGOTReloc(GV: GA->getGlobal());
9983}
9984
9985static SDValue
9986buildPCRelGlobalAddress(SelectionDAG &DAG, const GlobalValue *GV,
9987 const SDLoc &DL, int64_t Offset, EVT PtrVT,
9988 unsigned GAFlags = SIInstrInfo::MO_NONE) {
9989 assert(isInt<32>(Offset + 4) && "32-bit offset is expected!");
9990 // In order to support pc-relative addressing, the PC_ADD_REL_OFFSET SDNode is
9991 // lowered to the following code sequence:
9992 //
9993 // For constant address space:
9994 // s_getpc_b64 s[0:1]
9995 // s_add_u32 s0, s0, $symbol
9996 // s_addc_u32 s1, s1, 0
9997 //
9998 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
9999 // a fixup or relocation is emitted to replace $symbol with a literal
10000 // constant, which is a pc-relative offset from the encoding of the $symbol
10001 // operand to the global variable.
10002 //
10003 // For global address space:
10004 // s_getpc_b64 s[0:1]
10005 // s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
10006 // s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
10007 //
10008 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
10009 // fixups or relocations are emitted to replace $symbol@*@lo and
10010 // $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
10011 // which is a 64-bit pc-relative offset from the encoding of the $symbol
10012 // operand to the global variable.
10013 if (((const GCNSubtarget &)DAG.getSubtarget()).has64BitLiterals()) {
10014 assert(GAFlags != SIInstrInfo::MO_NONE);
10015
10016 SDValue Ptr =
10017 DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i64, offset: Offset, TargetFlags: GAFlags + 2);
10018 return DAG.getNode(Opcode: AMDGPUISD::PC_ADD_REL_OFFSET64, DL, VT: PtrVT, Operand: Ptr);
10019 }
10020
10021 SDValue PtrLo = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: Offset, TargetFlags: GAFlags);
10022 SDValue PtrHi;
10023 if (GAFlags == SIInstrInfo::MO_NONE)
10024 PtrHi = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32);
10025 else
10026 PtrHi = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: Offset, TargetFlags: GAFlags + 1);
10027 return DAG.getNode(Opcode: AMDGPUISD::PC_ADD_REL_OFFSET, DL, VT: PtrVT, N1: PtrLo, N2: PtrHi);
10028}
10029
10030SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
10031 SDValue Op,
10032 SelectionDAG &DAG) const {
10033 GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Val&: Op);
10034 SDLoc DL(GSD);
10035 EVT PtrVT = Op.getValueType();
10036
10037 const GlobalValue *GV = GSD->getGlobal();
10038 if ((GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS &&
10039 shouldUseLDSConstAddress(GV)) ||
10040 GSD->getAddressSpace() == AMDGPUAS::REGION_ADDRESS ||
10041 GSD->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) {
10042 if (GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS &&
10043 GV->hasExternalLinkage()) {
10044 const GlobalVariable &GVar = *cast<GlobalVariable>(Val: GV);
10045 // HIP uses an unsized array `extern __shared__ T s[]` or similar
10046 // zero-sized type in other languages to declare the dynamic shared
10047 // memory which size is not known at the compile time. They will be
10048 // allocated by the runtime and placed directly after the static
10049 // allocated ones. They all share the same offset.
10050 if (GVar.getGlobalSize(DL: GVar.getDataLayout()) == 0) {
10051 assert(PtrVT == MVT::i32 && "32-bit pointer is expected.");
10052 // Adjust alignment for that dynamic shared memory array.
10053 Function &F = DAG.getMachineFunction().getFunction();
10054 MFI->setDynLDSAlign(F, GV: GVar);
10055 MFI->setUsesDynamicLDS(true);
10056 return SDValue(
10057 DAG.getMachineNode(Opcode: AMDGPU::GET_GROUPSTATICSIZE, dl: DL, VT: PtrVT), 0);
10058 }
10059 }
10060 return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG);
10061 }
10062
10063 if (GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
10064 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: GSD->getOffset(),
10065 TargetFlags: SIInstrInfo::MO_ABS32_LO);
10066 return DAG.getNode(Opcode: AMDGPUISD::LDS, DL, VT: MVT::i32, Operand: GA);
10067 }
10068
10069 if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS()) {
10070 if (Subtarget->has64BitLiterals()) {
10071 SDValue Addr = DAG.getTargetGlobalAddress(
10072 GV, DL, VT: MVT::i64, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS64);
10073 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B64, dl: DL, VT: MVT::i64, Op1: Addr),
10074 0);
10075 }
10076
10077 SDValue AddrLo = DAG.getTargetGlobalAddress(
10078 GV, DL, VT: MVT::i32, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS32_LO);
10079 AddrLo = {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: AddrLo), 0};
10080
10081 SDValue AddrHi = DAG.getTargetGlobalAddress(
10082 GV, DL, VT: MVT::i32, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS32_HI);
10083 AddrHi = {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: AddrHi), 0};
10084
10085 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: AddrLo, N2: AddrHi);
10086 }
10087
10088 if (shouldEmitFixup(GV))
10089 return buildPCRelGlobalAddress(DAG, GV, DL, Offset: GSD->getOffset(), PtrVT);
10090
10091 if (shouldEmitPCReloc(GV))
10092 return buildPCRelGlobalAddress(DAG, GV, DL, Offset: GSD->getOffset(), PtrVT,
10093 GAFlags: SIInstrInfo::MO_REL32);
10094
10095 SDValue GOTAddr = buildPCRelGlobalAddress(DAG, GV, DL, Offset: 0, PtrVT,
10096 GAFlags: SIInstrInfo::MO_GOTPCREL32);
10097 PointerType *PtrTy =
10098 PointerType::get(C&: *DAG.getContext(), AddressSpace: AMDGPUAS::CONSTANT_ADDRESS);
10099 const DataLayout &DataLayout = DAG.getDataLayout();
10100 Align Alignment = DataLayout.getABITypeAlign(Ty: PtrTy);
10101 MachinePointerInfo PtrInfo =
10102 MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction());
10103
10104 return DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: GOTAddr, PtrInfo, Alignment,
10105 MMOFlags: MachineMemOperand::MODereferenceable |
10106 MachineMemOperand::MOInvariant);
10107}
10108
10109SDValue SITargetLowering::LowerExternalSymbol(SDValue Op,
10110 SelectionDAG &DAG) const {
10111 // TODO: Handle this. It should be mostly the same as LowerGlobalAddress.
10112 const Function &Fn = DAG.getMachineFunction().getFunction();
10113 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10114 Fn, "unsupported external symbol", Op.getDebugLoc()));
10115 return DAG.getPOISON(VT: Op.getValueType());
10116}
10117
10118SDValue SITargetLowering::copyToM0(SelectionDAG &DAG, SDValue Chain,
10119 const SDLoc &DL, SDValue V) const {
10120 // We can't use S_MOV_B32 directly, because there is no way to specify m0 as
10121 // the destination register.
10122 //
10123 // We can't use CopyToReg, because MachineCSE won't combine COPY instructions,
10124 // so we will end up with redundant moves to m0.
10125 //
10126 // We use a pseudo to ensure we emit s_mov_b32 with m0 as the direct result.
10127
10128 // A Null SDValue creates a glue result.
10129 SDNode *M0 = DAG.getMachineNode(Opcode: AMDGPU::SI_INIT_M0, dl: DL, VT1: MVT::Other, VT2: MVT::Glue,
10130 Op1: V, Op2: Chain);
10131 return SDValue(M0, 0);
10132}
10133
10134SDValue SITargetLowering::lowerImplicitZextParam(SelectionDAG &DAG, SDValue Op,
10135 MVT VT,
10136 unsigned Offset) const {
10137 SDLoc SL(Op);
10138 SDValue Param = lowerKernargMemParameter(
10139 DAG, VT: MVT::i32, MemVT: MVT::i32, SL, Chain: DAG.getEntryNode(), Offset, Alignment: Align(4), Signed: false);
10140 // The local size values will have the hi 16-bits as zero.
10141 return DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: MVT::i32, N1: Param,
10142 N2: DAG.getValueType(VT));
10143}
10144
10145static SDValue emitNonHSAIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
10146 EVT VT) {
10147 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10148 DAG.getMachineFunction().getFunction(),
10149 "non-hsa intrinsic with hsa target", DL.getDebugLoc()));
10150 return DAG.getPOISON(VT);
10151}
10152
10153static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
10154 EVT VT) {
10155 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10156 DAG.getMachineFunction().getFunction(),
10157 "intrinsic not supported on subtarget", DL.getDebugLoc()));
10158 return DAG.getPOISON(VT);
10159}
10160
10161static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL,
10162 ArrayRef<SDValue> Elts) {
10163 assert(!Elts.empty());
10164 MVT Type;
10165 unsigned NumElts = Elts.size();
10166
10167 if (NumElts <= 12) {
10168 Type = MVT::getVectorVT(VT: MVT::f32, NumElements: NumElts);
10169 } else {
10170 assert(Elts.size() <= 16);
10171 Type = MVT::v16f32;
10172 NumElts = 16;
10173 }
10174
10175 SmallVector<SDValue, 16> VecElts(NumElts);
10176 for (unsigned i = 0; i < Elts.size(); ++i) {
10177 SDValue Elt = Elts[i];
10178 if (Elt.getValueType() != MVT::f32)
10179 Elt = DAG.getBitcast(VT: MVT::f32, V: Elt);
10180 VecElts[i] = Elt;
10181 }
10182 for (unsigned i = Elts.size(); i < NumElts; ++i)
10183 VecElts[i] = DAG.getPOISON(VT: MVT::f32);
10184
10185 if (NumElts == 1)
10186 return VecElts[0];
10187 return DAG.getBuildVector(VT: Type, DL, Ops: VecElts);
10188}
10189
10190static SDValue padEltsToUndef(SelectionDAG &DAG, const SDLoc &DL, EVT CastVT,
10191 SDValue Src, int ExtraElts) {
10192 EVT SrcVT = Src.getValueType();
10193
10194 SmallVector<SDValue, 8> Elts;
10195
10196 if (SrcVT.isVector())
10197 DAG.ExtractVectorElements(Op: Src, Args&: Elts);
10198 else
10199 Elts.push_back(Elt: Src);
10200
10201 SDValue Undef = DAG.getPOISON(VT: SrcVT.getScalarType());
10202 while (ExtraElts--)
10203 Elts.push_back(Elt: Undef);
10204
10205 return DAG.getBuildVector(VT: CastVT, DL, Ops: Elts);
10206}
10207
10208// Re-construct the required return value for a image load intrinsic.
10209// This is more complicated due to the optional use TexFailCtrl which means the
10210// required return type is an aggregate
10211static SDValue constructRetValue(SelectionDAG &DAG, MachineSDNode *Result,
10212 ArrayRef<EVT> ResultTypes, bool IsTexFail,
10213 bool Unpacked, bool IsD16, int DMaskPop,
10214 int NumVDataDwords, bool IsAtomicPacked16Bit,
10215 const SDLoc &DL) {
10216 // Determine the required return type. This is the same regardless of
10217 // IsTexFail flag
10218 EVT ReqRetVT = ResultTypes[0];
10219 int ReqRetNumElts = ReqRetVT.isVector() ? ReqRetVT.getVectorNumElements() : 1;
10220 int NumDataDwords = ((IsD16 && !Unpacked) || IsAtomicPacked16Bit)
10221 ? (ReqRetNumElts + 1) / 2
10222 : ReqRetNumElts;
10223
10224 int MaskPopDwords = (!IsD16 || Unpacked) ? DMaskPop : (DMaskPop + 1) / 2;
10225
10226 MVT DataDwordVT =
10227 NumDataDwords == 1 ? MVT::i32 : MVT::getVectorVT(VT: MVT::i32, NumElements: NumDataDwords);
10228
10229 MVT MaskPopVT =
10230 MaskPopDwords == 1 ? MVT::i32 : MVT::getVectorVT(VT: MVT::i32, NumElements: MaskPopDwords);
10231
10232 SDValue Data(Result, 0);
10233 SDValue TexFail;
10234
10235 if (DMaskPop > 0 && Data.getValueType() != MaskPopVT) {
10236 SDValue ZeroIdx = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
10237 if (MaskPopVT.isVector()) {
10238 Data = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: MaskPopVT,
10239 N1: SDValue(Result, 0), N2: ZeroIdx);
10240 } else {
10241 Data = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MaskPopVT,
10242 N1: SDValue(Result, 0), N2: ZeroIdx);
10243 }
10244 }
10245
10246 if (DataDwordVT.isVector() && !IsAtomicPacked16Bit)
10247 Data = padEltsToUndef(DAG, DL, CastVT: DataDwordVT, Src: Data,
10248 ExtraElts: NumDataDwords - MaskPopDwords);
10249
10250 if (IsD16)
10251 Data = adjustLoadValueTypeImpl(Result: Data, LoadVT: ReqRetVT, DL, DAG, Unpacked);
10252
10253 EVT LegalReqRetVT = ReqRetVT;
10254 if (!ReqRetVT.isVector()) {
10255 if (!Data.getValueType().isInteger())
10256 Data = DAG.getNode(Opcode: ISD::BITCAST, DL,
10257 VT: Data.getValueType().changeTypeToInteger(), Operand: Data);
10258 Data = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ReqRetVT.changeTypeToInteger(), Operand: Data);
10259 } else {
10260 // We need to widen the return vector to a legal type
10261 if ((ReqRetVT.getVectorNumElements() % 2) == 1 &&
10262 ReqRetVT.getVectorElementType().getSizeInBits() == 16) {
10263 LegalReqRetVT =
10264 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ReqRetVT.getVectorElementType(),
10265 NumElements: ReqRetVT.getVectorNumElements() + 1);
10266 }
10267 }
10268 Data = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LegalReqRetVT, Operand: Data);
10269
10270 if (IsTexFail) {
10271 TexFail =
10272 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: SDValue(Result, 0),
10273 N2: DAG.getConstant(Val: MaskPopDwords, DL, VT: MVT::i32));
10274
10275 return DAG.getMergeValues(Ops: {Data, TexFail, SDValue(Result, 1)}, dl: DL);
10276 }
10277
10278 if (Result->getNumValues() == 1)
10279 return Data;
10280
10281 return DAG.getMergeValues(Ops: {Data, SDValue(Result, 1)}, dl: DL);
10282}
10283
10284static bool parseTexFail(SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE,
10285 SDValue *LWE, bool &IsTexFail) {
10286 auto *TexFailCtrlConst = cast<ConstantSDNode>(Val: TexFailCtrl.getNode());
10287
10288 uint64_t Value = TexFailCtrlConst->getZExtValue();
10289 if (Value) {
10290 IsTexFail = true;
10291 }
10292
10293 SDLoc DL(TexFailCtrlConst);
10294 *TFE = DAG.getTargetConstant(Val: (Value & 0x1) ? 1 : 0, DL, VT: MVT::i32);
10295 Value &= ~(uint64_t)0x1;
10296 *LWE = DAG.getTargetConstant(Val: (Value & 0x2) ? 1 : 0, DL, VT: MVT::i32);
10297 Value &= ~(uint64_t)0x2;
10298
10299 return Value == 0;
10300}
10301
10302static void packImage16bitOpsToDwords(SelectionDAG &DAG, SDValue Op,
10303 MVT PackVectorVT,
10304 SmallVectorImpl<SDValue> &PackedAddrs,
10305 unsigned DimIdx, unsigned EndIdx,
10306 unsigned NumGradients) {
10307 SDLoc DL(Op);
10308 for (unsigned I = DimIdx; I < EndIdx; I++) {
10309 SDValue Addr = Op.getOperand(i: I);
10310
10311 // Gradients are packed with undef for each coordinate.
10312 // In <hi 16 bit>,<lo 16 bit> notation, the registers look like this:
10313 // 1D: undef,dx/dh; undef,dx/dv
10314 // 2D: dy/dh,dx/dh; dy/dv,dx/dv
10315 // 3D: dy/dh,dx/dh; undef,dz/dh; dy/dv,dx/dv; undef,dz/dv
10316 if (((I + 1) >= EndIdx) ||
10317 ((NumGradients / 2) % 2 == 1 && (I == DimIdx + (NumGradients / 2) - 1 ||
10318 I == DimIdx + NumGradients - 1))) {
10319 if (Addr.getValueType() != MVT::i16)
10320 Addr = DAG.getBitcast(VT: MVT::i16, V: Addr);
10321 Addr = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Addr);
10322 } else {
10323 Addr = DAG.getBuildVector(VT: PackVectorVT, DL, Ops: {Addr, Op.getOperand(i: I + 1)});
10324 I++;
10325 }
10326 Addr = DAG.getBitcast(VT: MVT::f32, V: Addr);
10327 PackedAddrs.push_back(Elt: Addr);
10328 }
10329}
10330
10331/// Emit a DiagnosticInfoUnsupported for an unsupported image intrinsic and
10332/// return poison values of \p ResultTypes, preserving the chain if present.
10333static SDValue diagnoseUnsupportedImage(SelectionDAG &DAG, SDValue Op,
10334 ArrayRef<EVT> ResultTypes,
10335 const SDLoc &DL, const Twine &Msg) {
10336 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10337 DAG.getMachineFunction().getFunction(), Msg, DL.getDebugLoc()));
10338 return DAG.getErrorMergeValues(ResultTypes, Chain: Op.getOperand(i: 0), dl: DL);
10339}
10340
10341SDValue SITargetLowering::lowerImage(SDValue Op,
10342 const AMDGPU::ImageDimIntrinsicInfo *Intr,
10343 SelectionDAG &DAG, bool WithChain) const {
10344 SDLoc DL(Op);
10345 MachineFunction &MF = DAG.getMachineFunction();
10346 const GCNSubtarget *ST = &MF.getSubtarget<GCNSubtarget>();
10347 unsigned IntrOpcode = Intr->BaseOpcode;
10348 // For image atomic: use no-return opcode if result is unused.
10349 if (Intr->AtomicNoRetBaseOpcode != Intr->BaseOpcode &&
10350 !Op.getNode()->hasAnyUseOfValue(Value: 0))
10351 IntrOpcode = Intr->AtomicNoRetBaseOpcode;
10352 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
10353 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: IntrOpcode);
10354 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(DimEnum: Intr->Dim);
10355 bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI: *Subtarget);
10356 bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI: *Subtarget);
10357 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
10358 bool IsGFX13 = AMDGPU::isGFX13(STI: *Subtarget);
10359
10360 SmallVector<EVT, 3> ResultTypes(Op->values());
10361 SmallVector<EVT, 3> OrigResultTypes(Op->values());
10362 if (BaseOpcode->NoReturn && BaseOpcode->Atomic)
10363 ResultTypes.erase(CI: &ResultTypes[0]);
10364
10365 bool IsD16 = false;
10366 bool IsG16 = false;
10367 bool IsA16 = false;
10368 SDValue VData;
10369 int NumVDataDwords = 0;
10370 bool AdjustRetType = false;
10371 bool IsAtomicPacked16Bit = false;
10372
10373 // Offset of intrinsic arguments
10374 const unsigned ArgOffset = WithChain ? 2 : 1;
10375
10376 unsigned DMask;
10377 unsigned DMaskLanes = 0;
10378
10379 if (BaseOpcode->Atomic) {
10380 VData = Op.getOperand(i: 2);
10381
10382 IsAtomicPacked16Bit =
10383 (IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
10384 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16_NORTN ||
10385 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16 ||
10386 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16_NORTN);
10387
10388 if (!IsAtomicPacked16Bit && VData.getValueSizeInBits() != 32 &&
10389 VData.getValueSizeInBits() != 64) {
10390 return diagnoseUnsupportedImage(DAG, Op, ResultTypes: OrigResultTypes, DL,
10391 Msg: "unsupported image atomic data type");
10392 }
10393
10394 bool Is64Bit = VData.getValueSizeInBits() == 64;
10395 if (BaseOpcode->AtomicX2) {
10396 SDValue VData2 = Op.getOperand(i: 3);
10397 VData = DAG.getBuildVector(VT: Is64Bit ? MVT::v2i64 : MVT::v2i32, DL,
10398 Ops: {VData, VData2});
10399 if (Is64Bit)
10400 VData = DAG.getBitcast(VT: MVT::v4i32, V: VData);
10401
10402 if (!BaseOpcode->NoReturn)
10403 ResultTypes[0] = Is64Bit ? MVT::v2i64 : MVT::v2i32;
10404
10405 DMask = Is64Bit ? 0xf : 0x3;
10406 NumVDataDwords = Is64Bit ? 4 : 2;
10407 } else {
10408 DMask = Is64Bit ? 0x3 : 0x1;
10409 NumVDataDwords = Is64Bit ? 2 : 1;
10410 }
10411 } else {
10412 DMask = Op->getConstantOperandVal(Num: ArgOffset + Intr->DMaskIndex);
10413 DMaskLanes = BaseOpcode->Gather4 ? 4 : llvm::popcount(Value: DMask);
10414
10415 if (BaseOpcode->Store) {
10416 VData = Op.getOperand(i: 2);
10417
10418 MVT StoreVT = VData.getSimpleValueType();
10419 MVT StoreScalarVT = StoreVT.getScalarType();
10420 if (StoreScalarVT != MVT::f16 && StoreScalarVT.getSizeInBits() != 32 &&
10421 StoreScalarVT.getSizeInBits() != 64) {
10422 return diagnoseUnsupportedImage(DAG, Op, ResultTypes: OrigResultTypes, DL,
10423 Msg: "unsupported image store data type");
10424 }
10425 if (StoreScalarVT == MVT::f16) {
10426 if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
10427 return Op; // D16 is unsupported for this instruction
10428
10429 IsD16 = true;
10430 VData = handleD16VData(VData, DAG, ImageStore: true);
10431 }
10432
10433 NumVDataDwords = (VData.getValueType().getSizeInBits() + 31) / 32;
10434 } else if (!BaseOpcode->NoReturn) {
10435 // Work out the num dwords based on the dmask popcount and underlying type
10436 // and whether packing is supported.
10437 MVT LoadVT = ResultTypes[0].getSimpleVT();
10438 MVT LoadScalarVT = LoadVT.getScalarType();
10439 if (LoadScalarVT != MVT::f16 && LoadScalarVT.getSizeInBits() != 32 &&
10440 LoadScalarVT.getSizeInBits() != 64) {
10441 return diagnoseUnsupportedImage(DAG, Op, ResultTypes: OrigResultTypes, DL,
10442 Msg: "unsupported image load data type");
10443 }
10444 if (LoadScalarVT == MVT::f16) {
10445 if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
10446 return Op; // D16 is unsupported for this instruction
10447
10448 IsD16 = true;
10449 }
10450
10451 // Confirm that the return type is large enough for the dmask specified
10452 if ((LoadVT.isVector() && LoadVT.getVectorNumElements() < DMaskLanes) ||
10453 (!LoadVT.isVector() && DMaskLanes > 1))
10454 return Op;
10455
10456 // The sq block of gfx8 and gfx9 do not estimate register use correctly
10457 // for d16 image_gather4, image_gather4_l, and image_gather4_lz
10458 // instructions.
10459 if (IsD16 && !Subtarget->hasUnpackedD16VMem() &&
10460 !(BaseOpcode->Gather4 && Subtarget->hasImageGather4D16Bug()))
10461 NumVDataDwords = (DMaskLanes + 1) / 2;
10462 else
10463 NumVDataDwords = DMaskLanes;
10464
10465 AdjustRetType = true;
10466 }
10467 }
10468
10469 unsigned VAddrEnd = ArgOffset + Intr->VAddrEnd;
10470 SmallVector<SDValue, 4> VAddrs;
10471
10472 // Check for 16 bit addresses or derivatives and pack if true.
10473 MVT VAddrVT =
10474 Op.getOperand(i: ArgOffset + Intr->GradientStart).getSimpleValueType();
10475 MVT VAddrScalarVT = VAddrVT.getScalarType();
10476 MVT GradPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
10477 IsG16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
10478
10479 VAddrVT = Op.getOperand(i: ArgOffset + Intr->CoordStart).getSimpleValueType();
10480 VAddrScalarVT = VAddrVT.getScalarType();
10481 MVT AddrPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
10482 IsA16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
10483
10484 // Push back extra arguments.
10485 for (unsigned I = Intr->VAddrStart; I < Intr->GradientStart; I++) {
10486 if (IsA16 && (Op.getOperand(i: ArgOffset + I).getValueType() == MVT::f16)) {
10487 assert(I == Intr->BiasIndex && "Got unexpected 16-bit extra argument");
10488 // Special handling of bias when A16 is on. Bias is of type half but
10489 // occupies full 32-bit.
10490 SDValue Bias = DAG.getBuildVector(
10491 VT: MVT::v2f16, DL,
10492 Ops: {Op.getOperand(i: ArgOffset + I), DAG.getPOISON(VT: MVT::f16)});
10493 VAddrs.push_back(Elt: Bias);
10494 } else {
10495 assert((!IsA16 || Intr->NumBiasArgs == 0 || I != Intr->BiasIndex) &&
10496 "Bias needs to be converted to 16 bit in A16 mode");
10497 VAddrs.push_back(Elt: Op.getOperand(i: ArgOffset + I));
10498 }
10499 }
10500
10501 if (BaseOpcode->Gradients && !ST->hasG16() && (IsA16 != IsG16)) {
10502 // 16 bit gradients are supported, but are tied to the A16 control
10503 // so both gradients and addresses must be 16 bit
10504 LLVM_DEBUG(
10505 dbgs() << "Failed to lower image intrinsic: 16 bit addresses "
10506 "require 16 bit args for both gradients and addresses");
10507 return Op;
10508 }
10509
10510 if (IsA16) {
10511 if (!ST->hasA16()) {
10512 LLVM_DEBUG(dbgs() << "Failed to lower image intrinsic: Target does not "
10513 "support 16 bit addresses\n");
10514 return Op;
10515 }
10516 }
10517
10518 // We've dealt with incorrect input so we know that if IsA16, IsG16
10519 // are set then we have to compress/pack operands (either address,
10520 // gradient or both)
10521 // In the case where a16 and gradients are tied (no G16 support) then we
10522 // have already verified that both IsA16 and IsG16 are true
10523 if (BaseOpcode->Gradients && IsG16 && ST->hasG16()) {
10524 // Activate g16
10525 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
10526 AMDGPU::getMIMGG16MappingInfo(G: Intr->BaseOpcode);
10527 IntrOpcode = G16MappingInfo->G16; // set new opcode to variant with _g16
10528 }
10529
10530 // Add gradients (packed or unpacked)
10531 if (IsG16) {
10532 // Pack the gradients
10533 // const int PackEndIdx = IsA16 ? VAddrEnd : (ArgOffset + Intr->CoordStart);
10534 packImage16bitOpsToDwords(DAG, Op, PackVectorVT: GradPackVectorVT, PackedAddrs&: VAddrs,
10535 DimIdx: ArgOffset + Intr->GradientStart,
10536 EndIdx: ArgOffset + Intr->CoordStart, NumGradients: Intr->NumGradients);
10537 } else {
10538 for (unsigned I = ArgOffset + Intr->GradientStart;
10539 I < ArgOffset + Intr->CoordStart; I++)
10540 VAddrs.push_back(Elt: Op.getOperand(i: I));
10541 }
10542
10543 // Add addresses (packed or unpacked)
10544 if (IsA16) {
10545 packImage16bitOpsToDwords(DAG, Op, PackVectorVT: AddrPackVectorVT, PackedAddrs&: VAddrs,
10546 DimIdx: ArgOffset + Intr->CoordStart, EndIdx: VAddrEnd,
10547 NumGradients: 0 /* No gradients */);
10548 } else {
10549 // Add uncompressed address
10550 for (unsigned I = ArgOffset + Intr->CoordStart; I < VAddrEnd; I++)
10551 VAddrs.push_back(Elt: Op.getOperand(i: I));
10552 }
10553
10554 // If the register allocator cannot place the address registers contiguously
10555 // without introducing moves, then using the non-sequential address encoding
10556 // is always preferable, since it saves VALU instructions and is usually a
10557 // wash in terms of code size or even better.
10558 //
10559 // However, we currently have no way of hinting to the register allocator that
10560 // MIMG addresses should be placed contiguously when it is possible to do so,
10561 // so force non-NSA for the common 2-address case as a heuristic.
10562 //
10563 // SIShrinkInstructions will convert NSA encodings to non-NSA after register
10564 // allocation when possible.
10565 //
10566 // Partial NSA is allowed on GFX11+ where the final register is a contiguous
10567 // set of the remaining addresses.
10568 const unsigned NSAMaxSize = ST->getNSAMaxSize(HasSampler: BaseOpcode->Sampler);
10569 const bool HasPartialNSAEncoding = ST->hasPartialNSAEncoding();
10570 const bool UseNSA = ST->hasNSAEncoding() &&
10571 VAddrs.size() >= ST->getNSAThreshold(MF) &&
10572 (VAddrs.size() <= NSAMaxSize || HasPartialNSAEncoding);
10573 const bool UsePartialNSA =
10574 UseNSA && HasPartialNSAEncoding && VAddrs.size() > NSAMaxSize;
10575
10576 SDValue VAddr;
10577 if (UsePartialNSA) {
10578 VAddr = getBuildDwordsVector(DAG, DL,
10579 Elts: ArrayRef(VAddrs).drop_front(N: NSAMaxSize - 1));
10580 } else if (!UseNSA) {
10581 VAddr = getBuildDwordsVector(DAG, DL, Elts: VAddrs);
10582 }
10583
10584 SDValue True = DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1);
10585 SDValue False = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1);
10586 SDValue Unorm;
10587 if (!BaseOpcode->Sampler) {
10588 Unorm = True;
10589 } else {
10590 uint64_t UnormConst =
10591 Op.getConstantOperandVal(i: ArgOffset + Intr->UnormIndex);
10592
10593 Unorm = UnormConst ? True : False;
10594 }
10595
10596 SDValue TFE;
10597 SDValue LWE;
10598 SDValue TexFail = Op.getOperand(i: ArgOffset + Intr->TexFailCtrlIndex);
10599 bool IsTexFail = false;
10600 if (!parseTexFail(TexFailCtrl: TexFail, DAG, TFE: &TFE, LWE: &LWE, IsTexFail))
10601 return Op;
10602
10603 if (IsTexFail) {
10604 if (!DMaskLanes) {
10605 // Expecting to get an error flag since TFC is on - and dmask is 0
10606 // Force dmask to be at least 1 otherwise the instruction will fail
10607 DMask = 0x1;
10608 DMaskLanes = 1;
10609 NumVDataDwords = 1;
10610 }
10611 NumVDataDwords += 1;
10612 AdjustRetType = true;
10613 }
10614
10615 // Has something earlier tagged that the return type needs adjusting
10616 // This happens if the instruction is a load or has set TexFailCtrl flags
10617 if (AdjustRetType) {
10618 // NumVDataDwords reflects the true number of dwords required in the return
10619 // type
10620 if (DMaskLanes == 0 && !BaseOpcode->Store) {
10621 // This is a no-op load. This can be eliminated
10622 SDValue Undef = DAG.getPOISON(VT: Op.getValueType());
10623 if (isa<MemSDNode>(Val: Op))
10624 return DAG.getMergeValues(Ops: {Undef, Op.getOperand(i: 0)}, dl: DL);
10625 return Undef;
10626 }
10627
10628 EVT NewVT = NumVDataDwords > 1 ? EVT::getVectorVT(Context&: *DAG.getContext(),
10629 VT: MVT::i32, NumElements: NumVDataDwords)
10630 : MVT::i32;
10631
10632 ResultTypes[0] = NewVT;
10633 if (ResultTypes.size() == 3) {
10634 // Original result was aggregate type used for TexFailCtrl results
10635 // The actual instruction returns as a vector type which has now been
10636 // created. Remove the aggregate result.
10637 ResultTypes.erase(CI: &ResultTypes[1]);
10638 }
10639 }
10640
10641 unsigned CPol = Op.getConstantOperandVal(i: ArgOffset + Intr->CachePolicyIndex);
10642 // Keep GLC only when the atomic's result is actually used.
10643 if (BaseOpcode->Atomic && !BaseOpcode->NoReturn)
10644 CPol |= AMDGPU::CPol::GLC;
10645 if (CPol & ~((IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12) |
10646 AMDGPU::CPol::VOLATILE))
10647 return Op;
10648
10649 SmallVector<SDValue, 26> Ops;
10650 if (BaseOpcode->Store || BaseOpcode->Atomic)
10651 Ops.push_back(Elt: VData); // vdata
10652 if (UsePartialNSA) {
10653 append_range(C&: Ops, R: ArrayRef(VAddrs).take_front(N: NSAMaxSize - 1));
10654 Ops.push_back(Elt: VAddr);
10655 } else if (UseNSA)
10656 append_range(C&: Ops, R&: VAddrs);
10657 else
10658 Ops.push_back(Elt: VAddr);
10659 SDValue Rsrc = Op.getOperand(i: ArgOffset + Intr->RsrcIndex);
10660 EVT RsrcVT = Rsrc.getValueType();
10661 if (RsrcVT != MVT::v4i32 && RsrcVT != MVT::v8i32)
10662 return Op;
10663 Ops.push_back(Elt: Rsrc);
10664 if (BaseOpcode->Sampler) {
10665 SDValue Samp = Op.getOperand(i: ArgOffset + Intr->SampIndex);
10666 if (Samp.getValueType() != MVT::v4i32)
10667 return Op;
10668 Ops.push_back(Elt: Samp);
10669 }
10670 Ops.push_back(Elt: DAG.getTargetConstant(Val: DMask, DL, VT: MVT::i32));
10671 if (IsGFX10Plus)
10672 Ops.push_back(Elt: DAG.getTargetConstant(Val: DimInfo->Encoding, DL, VT: MVT::i32));
10673 if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
10674 Ops.push_back(Elt: Unorm);
10675 Ops.push_back(Elt: DAG.getTargetConstant(Val: CPol, DL, VT: MVT::i32));
10676 Ops.push_back(Elt: IsA16 && // r128, a16 for gfx9
10677 ST->hasFeature(Feature: AMDGPU::FeatureR128A16)
10678 ? True
10679 : False);
10680 if (IsGFX10Plus)
10681 Ops.push_back(Elt: IsA16 ? True : False);
10682
10683 if (!Subtarget->hasGFX90AInsts())
10684 Ops.push_back(Elt: TFE); // tfe
10685 else if (TFE->getAsZExtVal()) {
10686 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10687 DAG.getMachineFunction().getFunction(),
10688 "TFE is not supported on this GPU", DL.getDebugLoc()));
10689 }
10690
10691 if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
10692 Ops.push_back(Elt: LWE); // lwe
10693 if (!IsGFX10Plus)
10694 Ops.push_back(Elt: DimInfo->DA ? True : False);
10695 if (BaseOpcode->HasD16)
10696 Ops.push_back(Elt: IsD16 ? True : False);
10697 if (isa<MemSDNode>(Val: Op))
10698 Ops.push_back(Elt: Op.getOperand(i: 0)); // chain
10699
10700 int NumVAddrDwords =
10701 UseNSA ? VAddrs.size() : VAddr.getValueType().getSizeInBits() / 32;
10702 int Opcode = -1;
10703
10704 if (IsGFX13) {
10705 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx13,
10706 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10707 } else if (IsGFX12Plus) {
10708 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx12,
10709 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10710 } else if (IsGFX11Plus) {
10711 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
10712 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx11NSA
10713 : AMDGPU::MIMGEncGfx11Default,
10714 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10715 } else if (IsGFX10Plus) {
10716 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
10717 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx10NSA
10718 : AMDGPU::MIMGEncGfx10Default,
10719 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10720 } else {
10721 if (Subtarget->hasGFX90AInsts()) {
10722 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx90a,
10723 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10724 if (Opcode == -1) {
10725 return diagnoseUnsupportedImage(
10726 DAG, Op, ResultTypes: OrigResultTypes, DL,
10727 Msg: "requested image instruction is not supported on this GPU");
10728 }
10729 }
10730 if (Opcode == -1 &&
10731 Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
10732 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx8,
10733 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10734 if (Opcode == -1)
10735 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx6,
10736 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10737 }
10738 if (Opcode == -1)
10739 return Op;
10740
10741 MachineSDNode *NewNode = DAG.getMachineNode(Opcode, dl: DL, ResultTys: ResultTypes, Ops);
10742 if (auto *MemOp = dyn_cast<MemSDNode>(Val&: Op)) {
10743 MachineMemOperand *MemRef = MemOp->getMemOperand();
10744 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
10745 }
10746
10747 if (BaseOpcode->NoReturn) {
10748 if (BaseOpcode->Atomic)
10749 return DAG.getMergeValues(
10750 Ops: {DAG.getPOISON(VT: OrigResultTypes[0]), SDValue(NewNode, 0)}, dl: DL);
10751
10752 return SDValue(NewNode, 0);
10753 }
10754
10755 if (BaseOpcode->AtomicX2) {
10756 SmallVector<SDValue, 1> Elt;
10757 DAG.ExtractVectorElements(Op: SDValue(NewNode, 0), Args&: Elt, Start: 0, Count: 1);
10758 return DAG.getMergeValues(Ops: {Elt[0], SDValue(NewNode, 1)}, dl: DL);
10759 }
10760
10761 return constructRetValue(DAG, Result: NewNode, ResultTypes: OrigResultTypes, IsTexFail,
10762 Unpacked: Subtarget->hasUnpackedD16VMem(), IsD16, DMaskPop: DMaskLanes,
10763 NumVDataDwords, IsAtomicPacked16Bit, DL);
10764}
10765
10766SDValue SITargetLowering::lowerSBuffer(EVT VT, EVT MemVT, SDLoc DL,
10767 SDValue Chain, SDValue Rsrc,
10768 SDValue Offset, SDValue CachePolicy,
10769 SelectionDAG &DAG,
10770 MachineMemOperand *MMO) const {
10771 MachineFunction &MF = DAG.getMachineFunction();
10772 bool HasChainResult = MMO != nullptr;
10773
10774 if (!HasChainResult) {
10775 const DataLayout &DataLayout = DAG.getDataLayout();
10776 Align Alignment =
10777 DataLayout.getABITypeAlign(Ty: MemVT.getTypeForEVT(Context&: *DAG.getContext()));
10778
10779 MMO = MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
10780 F: MachineMemOperand::MOLoad |
10781 MachineMemOperand::MODereferenceable |
10782 MachineMemOperand::MOInvariant,
10783 Size: MemVT.getStoreSize(), BaseAlignment: Alignment);
10784 }
10785
10786 if (!Offset->isDivergent()) {
10787 SDValue Ops[] = {Chain, Rsrc, Offset, CachePolicy};
10788
10789 // Lower llvm.amdgcn.*s.buffer.load.{i,u}N intrinsics. First, generate
10790 // s_buffer_load_u* for signed and unsigned load instructions. Next, DAG
10791 // combiner tries to merge the s_buffer_load_uN with a sext instruction
10792 // (performSignExtendInRegCombine()) and it replaces s_buffer_load_uN with
10793 // s_buffer_load_iN.
10794 auto HandleScalarSubwordLoads = [&](unsigned Opcode) -> SDValue {
10795 SDValue BufferLoad = DAG.getMemIntrinsicNode(
10796 Opcode, dl: DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops, MemVT, MMO);
10797 SDValue LoadVal = DAG.getAnyExtOrTrunc(
10798 Op: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MemVT, Operand: BufferLoad), DL, VT);
10799 if (HasChainResult)
10800 return DAG.getMergeValues(Ops: {LoadVal, BufferLoad.getValue(R: 1)}, dl: DL);
10801 return LoadVal;
10802 };
10803 if (MemVT == MVT::i8 && Subtarget->hasScalarSubwordLoads())
10804 return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_UBYTE);
10805
10806 if (MemVT == MVT::i16 && Subtarget->hasScalarSubwordLoads())
10807 return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_USHORT);
10808
10809 // Widen vec3 load to vec4.
10810 if (VT.isVector() && VT.getVectorNumElements() == 3 &&
10811 !Subtarget->hasScalarDwordx3Loads()) {
10812 EVT WidenedVT =
10813 EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(), NumElements: 4);
10814 auto WidenedOp = DAG.getMemIntrinsicNode(
10815 Opcode: AMDGPUISD::SBUFFER_LOAD, dl: DL, VTList: DAG.getVTList(VT1: WidenedVT, VT2: MVT::Other),
10816 Ops, MemVT: WidenedVT,
10817 MMO: MF.getMachineMemOperand(MMO, Offset: 0, Size: WidenedVT.getStoreSize()));
10818 auto Subvector = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: WidenedOp,
10819 N2: DAG.getVectorIdxConstant(Val: 0, DL));
10820 if (HasChainResult)
10821 return DAG.getMergeValues(Ops: {Subvector, WidenedOp.getValue(R: 1)}, dl: DL);
10822 return Subvector;
10823 }
10824
10825 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::SBUFFER_LOAD, dl: DL,
10826 VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), Ops, MemVT,
10827 MMO);
10828 }
10829
10830 // We have a divergent offset. Emit a MUBUF buffer load instead. We can
10831 // assume that the buffer is unswizzled.
10832 SDValue Ops[] = {
10833 Chain, // Chain
10834 Rsrc, // rsrc
10835 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
10836 {}, // voffset
10837 {}, // soffset
10838 {}, // offset
10839 CachePolicy, // cachepolicy
10840 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
10841 };
10842 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
10843 Subtarget->hasScalarSubwordLoads()) {
10844 setBufferOffsets(CombinedOffset: Offset, DAG, Offsets: &Ops[3], Alignment: Align(4));
10845 SDValue Load = handleByteShortBufferLoads(DAG, LoadVT: MemVT, DL, Ops, MMO);
10846 SDValue LoadVal = DAG.getAnyExtOrTrunc(Op: Load.getOperand(i: 0), DL, VT);
10847 if (HasChainResult)
10848 return DAG.getMergeValues(Ops: {LoadVal, Load.getOperand(i: 1)}, dl: DL);
10849 return LoadVal;
10850 }
10851
10852 SmallVector<SDValue, 4> Loads;
10853 unsigned NumLoads = 1;
10854 MVT LoadVT = VT.getSimpleVT();
10855 unsigned NumElts = LoadVT.isVector() ? LoadVT.getVectorNumElements() : 1;
10856 assert((LoadVT.getScalarType() == MVT::i32 ||
10857 LoadVT.getScalarType() == MVT::f32));
10858
10859 if (NumElts == 8 || NumElts == 16) {
10860 NumLoads = NumElts / 4;
10861 LoadVT = MVT::getVectorVT(VT: LoadVT.getScalarType(), NumElements: 4);
10862 }
10863
10864 SDVTList VTList = DAG.getVTList(VTs: {LoadVT, MVT::Other});
10865
10866 // Use the alignment to ensure that the required offsets will fit into the
10867 // immediate offsets.
10868 setBufferOffsets(CombinedOffset: Offset, DAG, Offsets: &Ops[3],
10869 Alignment: NumLoads > 1 ? Align(16 * NumLoads) : Align(4));
10870
10871 uint64_t InstOffset = Ops[5]->getAsZExtVal();
10872 unsigned LoadSize = LoadVT.getStoreSize();
10873 for (unsigned i = 0; i < NumLoads; ++i) {
10874 Ops[5] = DAG.getTargetConstant(Val: InstOffset + 16 * i, DL, VT: MVT::i32);
10875 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(MMO, Offset: 16 * i, Size: LoadSize);
10876 Loads.push_back(Elt: getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_LOAD, DL, VTList, Ops,
10877 MemVT: LoadVT, MMO: LoadMMO, DAG));
10878 }
10879
10880 if (NumElts == 8 || NumElts == 16) {
10881 SDValue LoadVal = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: Loads);
10882 if (HasChainResult) {
10883 SmallVector<SDValue, 4> LoadChains;
10884 for (SDValue Load : Loads)
10885 LoadChains.push_back(Elt: Load.getValue(R: 1));
10886 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: LoadChains);
10887 return DAG.getMergeValues(Ops: {LoadVal, Chain}, dl: DL);
10888 }
10889 return LoadVal;
10890 }
10891
10892 return Loads[0];
10893}
10894
10895SDValue SITargetLowering::lowerWaveID(SelectionDAG &DAG, SDValue Op) const {
10896 // With architected SGPRs, waveIDinGroup is in TTMP8[29:25].
10897 if (!Subtarget->hasArchitectedSGPRs())
10898 return {};
10899 SDLoc SL(Op);
10900 MVT VT = MVT::i32;
10901 SDValue TTMP8 = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: AMDGPU::TTMP8, VT);
10902 return DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT, N1: TTMP8,
10903 N2: DAG.getConstant(Val: 25, DL: SL, VT), N3: DAG.getConstant(Val: 5, DL: SL, VT));
10904}
10905
10906SDValue SITargetLowering::lowerConstHwRegRead(SelectionDAG &DAG, SDValue Op,
10907 AMDGPU::Hwreg::Id HwReg,
10908 unsigned LowBit,
10909 unsigned Width) const {
10910 SDLoc SL(Op);
10911 using namespace AMDGPU::Hwreg;
10912 return {DAG.getMachineNode(
10913 Opcode: AMDGPU::S_GETREG_B32_const, dl: SL, VT: MVT::i32,
10914 Op1: DAG.getTargetConstant(Val: HwregEncoding::encode(Values: HwReg, Values: LowBit, Values: Width),
10915 DL: SL, VT: MVT::i32)),
10916 0};
10917}
10918
10919SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
10920 unsigned Dim,
10921 const ArgDescriptor &Arg) const {
10922 SDLoc SL(Op);
10923 MachineFunction &MF = DAG.getMachineFunction();
10924 unsigned MaxID = Subtarget->getMaxWorkitemID(Kernel: MF.getFunction(), Dimension: Dim);
10925 if (MaxID == 0)
10926 return DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
10927
10928 // It's undefined behavior if a function marked with the amdgpu-no-*
10929 // attributes uses the corresponding intrinsic.
10930 if (!Arg)
10931 return DAG.getPOISON(VT: Op->getValueType(ResNo: 0));
10932
10933 SDValue Val = loadInputValue(DAG, RC: &AMDGPU::VGPR_32RegClass, VT: MVT::i32,
10934 SL: SDLoc(DAG.getEntryNode()), Arg);
10935
10936 // Don't bother inserting AssertZext for packed IDs since we're emitting the
10937 // masking operations anyway.
10938 //
10939 // TODO: We could assert the top bit is 0 for the source copy.
10940 if (Arg.isMasked())
10941 return Val;
10942
10943 // Preserve the known bits after expansion to a copy.
10944 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: llvm::bit_width(Value: MaxID));
10945 return DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: MVT::i32, N1: Val,
10946 N2: DAG.getValueType(SmallVT));
10947}
10948
10949SDValue SITargetLowering::lowerFromFP8(SDValue Op, bool IsBF8,
10950 SelectionDAG &DAG) const {
10951 SDLoc SL(Op);
10952 SDValue Src = Op.getOperand(i: 0);
10953 EVT DstVT = Op.getValueType();
10954 bool IsF16 = DstVT.getVectorElementType() == MVT::f16;
10955 assert((!IsF16 || Subtarget->hasFP8F16ConversionInsts()) &&
10956 "fp8/bf8 -> f16 conversion requires FP8F16ConversionInsts");
10957
10958 unsigned Opc;
10959 if (IsF16)
10960 Opc = IsBF8 ? AMDGPUISD::CVT_PK_F16_BF8 : AMDGPUISD::CVT_PK_F16_FP8;
10961 else
10962 Opc = IsBF8 ? AMDGPUISD::CVT_PK_F32_BF8 : AMDGPUISD::CVT_PK_F32_FP8;
10963
10964 // Pack the two i8 lanes into the integer type the packed HW node reads. The
10965 // f16 form takes i16 and the f32 form takes i32. v2i8 bitcasts to i16
10966 // directly and the f32 node reads the low half of an any-extended i32.
10967 EVT PackedVT =
10968 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DstVT.getScalarSizeInBits());
10969 SDValue AsI16 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Src);
10970 SDValue Packed = DAG.getAnyExtOrTrunc(Op: AsI16, DL: SL, VT: PackedVT);
10971 return DAG.getNode(Opcode: Opc, DL: SL, VT: DstVT, Operand: Packed);
10972}
10973
10974SDValue
10975SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
10976 SelectionDAG &DAG) const {
10977 // Handle the OCP FP8 formats (E4M3FN, E5M2) and unsigned E5M3 on subtargets
10978 // with matching HW conversions. Other formats use the generic expansion.
10979 APFloatBase::Semantics FPSemantic =
10980 static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(i: 1));
10981 const bool IsFP8 = FPSemantic == APFloatBase::S_Float8E4M3FN;
10982 const bool IsBF8 = FPSemantic == APFloatBase::S_Float8E5M2;
10983 const bool IsE5M3 = FPSemantic == APFloatBase::S_Float8E5M3FNU;
10984 const bool HasE5M3ConversionInsts =
10985 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
10986 const bool IsSupported = IsFP8 || IsBF8 || (IsE5M3 && HasE5M3ConversionInsts);
10987 if (!IsSupported)
10988 return SDValue();
10989
10990 EVT DstVT = Op.getValueType();
10991 if (IsE5M3) {
10992 if (DstVT.getScalarType() != MVT::f32)
10993 return SDValue();
10994
10995 SDLoc SL(Op);
10996 SDValue Src = Op.getOperand(i: 0);
10997 assert((!DstVT.isVector() || DstVT == MVT::v2f32) &&
10998 "only the v2f32 vector result is custom lowered");
10999
11000 if (DstVT.isVector())
11001 Src = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Src);
11002 Src = DAG.getAnyExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32);
11003
11004 auto ConvertByte = [&](unsigned ByteSel) {
11005 return DAG.getNode(Opcode: AMDGPUISD::CVT_F32_FP8_E5M3, DL: SL, VT: MVT::f32, N1: Src,
11006 N2: DAG.getTargetConstant(Val: ByteSel, DL: SL, VT: MVT::i32));
11007 };
11008
11009 if (!DstVT.isVector())
11010 return ConvertByte(0);
11011 return DAG.getBuildVector(VT: DstVT, DL: SL, Ops: {ConvertByte(0), ConvertByte(1)});
11012 }
11013
11014 if (!DstVT.isVector()) {
11015 SDValue Src = Op.getOperand(i: 0);
11016 if (Src.getValueType() != MVT::i32) {
11017 SDLoc SL(Op);
11018 SDValue SrcI32 = DAG.getAnyExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32);
11019 return DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL: SL, VT: DstVT, N1: SrcI32,
11020 N2: Op.getOperand(i: 1));
11021 }
11022 return Op;
11023 }
11024
11025 EVT EltVT = DstVT.getVectorElementType();
11026 if (EltVT == MVT::f16 || EltVT == MVT::f32)
11027 return lowerFromFP8(Op, IsBF8, DAG);
11028 return SDValue();
11029}
11030
11031SDValue SITargetLowering::lowerToFP8(SDValue Op, bool IsBF8, bool IsE5M3,
11032 SelectionDAG &DAG) const {
11033 SDLoc SL(Op);
11034 SDValue Src = Op.getOperand(i: 0);
11035 EVT ResVT = Op.getValueType();
11036 bool IsF16 = Src.getValueType().getScalarType() == MVT::f16;
11037 assert((!IsF16 || Subtarget->hasF16FP8ConversionInsts()) &&
11038 "f16 -> fp8/bf8 conversion requires F16FP8ConversionInsts");
11039 assert((!ResVT.isVector() || ResVT == MVT::v2i8) &&
11040 "only the v2i8 vector result is custom lowered");
11041
11042 if (IsF16) {
11043 unsigned Opc =
11044 IsBF8 ? AMDGPUISD::CVT_PK_BF8_F16 : AMDGPUISD::CVT_PK_FP8_F16;
11045 SDValue Bytes = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i16, Operand: Src);
11046 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ResVT, Operand: Bytes);
11047 }
11048
11049 unsigned Opc = IsBF8 ? AMDGPUISD::CVT_PK_BF8_F32
11050 : IsE5M3 ? AMDGPUISD::CVT_PK_FP8_F32_E5M3
11051 : AMDGPUISD::CVT_PK_FP8_F32;
11052 SDValue PoisonI32 = DAG.getPOISON(VT: MVT::i32);
11053 SDValue WordSel = DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i1);
11054
11055 if (!ResVT.isVector()) {
11056 // Convert one lane, the second is unused. Feed it the same source so the
11057 // instruction does not read an undefined register.
11058 SDValue Packed =
11059 DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: Src, N2: Src, N3: PoisonI32, N4: WordSel);
11060 return DAG.getAnyExtOrTrunc(Op: Packed, DL: SL, VT: ResVT);
11061 }
11062
11063 SDValue A = DAG.getExtractVectorElt(DL: SL, VT: MVT::f32, Vec: Src, Idx: 0);
11064 SDValue B = DAG.getExtractVectorElt(DL: SL, VT: MVT::f32, Vec: Src, Idx: 1);
11065 SDValue Packed = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: A, N2: B, N3: PoisonI32, N4: WordSel);
11066 SDValue Bytes = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i16, Operand: Packed);
11067 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ResVT, Operand: Bytes);
11068}
11069
11070SDValue
11071SITargetLowering::LowerCONVERT_TO_ARBITRARY_FP(SDValue Op,
11072 SelectionDAG &DAG) const {
11073 // The OCP FP8 formats (E4M3FN, E5M2) and unsigned E5M3 map to HW conversions
11074 // on subtargets that support them. Everything else uses generic expansion.
11075 APFloatBase::Semantics Sem =
11076 static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(i: 1));
11077 const bool IsFP8 = Sem == APFloatBase::S_Float8E4M3FN;
11078 const bool IsBF8 = Sem == APFloatBase::S_Float8E5M2;
11079 const bool IsE5M3 = Sem == APFloatBase::S_Float8E5M3FNU;
11080 const bool HasE5M3ConversionInsts =
11081 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
11082 const bool IsSupported = IsFP8 || IsBF8 || (IsE5M3 && HasE5M3ConversionInsts);
11083 if (!IsSupported)
11084 return SDValue();
11085
11086 // The HW conversions only support nearest-even. The OCP conversions do not
11087 // saturate. The unsigned E5M3 conversion always clamps out-of-range inputs,
11088 // which also refines the non-saturating form where those inputs are poison.
11089 if (static_cast<RoundingMode>(Op.getConstantOperandVal(i: 2)) !=
11090 RoundingMode::NearestTiesToEven)
11091 return SDValue();
11092 if (!IsE5M3 && Op.getConstantOperandVal(i: 3) != 0)
11093 return SDValue();
11094
11095 EVT SrcEltVT = Op.getOperand(i: 0).getValueType().getScalarType();
11096 // The f32 form is built here rather than by a tablegen pattern because the
11097 // HW result is i32 while the node result is i16 after the i8 promotion.
11098 if (SrcEltVT == MVT::f32)
11099 return lowerToFP8(Op, IsBF8, IsE5M3, DAG);
11100 if (!IsE5M3 && SrcEltVT == MVT::f16 &&
11101 Subtarget->hasF16FP8ConversionInsts()) {
11102 // A scalar conversion is selected from the generic node by tablegen, only
11103 // the illegal v2i8 result type needs lowering here.
11104 if (!Op.getValueType().isVector())
11105 return Op;
11106 return lowerToFP8(Op, IsBF8, IsE5M3: false, DAG);
11107 }
11108 return SDValue();
11109}
11110
11111SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
11112 SelectionDAG &DAG) const {
11113 MachineFunction &MF = DAG.getMachineFunction();
11114 auto *MFI = MF.getInfo<SIMachineFunctionInfo>();
11115
11116 EVT VT = Op.getValueType();
11117 SDLoc DL(Op);
11118 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
11119
11120 // TODO: Should this propagate fast-math-flags?
11121
11122 switch (IntrinsicID) {
11123 case Intrinsic::amdgcn_wave_reduce_min:
11124 case Intrinsic::amdgcn_wave_reduce_umin:
11125 case Intrinsic::amdgcn_wave_reduce_fmin:
11126 case Intrinsic::amdgcn_wave_reduce_max:
11127 case Intrinsic::amdgcn_wave_reduce_umax:
11128 case Intrinsic::amdgcn_wave_reduce_fmax:
11129 case Intrinsic::amdgcn_wave_reduce_add:
11130 case Intrinsic::amdgcn_wave_reduce_fadd:
11131 case Intrinsic::amdgcn_wave_reduce_sub:
11132 case Intrinsic::amdgcn_wave_reduce_fsub:
11133 case Intrinsic::amdgcn_wave_reduce_and:
11134 case Intrinsic::amdgcn_wave_reduce_or:
11135 case Intrinsic::amdgcn_wave_reduce_xor: {
11136 EVT SrcVT = Op.getOperand(i: 1).getValueType();
11137 if (SrcVT.getFixedSizeInBits() == 16) {
11138 bool IsFPOp = SrcVT.isFloatingPoint();
11139 bool NeedsSignExt = IntrinsicID == Intrinsic::amdgcn_wave_reduce_min ||
11140 IntrinsicID == Intrinsic::amdgcn_wave_reduce_max ||
11141 IntrinsicID == Intrinsic::amdgcn_wave_reduce_add ||
11142 IntrinsicID == Intrinsic::amdgcn_wave_reduce_sub;
11143 unsigned ExtOpc = IsFPOp ? ISD::FP_EXTEND
11144 : NeedsSignExt ? ISD::SIGN_EXTEND
11145 : ISD::ZERO_EXTEND;
11146 auto SrcType = IsFPOp ? MVT::f16 : MVT::i16;
11147 auto ExtType = IsFPOp ? MVT::f32 : MVT::i32;
11148 SDValue ExtendedSrc = DAG.getNode(Opcode: ExtOpc, DL, VT: ExtType, Operand: Op.getOperand(i: 1));
11149 SDValue Strategy = Op.getOperand(i: 2);
11150 SDValue Result = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: ExtType,
11151 N1: Op.getOperand(i: 0), N2: ExtendedSrc, N3: Strategy);
11152 if (IsFPOp)
11153 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: SrcType, N1: Result,
11154 N2: DAG.getTargetConstant(Val: 1, DL, VT: MVT::i32));
11155 else
11156 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: SrcType, Operand: Result);
11157 }
11158 return SDValue();
11159 }
11160 case Intrinsic::amdgcn_implicit_buffer_ptr: {
11161 if (getSubtarget()->isAmdHsaOrMesa(F: MF.getFunction()))
11162 return emitNonHSAIntrinsicError(DAG, DL, VT);
11163 return getPreloadedValue(DAG, MFI: *MFI, VT,
11164 PVID: AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR);
11165 }
11166 case Intrinsic::amdgcn_dispatch_ptr:
11167 case Intrinsic::amdgcn_queue_ptr: {
11168 if (!Subtarget->isAmdHsaOrMesa(F: MF.getFunction())) {
11169 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11170 MF.getFunction(), "unsupported hsa intrinsic without hsa target",
11171 DL.getDebugLoc()));
11172 return DAG.getPOISON(VT);
11173 }
11174
11175 auto RegID = IntrinsicID == Intrinsic::amdgcn_dispatch_ptr
11176 ? AMDGPUFunctionArgInfo::DISPATCH_PTR
11177 : AMDGPUFunctionArgInfo::QUEUE_PTR;
11178 return getPreloadedValue(DAG, MFI: *MFI, VT, PVID: RegID);
11179 }
11180 case Intrinsic::amdgcn_implicitarg_ptr: {
11181 if (MFI->isEntryFunction())
11182 return getImplicitArgPtr(DAG, SL: DL);
11183 return getPreloadedValue(DAG, MFI: *MFI, VT,
11184 PVID: AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
11185 }
11186 case Intrinsic::amdgcn_kernarg_segment_ptr: {
11187 if (!AMDGPU::isKernel(F: MF.getFunction())) {
11188 // This only makes sense to call in a kernel, so just lower to null.
11189 return DAG.getConstant(Val: 0, DL, VT);
11190 }
11191
11192 return getPreloadedValue(DAG, MFI: *MFI, VT,
11193 PVID: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
11194 }
11195 case Intrinsic::amdgcn_dispatch_id: {
11196 return getPreloadedValue(DAG, MFI: *MFI, VT, PVID: AMDGPUFunctionArgInfo::DISPATCH_ID);
11197 }
11198 case Intrinsic::amdgcn_rcp:
11199 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT, Operand: Op.getOperand(i: 1));
11200 case Intrinsic::amdgcn_rsq:
11201 return DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: Op.getOperand(i: 1));
11202 case Intrinsic::amdgcn_rsq_legacy:
11203 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
11204 return emitRemovedIntrinsicError(DAG, DL, VT);
11205 return SDValue();
11206 case Intrinsic::amdgcn_rcp_legacy:
11207 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
11208 return emitRemovedIntrinsicError(DAG, DL, VT);
11209 return DAG.getNode(Opcode: AMDGPUISD::RCP_LEGACY, DL, VT, Operand: Op.getOperand(i: 1));
11210 case Intrinsic::amdgcn_fma_legacy:
11211 case Intrinsic::amdgcn_sudot4:
11212 case Intrinsic::amdgcn_sudot8:
11213 case Intrinsic::amdgcn_tanh:
11214 return SDValue();
11215 case Intrinsic::amdgcn_rsq_clamp: {
11216 if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
11217 return DAG.getNode(Opcode: AMDGPUISD::RSQ_CLAMP, DL, VT, Operand: Op.getOperand(i: 1));
11218
11219 Type *Type = VT.getTypeForEVT(Context&: *DAG.getContext());
11220 APFloat Max = APFloat::getLargest(Sem: Type->getFltSemantics());
11221 APFloat Min = APFloat::getLargest(Sem: Type->getFltSemantics(), Negative: true);
11222
11223 SDValue Rsq = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: Op.getOperand(i: 1));
11224 SDValue Tmp =
11225 DAG.getNode(Opcode: ISD::FMINNUM, DL, VT, N1: Rsq, N2: DAG.getConstantFP(Val: Max, DL, VT));
11226 return DAG.getNode(Opcode: ISD::FMAXNUM, DL, VT, N1: Tmp,
11227 N2: DAG.getConstantFP(Val: Min, DL, VT));
11228 }
11229 case Intrinsic::r600_read_ngroups_x:
11230 if (Subtarget->isAmdHsaOS())
11231 return emitNonHSAIntrinsicError(DAG, DL, VT);
11232
11233 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
11234 Offset: SI::KernelInputOffsets::NGROUPS_X, Alignment: Align(4),
11235 Signed: false);
11236 case Intrinsic::r600_read_ngroups_y:
11237 if (Subtarget->isAmdHsaOS())
11238 return emitNonHSAIntrinsicError(DAG, DL, VT);
11239
11240 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
11241 Offset: SI::KernelInputOffsets::NGROUPS_Y, Alignment: Align(4),
11242 Signed: false);
11243 case Intrinsic::r600_read_ngroups_z:
11244 if (Subtarget->isAmdHsaOS())
11245 return emitNonHSAIntrinsicError(DAG, DL, VT);
11246
11247 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
11248 Offset: SI::KernelInputOffsets::NGROUPS_Z, Alignment: Align(4),
11249 Signed: false);
11250 case Intrinsic::r600_read_local_size_x:
11251 if (Subtarget->isAmdHsaOS())
11252 return emitNonHSAIntrinsicError(DAG, DL, VT);
11253
11254 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
11255 Offset: SI::KernelInputOffsets::LOCAL_SIZE_X);
11256 case Intrinsic::r600_read_local_size_y:
11257 if (Subtarget->isAmdHsaOS())
11258 return emitNonHSAIntrinsicError(DAG, DL, VT);
11259
11260 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
11261 Offset: SI::KernelInputOffsets::LOCAL_SIZE_Y);
11262 case Intrinsic::r600_read_local_size_z:
11263 if (Subtarget->isAmdHsaOS())
11264 return emitNonHSAIntrinsicError(DAG, DL, VT);
11265
11266 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
11267 Offset: SI::KernelInputOffsets::LOCAL_SIZE_Z);
11268 case Intrinsic::amdgcn_workgroup_id_x:
11269 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
11270 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_X,
11271 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X,
11272 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X);
11273 case Intrinsic::amdgcn_workgroup_id_y:
11274 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
11275 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,
11276 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y,
11277 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y);
11278 case Intrinsic::amdgcn_workgroup_id_z:
11279 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
11280 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,
11281 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z,
11282 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z);
11283 case Intrinsic::amdgcn_cluster_id_x:
11284 return Subtarget->hasClusters()
11285 ? getPreloadedValue(DAG, MFI: *MFI, VT,
11286 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_X)
11287 : DAG.getPOISON(VT);
11288 case Intrinsic::amdgcn_cluster_id_y:
11289 return Subtarget->hasClusters()
11290 ? getPreloadedValue(DAG, MFI: *MFI, VT,
11291 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_Y)
11292 : DAG.getPOISON(VT);
11293 case Intrinsic::amdgcn_cluster_id_z:
11294 return Subtarget->hasClusters()
11295 ? getPreloadedValue(DAG, MFI: *MFI, VT,
11296 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_Z)
11297 : DAG.getPOISON(VT);
11298 case Intrinsic::amdgcn_cluster_workgroup_id_x:
11299 return Subtarget->hasClusters()
11300 ? getPreloadedValue(
11301 DAG, MFI: *MFI, VT,
11302 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X)
11303 : DAG.getPOISON(VT);
11304 case Intrinsic::amdgcn_cluster_workgroup_id_y:
11305 return Subtarget->hasClusters()
11306 ? getPreloadedValue(
11307 DAG, MFI: *MFI, VT,
11308 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y)
11309 : DAG.getPOISON(VT);
11310 case Intrinsic::amdgcn_cluster_workgroup_id_z:
11311 return Subtarget->hasClusters()
11312 ? getPreloadedValue(
11313 DAG, MFI: *MFI, VT,
11314 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z)
11315 : DAG.getPOISON(VT);
11316 case Intrinsic::amdgcn_cluster_workgroup_flat_id:
11317 return Subtarget->hasClusters()
11318 ? lowerConstHwRegRead(DAG, Op, HwReg: AMDGPU::Hwreg::ID_IB_STS2, LowBit: 21, Width: 4)
11319 : SDValue();
11320 case Intrinsic::amdgcn_cluster_workgroup_max_id_x:
11321 return Subtarget->hasClusters()
11322 ? getPreloadedValue(
11323 DAG, MFI: *MFI, VT,
11324 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X)
11325 : DAG.getPOISON(VT);
11326 case Intrinsic::amdgcn_cluster_workgroup_max_id_y:
11327 return Subtarget->hasClusters()
11328 ? getPreloadedValue(
11329 DAG, MFI: *MFI, VT,
11330 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y)
11331 : DAG.getPOISON(VT);
11332 case Intrinsic::amdgcn_cluster_workgroup_max_id_z:
11333 return Subtarget->hasClusters()
11334 ? getPreloadedValue(
11335 DAG, MFI: *MFI, VT,
11336 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z)
11337 : DAG.getPOISON(VT);
11338 case Intrinsic::amdgcn_cluster_workgroup_max_flat_id:
11339 return Subtarget->hasClusters()
11340 ? getPreloadedValue(
11341 DAG, MFI: *MFI, VT,
11342 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID)
11343 : DAG.getPOISON(VT);
11344 case Intrinsic::amdgcn_wave_id:
11345 return lowerWaveID(DAG, Op);
11346 case Intrinsic::amdgcn_lds_kernel_id: {
11347 if (MFI->isEntryFunction())
11348 return getLDSKernelId(DAG, SL: DL);
11349 return getPreloadedValue(DAG, MFI: *MFI, VT,
11350 PVID: AMDGPUFunctionArgInfo::LDS_KERNEL_ID);
11351 }
11352 case Intrinsic::amdgcn_workitem_id_x:
11353 return lowerWorkitemID(DAG, Op, Dim: 0, Arg: MFI->getArgInfo().WorkItemIDX);
11354 case Intrinsic::amdgcn_workitem_id_y:
11355 return lowerWorkitemID(DAG, Op, Dim: 1, Arg: MFI->getArgInfo().WorkItemIDY);
11356 case Intrinsic::amdgcn_workitem_id_z:
11357 return lowerWorkitemID(DAG, Op, Dim: 2, Arg: MFI->getArgInfo().WorkItemIDZ);
11358 case Intrinsic::amdgcn_wavefrontsize:
11359 return DAG.getConstant(Val: MF.getSubtarget<GCNSubtarget>().getWavefrontSize(),
11360 DL: SDLoc(Op), VT: MVT::i32);
11361 case Intrinsic::amdgcn_s_buffer_load: {
11362 unsigned CPol = Op.getConstantOperandVal(i: 3);
11363 // s_buffer_load, because of how it's optimized, can't be volatile
11364 // so reject ones with the volatile bit set.
11365 if (CPol & ~((Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12)
11366 ? AMDGPU::CPol::ALL
11367 : AMDGPU::CPol::ALL_pregfx12))
11368 return Op;
11369 return lowerSBuffer(VT, MemVT: VT, DL, Chain: DAG.getEntryNode(), Rsrc: Op.getOperand(i: 1),
11370 Offset: Op.getOperand(i: 2), CachePolicy: Op.getOperand(i: 3), DAG);
11371 }
11372 case Intrinsic::amdgcn_fdiv_fast:
11373 return lowerFDIV_FAST(Op, DAG);
11374 case Intrinsic::amdgcn_sin:
11375 return DAG.getNode(Opcode: AMDGPUISD::SIN_HW, DL, VT, Operand: Op.getOperand(i: 1));
11376
11377 case Intrinsic::amdgcn_cos:
11378 return DAG.getNode(Opcode: AMDGPUISD::COS_HW, DL, VT, Operand: Op.getOperand(i: 1));
11379
11380 case Intrinsic::amdgcn_mul_u24:
11381 return DAG.getNode(Opcode: AMDGPUISD::MUL_U24, DL, VT, N1: Op.getOperand(i: 1),
11382 N2: Op.getOperand(i: 2));
11383 case Intrinsic::amdgcn_mul_i24:
11384 return DAG.getNode(Opcode: AMDGPUISD::MUL_I24, DL, VT, N1: Op.getOperand(i: 1),
11385 N2: Op.getOperand(i: 2));
11386
11387 case Intrinsic::amdgcn_log_clamp: {
11388 if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
11389 return SDValue();
11390
11391 return emitRemovedIntrinsicError(DAG, DL, VT);
11392 }
11393 case Intrinsic::amdgcn_fract:
11394 return DAG.getNode(Opcode: AMDGPUISD::FRACT, DL, VT, Operand: Op.getOperand(i: 1));
11395
11396 case Intrinsic::amdgcn_class: {
11397 SDValue Src = Op.getOperand(i: 1);
11398 EVT SrcVT = Src.getValueType();
11399 bool IsLegal = SrcVT == MVT::f32 || SrcVT == MVT::f64 ||
11400 (SrcVT == MVT::f16 && Subtarget->has16BitInsts());
11401 if (!IsLegal) {
11402 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11403 DAG.getMachineFunction().getFunction(),
11404 "llvm.amdgcn.class only supports f16, f32, and f64",
11405 DL.getDebugLoc()));
11406 return DAG.getPOISON(VT);
11407 }
11408 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT, N1: Src, N2: Op.getOperand(i: 2));
11409 }
11410 case Intrinsic::amdgcn_div_fmas:
11411 return DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL, VT, N1: Op.getOperand(i: 1),
11412 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), N4: Op.getOperand(i: 4));
11413
11414 case Intrinsic::amdgcn_div_fixup:
11415 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL, VT, N1: Op.getOperand(i: 1),
11416 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11417
11418 case Intrinsic::amdgcn_div_scale: {
11419 const ConstantSDNode *Param = cast<ConstantSDNode>(Val: Op.getOperand(i: 3));
11420
11421 // Translate to the operands expected by the machine instruction. The
11422 // first parameter must be the same as the first instruction.
11423 SDValue Numerator = Op.getOperand(i: 1);
11424 SDValue Denominator = Op.getOperand(i: 2);
11425
11426 // Note this order is opposite of the machine instruction's operations,
11427 // which is s0.f = Quotient, s1.f = Denominator, s2.f = Numerator. The
11428 // intrinsic has the numerator as the first operand to match a normal
11429 // division operation.
11430
11431 SDValue Src0 = Param->isAllOnes() ? Numerator : Denominator;
11432
11433 return DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL, VTList: Op->getVTList(), N1: Src0,
11434 N2: Denominator, N3: Numerator);
11435 }
11436 case Intrinsic::amdgcn_ballot:
11437 return lowerBALLOTIntrinsic(TLI: *this, N: Op.getNode(), DAG);
11438 case Intrinsic::amdgcn_fmed3:
11439 return DAG.getNode(Opcode: AMDGPUISD::FMED3, DL, VT, N1: Op.getOperand(i: 1),
11440 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), Flags: Op->getFlags());
11441 case Intrinsic::amdgcn_fdot2:
11442 return DAG.getNode(Opcode: AMDGPUISD::FDOT2, DL, VT, N1: Op.getOperand(i: 1),
11443 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), N4: Op.getOperand(i: 4));
11444 case Intrinsic::amdgcn_fmul_legacy:
11445 return DAG.getNode(Opcode: AMDGPUISD::FMUL_LEGACY, DL, VT, N1: Op.getOperand(i: 1),
11446 N2: Op.getOperand(i: 2));
11447 case Intrinsic::amdgcn_sbfe:
11448 case Intrinsic::amdgcn_ubfe:
11449 return lowerBFEIntrinsic(Op, DAG, IntrinsicID);
11450 case Intrinsic::amdgcn_cvt_pkrtz:
11451 case Intrinsic::amdgcn_cvt_pknorm_i16:
11452 case Intrinsic::amdgcn_cvt_pknorm_u16:
11453 case Intrinsic::amdgcn_cvt_pk_i16:
11454 case Intrinsic::amdgcn_cvt_pk_u16: {
11455 // FIXME: Stop adding cast if v2f16/v2i16 are legal.
11456 EVT VT = Op.getValueType();
11457 unsigned Opcode;
11458
11459 if (IntrinsicID == Intrinsic::amdgcn_cvt_pkrtz)
11460 Opcode = AMDGPUISD::CVT_PKRTZ_F16_F32;
11461 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_i16)
11462 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
11463 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_u16)
11464 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
11465 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pk_i16)
11466 Opcode = AMDGPUISD::CVT_PK_I16_I32;
11467 else
11468 Opcode = AMDGPUISD::CVT_PK_U16_U32;
11469
11470 if (isTypeLegal(VT))
11471 return DAG.getNode(Opcode, DL, VT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
11472
11473 SDValue Node =
11474 DAG.getNode(Opcode, DL, VT: MVT::i32, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
11475 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Node);
11476 }
11477 case Intrinsic::amdgcn_fmad_ftz:
11478 return DAG.getNode(Opcode: AMDGPUISD::FMAD_FTZ, DL, VT, N1: Op.getOperand(i: 1),
11479 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11480
11481 case Intrinsic::amdgcn_if_break:
11482 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::SI_IF_BREAK, dl: DL, VT,
11483 Op1: Op->getOperand(Num: 1), Op2: Op->getOperand(Num: 2)),
11484 0);
11485
11486 case Intrinsic::amdgcn_groupstaticsize: {
11487 Triple::OSType OS = getTargetMachine().getTargetTriple().getOS();
11488 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL)
11489 return Op;
11490
11491 const Module *M = MF.getFunction().getParent();
11492 const GlobalValue *GV =
11493 Intrinsic::getDeclarationIfExists(M, id: Intrinsic::amdgcn_groupstaticsize);
11494 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: 0,
11495 TargetFlags: SIInstrInfo::MO_ABS32_LO);
11496 return {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: GA), 0};
11497 }
11498 case Intrinsic::amdgcn_is_shared:
11499 case Intrinsic::amdgcn_is_private: {
11500 SDLoc SL(Op);
11501 SDValue SrcVec =
11502 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
11503 SDValue SrcHi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: SrcVec,
11504 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
11505
11506 unsigned AS = (IntrinsicID == Intrinsic::amdgcn_is_shared)
11507 ? AMDGPUAS::LOCAL_ADDRESS
11508 : AMDGPUAS::PRIVATE_ADDRESS;
11509 if (AS == AMDGPUAS::PRIVATE_ADDRESS &&
11510 Subtarget->hasGloballyAddressableScratch()) {
11511 SDValue FlatScratchBaseHi(
11512 DAG.getMachineNode(
11513 Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32,
11514 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, VT: MVT::i32)),
11515 0);
11516 // Test bits 63..58 against the aperture address.
11517 return DAG.getSetCC(
11518 DL: SL, VT: MVT::i1,
11519 LHS: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: SrcHi, N2: FlatScratchBaseHi),
11520 RHS: DAG.getConstant(Val: 1u << 26, DL: SL, VT: MVT::i32), Cond: ISD::SETULT);
11521 }
11522
11523 SDValue Aperture = getSegmentAperture(AS, DL: SL, DAG);
11524 return DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: SrcHi, RHS: Aperture, Cond: ISD::SETEQ);
11525 }
11526 case Intrinsic::amdgcn_perm:
11527 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: Op.getOperand(i: 1),
11528 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11529 case Intrinsic::amdgcn_reloc_constant: {
11530 Module *M = MF.getFunction().getParent();
11531 const MDNode *Metadata = cast<MDNodeSDNode>(Val: Op.getOperand(i: 1))->getMD();
11532 auto SymbolName = cast<MDString>(Val: Metadata->getOperand(I: 0))->getString();
11533 auto *RelocSymbol = cast<GlobalVariable>(
11534 Val: M->getOrInsertGlobal(Name: SymbolName, Ty: Type::getInt32Ty(C&: M->getContext())));
11535 SDValue GA = DAG.getTargetGlobalAddress(GV: RelocSymbol, DL, VT: MVT::i32, offset: 0,
11536 TargetFlags: SIInstrInfo::MO_ABS32_LO);
11537 return {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: GA), 0};
11538 }
11539 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
11540 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
11541 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
11542 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
11543 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
11544 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
11545 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
11546 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8: {
11547 if (Op.getOperand(i: 4).getValueType() == MVT::i32)
11548 return SDValue();
11549
11550 SDLoc SL(Op);
11551 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 4), DL: SL, VT: MVT::i32);
11552 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
11553 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1), N3: Op.getOperand(i: 2),
11554 N4: Op.getOperand(i: 3), N5: IndexKeyi32);
11555 }
11556 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
11557 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
11558 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
11559 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
11560 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
11561 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
11562 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
11563 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8: {
11564 if (Op.getOperand(i: 4).getValueType() == MVT::i64)
11565 return SDValue();
11566
11567 SDLoc SL(Op);
11568 auto IndexKeyi64 =
11569 Op.getOperand(i: 4).getValueType() == MVT::v2i32
11570 ? DAG.getBitcast(VT: MVT::i64, V: Op.getOperand(i: 4))
11571 : DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 4), DL: SL, VT: MVT::i64);
11572 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
11573 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
11574 Op.getOperand(i: 3), IndexKeyi64, Op.getOperand(i: 5),
11575 Op.getOperand(i: 6)});
11576 }
11577 case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
11578 case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
11579 case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
11580 case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
11581 case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
11582 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8: {
11583 EVT IndexKeyTy = IntrinsicID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8
11584 ? MVT::i64
11585 : MVT::i32;
11586 if (Op.getOperand(i: 6).getValueType() == IndexKeyTy)
11587 return SDValue();
11588
11589 SDLoc SL(Op);
11590 auto IndexKey =
11591 Op.getOperand(i: 6).getValueType().isVector()
11592 ? DAG.getBitcast(VT: IndexKeyTy, V: Op.getOperand(i: 6))
11593 : DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 6), DL: SL, VT: IndexKeyTy);
11594 SmallVector<SDValue> Args{
11595 Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
11596 Op.getOperand(i: 3), Op.getOperand(i: 4), Op.getOperand(i: 5),
11597 IndexKey, Op.getOperand(i: 7), Op.getOperand(i: 8)};
11598 if (IntrinsicID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8)
11599 Args.push_back(Elt: Op.getOperand(i: 9));
11600 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(), Ops: Args);
11601 }
11602 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
11603 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
11604 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4: {
11605 if (Op.getOperand(i: 6).getValueType() == MVT::i32)
11606 return SDValue();
11607
11608 SDLoc SL(Op);
11609 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 6), DL: SL, VT: MVT::i32);
11610 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
11611 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
11612 Op.getOperand(i: 3), Op.getOperand(i: 4), Op.getOperand(i: 5),
11613 IndexKeyi32, Op.getOperand(i: 7)});
11614 }
11615 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
11616 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
11617 unsigned AFmt = (unsigned)Op.getConstantOperandVal(i: 1);
11618 unsigned BFmt = (unsigned)Op.getConstantOperandVal(i: 3);
11619 unsigned AScaleFmt = (unsigned)Op.getConstantOperandVal(i: 8);
11620 unsigned BScaleFmt = (unsigned)Op.getConstantOperandVal(i: 11);
11621 if (!AMDGPU::isValidWMMAScaleFmtCombination(AFmt, AScale: AScaleFmt, BFmt,
11622 BScale: BScaleFmt)) {
11623 DAG.getMachineFunction().getFunction().getContext().emitError(
11624 ErrorStr: "invalid matrix and scale format combination in wmma call");
11625 Op->print(OS&: errs());
11626 errs() << '\n';
11627 }
11628 return SDValue();
11629 }
11630 case Intrinsic::amdgcn_addrspacecast_nonnull:
11631 return lowerADDRSPACECAST(Op, DAG);
11632 case Intrinsic::amdgcn_readlane:
11633 case Intrinsic::amdgcn_readfirstlane:
11634 case Intrinsic::amdgcn_writelane:
11635 case Intrinsic::amdgcn_permlane16:
11636 case Intrinsic::amdgcn_permlanex16:
11637 case Intrinsic::amdgcn_permlane64:
11638 case Intrinsic::amdgcn_set_inactive:
11639 case Intrinsic::amdgcn_set_inactive_chain_arg:
11640 case Intrinsic::amdgcn_mov_dpp8:
11641 case Intrinsic::amdgcn_update_dpp:
11642 case Intrinsic::amdgcn_permlane_bcast:
11643 case Intrinsic::amdgcn_permlane_up:
11644 case Intrinsic::amdgcn_permlane_down:
11645 case Intrinsic::amdgcn_permlane_xor:
11646 return lowerLaneOp(TLI: *this, N: Op.getNode(), DAG);
11647 case Intrinsic::amdgcn_dead: {
11648 SmallVector<SDValue, 8> Poisons;
11649 for (const EVT ValTy : Op.getNode()->values())
11650 Poisons.push_back(Elt: DAG.getPOISON(VT: ValTy));
11651 return DAG.getMergeValues(Ops: Poisons, dl: SDLoc(Op));
11652 }
11653 case Intrinsic::amdgcn_wave_shuffle:
11654 return lowerWaveShuffle(TLI: *this, N: Op.getNode(), DAG);
11655 default:
11656 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
11657 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrinsicID))
11658 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: false);
11659
11660 return Op;
11661 }
11662}
11663
11664// On targets not supporting constant in soffset field, turn zero to
11665// SGPR_NULL to avoid generating an extra s_mov with zero.
11666static SDValue selectSOffset(SDValue SOffset, SelectionDAG &DAG,
11667 const GCNSubtarget *Subtarget) {
11668 if (Subtarget->hasRestrictedSOffset() && isNullConstant(V: SOffset))
11669 return DAG.getRegister(Reg: AMDGPU::SGPR_NULL, VT: MVT::i32);
11670 return SOffset;
11671}
11672
11673SDValue SITargetLowering::lowerRawBufferAtomicIntrin(SDValue Op,
11674 SelectionDAG &DAG,
11675 unsigned NewOpcode) const {
11676 SDLoc DL(Op);
11677
11678 SDValue VData = Op.getOperand(i: 2);
11679 if (VData.getValueSizeInBits() != 32 && VData.getValueSizeInBits() != 64) {
11680 SmallVector<EVT, 2> ResultTypes(Op->values());
11681 return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
11682 Msg: "unsupported buffer atomic data type");
11683 }
11684 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
11685 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
11686 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
11687 SDValue Ops[] = {
11688 Op.getOperand(i: 0), // Chain
11689 VData, // vdata
11690 Rsrc, // rsrc
11691 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
11692 VOffset, // voffset
11693 SOffset, // soffset
11694 Offset, // offset
11695 Op.getOperand(i: 6), // cachepolicy
11696 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
11697 };
11698
11699 auto *M = cast<MemSDNode>(Val&: Op);
11700
11701 EVT MemVT = VData.getValueType();
11702 return DAG.getMemIntrinsicNode(Opcode: NewOpcode, dl: DL, VTList: Op->getVTList(), Ops, MemVT,
11703 MMO: M->getMemOperand());
11704}
11705
11706SDValue
11707SITargetLowering::lowerStructBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG,
11708 unsigned NewOpcode) const {
11709 SDLoc DL(Op);
11710
11711 SDValue VData = Op.getOperand(i: 2);
11712 if (VData.getValueSizeInBits() != 32 && VData.getValueSizeInBits() != 64) {
11713 SmallVector<EVT, 2> ResultTypes(Op->values());
11714 return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
11715 Msg: "unsupported buffer atomic data type");
11716 }
11717 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
11718 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
11719 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
11720 SDValue Ops[] = {
11721 Op.getOperand(i: 0), // Chain
11722 VData, // vdata
11723 Rsrc, // rsrc
11724 Op.getOperand(i: 4), // vindex
11725 VOffset, // voffset
11726 SOffset, // soffset
11727 Offset, // offset
11728 Op.getOperand(i: 7), // cachepolicy
11729 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
11730 };
11731
11732 auto *M = cast<MemSDNode>(Val&: Op);
11733
11734 EVT MemVT = VData.getValueType();
11735 return DAG.getMemIntrinsicNode(Opcode: NewOpcode, dl: DL, VTList: Op->getVTList(), Ops, MemVT,
11736 MMO: M->getMemOperand());
11737}
11738
11739static void initializeM0ToZeroForClusterLoad(SDValue Op, SelectionDAG &DAG,
11740 SDLoc DL) {
11741 SDNode *N = Op.getNode();
11742 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
11743 unsigned NumOperands = N->getNumOperands();
11744 if (N->getOperand(Num: NumOperands - 1) == Zero)
11745 return;
11746 SmallVector<SDValue, 7> Ops(N->ops());
11747 Ops[NumOperands - 1] = Zero; // M0 = 0
11748 DAG.UpdateNodeOperands(N, Ops);
11749}
11750
11751SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
11752 SelectionDAG &DAG) const {
11753 unsigned IntrID = Op.getConstantOperandVal(i: 1);
11754 SDLoc DL(Op);
11755
11756 switch (IntrID) {
11757 case Intrinsic::amdgcn_cluster_load_b32:
11758 case Intrinsic::amdgcn_cluster_load_b64:
11759 case Intrinsic::amdgcn_cluster_load_b128: {
11760 if (Subtarget->hasGFX1250_STRICT())
11761 initializeM0ToZeroForClusterLoad(Op, DAG, DL);
11762 return SDValue();
11763 }
11764 case Intrinsic::amdgcn_ds_ordered_add:
11765 case Intrinsic::amdgcn_ds_ordered_swap: {
11766 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11767 SDValue Chain = M->getOperand(Num: 0);
11768 SDValue M0 = M->getOperand(Num: 2);
11769 SDValue Value = M->getOperand(Num: 3);
11770 unsigned IndexOperand = M->getConstantOperandVal(Num: 7);
11771 unsigned WaveRelease = M->getConstantOperandVal(Num: 8);
11772 unsigned WaveDone = M->getConstantOperandVal(Num: 9);
11773
11774 unsigned OrderedCountIndex = IndexOperand & 0x3f;
11775 IndexOperand &= ~0x3f;
11776 unsigned CountDw = 0;
11777
11778 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10) {
11779 CountDw = (IndexOperand >> 24) & 0xf;
11780 IndexOperand &= ~(0xf << 24);
11781
11782 if (CountDw < 1 || CountDw > 4) {
11783 const Function &Fn = DAG.getMachineFunction().getFunction();
11784 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11785 Fn, "ds_ordered_count: dword count must be between 1 and 4",
11786 DL.getDebugLoc()));
11787 CountDw = 1;
11788 }
11789 }
11790
11791 if (IndexOperand) {
11792 const Function &Fn = DAG.getMachineFunction().getFunction();
11793 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11794 Fn, "ds_ordered_count: bad index operand", DL.getDebugLoc()));
11795 }
11796
11797 if (WaveDone && !WaveRelease) {
11798 // TODO: Move this to IR verifier
11799 const Function &Fn = DAG.getMachineFunction().getFunction();
11800 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11801 Fn, "ds_ordered_count: wave_done requires wave_release",
11802 DL.getDebugLoc()));
11803 }
11804
11805 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
11806 unsigned ShaderType =
11807 SIInstrInfo::getDSShaderTypeValue(MF: DAG.getMachineFunction());
11808 unsigned Offset0 = OrderedCountIndex << 2;
11809 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (Instruction << 4);
11810
11811 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10)
11812 Offset1 |= (CountDw - 1) << 6;
11813
11814 if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX11)
11815 Offset1 |= ShaderType << 2;
11816
11817 unsigned Offset = Offset0 | (Offset1 << 8);
11818
11819 SDValue Ops[] = {
11820 Chain, Value, DAG.getTargetConstant(Val: Offset, DL, VT: MVT::i16),
11821 copyToM0(DAG, Chain, DL, V: M0).getValue(R: 1), // Glue
11822 };
11823 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::DS_ORDERED_COUNT, dl: DL,
11824 VTList: M->getVTList(), Ops, MemVT: M->getMemoryVT(),
11825 MMO: M->getMemOperand());
11826 }
11827 case Intrinsic::amdgcn_ptr_s_buffer_load: {
11828 unsigned CPol = Op.getConstantOperandVal(i: 4);
11829 if (CPol & ~((Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12)
11830 ? AMDGPU::CPol::ALL
11831 : AMDGPU::CPol::ALL_pregfx12))
11832 return Op;
11833
11834 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11835 return lowerSBuffer(
11836 VT: Op.getValueType(), MemVT: M->getMemoryVT(), DL, Chain: Op.getOperand(i: 0),
11837 Rsrc: bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG), Offset: Op.getOperand(i: 3),
11838 CachePolicy: Op.getOperand(i: 4), DAG, MMO: M->getMemOperand());
11839 }
11840 case Intrinsic::amdgcn_raw_buffer_load:
11841 case Intrinsic::amdgcn_raw_ptr_buffer_load:
11842 case Intrinsic::amdgcn_raw_atomic_buffer_load:
11843 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
11844 case Intrinsic::amdgcn_raw_buffer_load_format:
11845 case Intrinsic::amdgcn_raw_ptr_buffer_load_format: {
11846 const bool IsFormat =
11847 IntrID == Intrinsic::amdgcn_raw_buffer_load_format ||
11848 IntrID == Intrinsic::amdgcn_raw_ptr_buffer_load_format;
11849
11850 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
11851 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 3), DAG);
11852 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 4), DAG, Subtarget);
11853 SDValue Ops[] = {
11854 Op.getOperand(i: 0), // Chain
11855 Rsrc, // rsrc
11856 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
11857 VOffset, // voffset
11858 SOffset, // soffset
11859 Offset, // offset
11860 Op.getOperand(i: 5), // cachepolicy, swizzled buffer
11861 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
11862 };
11863
11864 auto *M = cast<MemSDNode>(Val&: Op);
11865 return lowerIntrinsicLoad(M, IsFormat, DAG, Ops);
11866 }
11867 case Intrinsic::amdgcn_struct_buffer_load:
11868 case Intrinsic::amdgcn_struct_ptr_buffer_load:
11869 case Intrinsic::amdgcn_struct_buffer_load_format:
11870 case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
11871 case Intrinsic::amdgcn_struct_atomic_buffer_load:
11872 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
11873 const bool IsFormat =
11874 IntrID == Intrinsic::amdgcn_struct_buffer_load_format ||
11875 IntrID == Intrinsic::amdgcn_struct_ptr_buffer_load_format;
11876
11877 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
11878 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
11879 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
11880 SDValue Ops[] = {
11881 Op.getOperand(i: 0), // Chain
11882 Rsrc, // rsrc
11883 Op.getOperand(i: 3), // vindex
11884 VOffset, // voffset
11885 SOffset, // soffset
11886 Offset, // offset
11887 Op.getOperand(i: 6), // cachepolicy, swizzled buffer
11888 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
11889 };
11890
11891 return lowerIntrinsicLoad(M: cast<MemSDNode>(Val&: Op), IsFormat, DAG, Ops);
11892 }
11893 case Intrinsic::amdgcn_raw_tbuffer_load:
11894 case Intrinsic::amdgcn_raw_ptr_tbuffer_load: {
11895 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11896 EVT LoadVT = Op.getValueType();
11897 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
11898 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 3), DAG);
11899 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 4), DAG, Subtarget);
11900
11901 SDValue Ops[] = {
11902 Op.getOperand(i: 0), // Chain
11903 Rsrc, // rsrc
11904 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
11905 VOffset, // voffset
11906 SOffset, // soffset
11907 Offset, // offset
11908 Op.getOperand(i: 5), // format
11909 Op.getOperand(i: 6), // cachepolicy, swizzled buffer
11910 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
11911 };
11912
11913 if (LoadVT.getScalarSizeInBits() == 16)
11914 return adjustLoadValueType(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
11915 Ops);
11916 return getMemIntrinsicNode(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
11917 VTList: Op->getVTList(), Ops, MemVT: LoadVT, MMO: M->getMemOperand(),
11918 DAG);
11919 }
11920 case Intrinsic::amdgcn_struct_tbuffer_load:
11921 case Intrinsic::amdgcn_struct_ptr_tbuffer_load: {
11922 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11923 EVT LoadVT = Op.getValueType();
11924 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
11925 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
11926 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
11927
11928 SDValue Ops[] = {
11929 Op.getOperand(i: 0), // Chain
11930 Rsrc, // rsrc
11931 Op.getOperand(i: 3), // vindex
11932 VOffset, // voffset
11933 SOffset, // soffset
11934 Offset, // offset
11935 Op.getOperand(i: 6), // format
11936 Op.getOperand(i: 7), // cachepolicy, swizzled buffer
11937 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
11938 };
11939
11940 if (LoadVT.getScalarSizeInBits() == 16)
11941 return adjustLoadValueType(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
11942 Ops);
11943 return getMemIntrinsicNode(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
11944 VTList: Op->getVTList(), Ops, MemVT: LoadVT, MMO: M->getMemOperand(),
11945 DAG);
11946 }
11947 case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
11948 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
11949 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FADD);
11950 case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
11951 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
11952 return lowerStructBufferAtomicIntrin(Op, DAG,
11953 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FADD);
11954 case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
11955 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
11956 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMIN);
11957 case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
11958 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
11959 return lowerStructBufferAtomicIntrin(Op, DAG,
11960 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMIN);
11961 case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
11962 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
11963 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMAX);
11964 case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
11965 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
11966 return lowerStructBufferAtomicIntrin(Op, DAG,
11967 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMAX);
11968 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
11969 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
11970 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SWAP);
11971 case Intrinsic::amdgcn_raw_buffer_atomic_add:
11972 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
11973 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_ADD);
11974 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
11975 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
11976 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SUB);
11977 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
11978 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
11979 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMIN);
11980 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
11981 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
11982 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMIN);
11983 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
11984 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
11985 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMAX);
11986 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
11987 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
11988 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMAX);
11989 case Intrinsic::amdgcn_raw_buffer_atomic_and:
11990 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
11991 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_AND);
11992 case Intrinsic::amdgcn_raw_buffer_atomic_or:
11993 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
11994 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_OR);
11995 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
11996 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
11997 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_XOR);
11998 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
11999 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
12000 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_INC);
12001 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
12002 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
12003 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_DEC);
12004 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
12005 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
12006 return lowerStructBufferAtomicIntrin(Op, DAG,
12007 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SWAP);
12008 case Intrinsic::amdgcn_struct_buffer_atomic_add:
12009 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
12010 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_ADD);
12011 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
12012 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
12013 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SUB);
12014 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
12015 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
12016 return lowerStructBufferAtomicIntrin(Op, DAG,
12017 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMIN);
12018 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
12019 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
12020 return lowerStructBufferAtomicIntrin(Op, DAG,
12021 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMIN);
12022 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
12023 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
12024 return lowerStructBufferAtomicIntrin(Op, DAG,
12025 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMAX);
12026 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
12027 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
12028 return lowerStructBufferAtomicIntrin(Op, DAG,
12029 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMAX);
12030 case Intrinsic::amdgcn_struct_buffer_atomic_and:
12031 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
12032 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_AND);
12033 case Intrinsic::amdgcn_struct_buffer_atomic_or:
12034 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
12035 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_OR);
12036 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
12037 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
12038 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_XOR);
12039 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
12040 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
12041 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_INC);
12042 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
12043 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
12044 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_DEC);
12045 case Intrinsic::amdgcn_raw_buffer_atomic_sub_clamp_u32:
12046 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
12047 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_CSUB);
12048 case Intrinsic::amdgcn_struct_buffer_atomic_sub_clamp_u32:
12049 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
12050 return lowerStructBufferAtomicIntrin(Op, DAG,
12051 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_CSUB);
12052 case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
12053 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
12054 return lowerRawBufferAtomicIntrin(Op, DAG,
12055 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_COND_SUB_U32);
12056 case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
12057 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
12058 return lowerStructBufferAtomicIntrin(Op, DAG,
12059 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_COND_SUB_U32);
12060 case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
12061 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap: {
12062 SDValue Src = Op.getOperand(i: 2);
12063 if (Src.getValueSizeInBits() != 32 && Src.getValueSizeInBits() != 64) {
12064 SmallVector<EVT, 2> ResultTypes(Op->values());
12065 return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
12066 Msg: "unsupported buffer atomic data type");
12067 }
12068 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 4), DAG);
12069 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
12070 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
12071 SDValue Ops[] = {
12072 Op.getOperand(i: 0), // Chain
12073 Op.getOperand(i: 2), // src
12074 Op.getOperand(i: 3), // cmp
12075 Rsrc, // rsrc
12076 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
12077 VOffset, // voffset
12078 SOffset, // soffset
12079 Offset, // offset
12080 Op.getOperand(i: 7), // cachepolicy
12081 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
12082 };
12083 EVT VT = Op.getValueType();
12084 auto *M = cast<MemSDNode>(Val&: Op);
12085
12086 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, dl: DL,
12087 VTList: Op->getVTList(), Ops, MemVT: VT,
12088 MMO: M->getMemOperand());
12089 }
12090 case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
12091 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap: {
12092 SDValue Src = Op.getOperand(i: 2);
12093 if (Src.getValueSizeInBits() != 32 && Src.getValueSizeInBits() != 64) {
12094 SmallVector<EVT, 2> ResultTypes(Op->values());
12095 return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
12096 Msg: "unsupported buffer atomic data type");
12097 }
12098 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op->getOperand(Num: 4), DAG);
12099 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 6), DAG);
12100 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 7), DAG, Subtarget);
12101 SDValue Ops[] = {
12102 Op.getOperand(i: 0), // Chain
12103 Op.getOperand(i: 2), // src
12104 Op.getOperand(i: 3), // cmp
12105 Rsrc, // rsrc
12106 Op.getOperand(i: 5), // vindex
12107 VOffset, // voffset
12108 SOffset, // soffset
12109 Offset, // offset
12110 Op.getOperand(i: 8), // cachepolicy
12111 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
12112 };
12113 EVT VT = Op.getValueType();
12114 auto *M = cast<MemSDNode>(Val&: Op);
12115
12116 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, dl: DL,
12117 VTList: Op->getVTList(), Ops, MemVT: VT,
12118 MMO: M->getMemOperand());
12119 }
12120 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
12121 case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
12122 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12123 SDValue NodePtr = M->getOperand(Num: 2);
12124 SDValue RayExtent = M->getOperand(Num: 3);
12125 SDValue InstanceMask = M->getOperand(Num: 4);
12126 SDValue RayOrigin = M->getOperand(Num: 5);
12127 SDValue RayDir = M->getOperand(Num: 6);
12128 SDValue Offsets = M->getOperand(Num: 7);
12129 SDValue TDescr = M->getOperand(Num: 8);
12130
12131 assert(NodePtr.getValueType() == MVT::i64);
12132 assert(RayDir.getValueType() == MVT::v3f32);
12133
12134 bool IsBVH8 = IntrID == Intrinsic::amdgcn_image_bvh8_intersect_ray;
12135 const unsigned NumVDataDwords = 10;
12136 const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12;
12137 int Opcode = AMDGPU::getMIMGOpcode(
12138 BaseOpcode: IsBVH8 ? AMDGPU::IMAGE_BVH8_INTERSECT_RAY
12139 : AMDGPU::IMAGE_BVH_DUAL_INTERSECT_RAY,
12140 MIMGEncoding: AMDGPU::MIMGEncGfx12, VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
12141 assert(Opcode != -1);
12142
12143 SmallVector<SDValue, 7> Ops;
12144 Ops.push_back(Elt: NodePtr);
12145 Ops.push_back(Elt: DAG.getBuildVector(
12146 VT: MVT::v2i32, DL,
12147 Ops: {DAG.getBitcast(VT: MVT::i32, V: RayExtent),
12148 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: InstanceMask)}));
12149 Ops.push_back(Elt: RayOrigin);
12150 Ops.push_back(Elt: RayDir);
12151 Ops.push_back(Elt: Offsets);
12152 Ops.push_back(Elt: TDescr);
12153 Ops.push_back(Elt: M->getChain());
12154
12155 auto *NewNode = DAG.getMachineNode(Opcode, dl: DL, VTs: M->getVTList(), Ops);
12156 MachineMemOperand *MemRef = M->getMemOperand();
12157 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
12158 return SDValue(NewNode, 0);
12159 }
12160 case Intrinsic::amdgcn_image_bvh_intersect_ray: {
12161 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12162 SDValue NodePtr = M->getOperand(Num: 2);
12163 SDValue RayExtent = M->getOperand(Num: 3);
12164 SDValue RayOrigin = M->getOperand(Num: 4);
12165 SDValue RayDir = M->getOperand(Num: 5);
12166 SDValue RayInvDir = M->getOperand(Num: 6);
12167 SDValue TDescr = M->getOperand(Num: 7);
12168
12169 assert(NodePtr.getValueType() == MVT::i32 ||
12170 NodePtr.getValueType() == MVT::i64);
12171 assert(RayDir.getValueType() == MVT::v3f16 ||
12172 RayDir.getValueType() == MVT::v3f32);
12173
12174 const bool IsGFX11 = AMDGPU::isGFX11(STI: *Subtarget);
12175 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI: *Subtarget);
12176 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
12177 const bool IsA16 = RayDir.getValueType().getVectorElementType() == MVT::f16;
12178 const bool Is64 = NodePtr.getValueType() == MVT::i64;
12179 const unsigned NumVDataDwords = 4;
12180 const unsigned NumVAddrDwords = IsA16 ? (Is64 ? 9 : 8) : (Is64 ? 12 : 11);
12181 const unsigned NumVAddrs = IsGFX11Plus ? (IsA16 ? 4 : 5) : NumVAddrDwords;
12182 const bool UseNSA = (Subtarget->hasNSAEncoding() &&
12183 NumVAddrs <= Subtarget->getNSAMaxSize()) ||
12184 IsGFX12Plus;
12185 const unsigned BaseOpcodes[2][2] = {
12186 {AMDGPU::IMAGE_BVH_INTERSECT_RAY, AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16},
12187 {AMDGPU::IMAGE_BVH64_INTERSECT_RAY,
12188 AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16}};
12189 int Opcode;
12190 if (UseNSA) {
12191 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: BaseOpcodes[Is64][IsA16],
12192 MIMGEncoding: IsGFX12Plus ? AMDGPU::MIMGEncGfx12
12193 : IsGFX11 ? AMDGPU::MIMGEncGfx11NSA
12194 : AMDGPU::MIMGEncGfx10NSA,
12195 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
12196 } else {
12197 assert(!IsGFX12Plus);
12198 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: BaseOpcodes[Is64][IsA16],
12199 MIMGEncoding: IsGFX11 ? AMDGPU::MIMGEncGfx11Default
12200 : AMDGPU::MIMGEncGfx10Default,
12201 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
12202 }
12203 assert(Opcode != -1);
12204
12205 SmallVector<SDValue, 16> Ops;
12206
12207 auto packLanes = [&DAG, &Ops, &DL](SDValue Op, bool IsAligned) {
12208 SmallVector<SDValue, 3> Lanes;
12209 DAG.ExtractVectorElements(Op, Args&: Lanes, Start: 0, Count: 3);
12210 if (Lanes[0].getValueSizeInBits() == 32) {
12211 for (unsigned I = 0; I < 3; ++I)
12212 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: Lanes[I]));
12213 } else {
12214 if (IsAligned) {
12215 Ops.push_back(Elt: DAG.getBitcast(
12216 VT: MVT::i32,
12217 V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Lanes[0], Lanes[1]})));
12218 Ops.push_back(Elt: Lanes[2]);
12219 } else {
12220 SDValue Elt0 = Ops.pop_back_val();
12221 Ops.push_back(Elt: DAG.getBitcast(
12222 VT: MVT::i32, V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Elt0, Lanes[0]})));
12223 Ops.push_back(Elt: DAG.getBitcast(
12224 VT: MVT::i32,
12225 V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Lanes[1], Lanes[2]})));
12226 }
12227 }
12228 };
12229
12230 if (UseNSA && IsGFX11Plus) {
12231 Ops.push_back(Elt: NodePtr);
12232 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: RayExtent));
12233 Ops.push_back(Elt: RayOrigin);
12234 if (IsA16) {
12235 SmallVector<SDValue, 3> DirLanes, InvDirLanes, MergedLanes;
12236 DAG.ExtractVectorElements(Op: RayDir, Args&: DirLanes, Start: 0, Count: 3);
12237 DAG.ExtractVectorElements(Op: RayInvDir, Args&: InvDirLanes, Start: 0, Count: 3);
12238 for (unsigned I = 0; I < 3; ++I) {
12239 MergedLanes.push_back(Elt: DAG.getBitcast(
12240 VT: MVT::i32, V: DAG.getBuildVector(VT: MVT::v2f16, DL,
12241 Ops: {DirLanes[I], InvDirLanes[I]})));
12242 }
12243 Ops.push_back(Elt: DAG.getBuildVector(VT: MVT::v3i32, DL, Ops: MergedLanes));
12244 } else {
12245 Ops.push_back(Elt: RayDir);
12246 Ops.push_back(Elt: RayInvDir);
12247 }
12248 } else {
12249 if (Is64)
12250 DAG.ExtractVectorElements(Op: DAG.getBitcast(VT: MVT::v2i32, V: NodePtr), Args&: Ops, Start: 0,
12251 Count: 2);
12252 else
12253 Ops.push_back(Elt: NodePtr);
12254
12255 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: RayExtent));
12256 packLanes(RayOrigin, true);
12257 packLanes(RayDir, true);
12258 packLanes(RayInvDir, false);
12259 }
12260
12261 if (!UseNSA) {
12262 // Build a single vector containing all the operands so far prepared.
12263 if (NumVAddrDwords > 12) {
12264 SDValue Undef = DAG.getPOISON(VT: MVT::i32);
12265 Ops.append(NumInputs: 16 - Ops.size(), Elt: Undef);
12266 }
12267 assert(Ops.size() >= 8 && Ops.size() <= 12);
12268 SDValue MergedOps =
12269 DAG.getBuildVector(VT: MVT::getVectorVT(VT: MVT::i32, NumElements: Ops.size()), DL, Ops);
12270 Ops.clear();
12271 Ops.push_back(Elt: MergedOps);
12272 }
12273
12274 Ops.push_back(Elt: TDescr);
12275 Ops.push_back(Elt: DAG.getTargetConstant(Val: IsA16, DL, VT: MVT::i1));
12276 Ops.push_back(Elt: M->getChain());
12277
12278 auto *NewNode = DAG.getMachineNode(Opcode, dl: DL, VTs: M->getVTList(), Ops);
12279 MachineMemOperand *MemRef = M->getMemOperand();
12280 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
12281 return SDValue(NewNode, 0);
12282 }
12283 case Intrinsic::amdgcn_global_atomic_fmin_num:
12284 case Intrinsic::amdgcn_global_atomic_fmax_num:
12285 case Intrinsic::amdgcn_flat_atomic_fmin_num:
12286 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
12287 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12288 SDValue Ops[] = {
12289 M->getOperand(Num: 0), // Chain
12290 M->getOperand(Num: 2), // Ptr
12291 M->getOperand(Num: 3) // Value
12292 };
12293 unsigned Opcode = 0;
12294 switch (IntrID) {
12295 case Intrinsic::amdgcn_global_atomic_fmin_num:
12296 case Intrinsic::amdgcn_flat_atomic_fmin_num: {
12297 Opcode = ISD::ATOMIC_LOAD_FMIN;
12298 break;
12299 }
12300 case Intrinsic::amdgcn_global_atomic_fmax_num:
12301 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
12302 Opcode = ISD::ATOMIC_LOAD_FMAX;
12303 break;
12304 }
12305 default:
12306 llvm_unreachable("unhandled atomic opcode");
12307 }
12308 return DAG.getAtomic(Opcode, dl: SDLoc(Op), MemVT: M->getMemoryVT(), VTList: M->getVTList(),
12309 Ops, MMO: M->getMemOperand());
12310 }
12311 case Intrinsic::amdgcn_s_alloc_vgpr: {
12312 SDValue NumVGPRs = Op.getOperand(i: 2);
12313 if (!NumVGPRs->isDivergent())
12314 return Op;
12315
12316 SDValue ReadFirstLaneID =
12317 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
12318 NumVGPRs = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
12319 N1: ReadFirstLaneID, N2: NumVGPRs);
12320
12321 return DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL, VTList: Op->getVTList(),
12322 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1), N3: NumVGPRs);
12323 }
12324 case Intrinsic::amdgcn_s_get_barrier_state:
12325 case Intrinsic::amdgcn_s_get_named_barrier_state: {
12326 SDValue Chain = Op->getOperand(Num: 0);
12327 SmallVector<SDValue, 2> Ops;
12328 unsigned Opc;
12329
12330 if (isa<ConstantSDNode>(Val: Op->getOperand(Num: 2))) {
12331 uint64_t BarID = cast<ConstantSDNode>(Val: Op->getOperand(Num: 2))->getZExtValue();
12332 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state)
12333 BarID = (BarID >> 4) & 0x3F;
12334 Opc = AMDGPU::S_GET_BARRIER_STATE_IMM;
12335 SDValue K = DAG.getTargetConstant(Val: BarID, DL, VT: MVT::i32);
12336 Ops.push_back(Elt: K);
12337 Ops.push_back(Elt: Chain);
12338 } else {
12339 Opc = AMDGPU::S_GET_BARRIER_STATE_M0;
12340 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
12341 SDValue M0Val;
12342 M0Val = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Op->getOperand(Num: 2),
12343 N2: DAG.getShiftAmountConstant(Val: 4, VT: MVT::i32, DL));
12344 M0Val = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: M0Val,
12345 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
12346 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
12347 } else
12348 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: Op->getOperand(Num: 2)).getValue(R: 0));
12349 }
12350
12351 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
12352 return SDValue(NewMI, 0);
12353 }
12354 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
12355 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
12356 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
12357 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12358 SDValue Chain = Op->getOperand(Num: 0);
12359 SDValue Ptr = Op->getOperand(Num: 2);
12360 EVT VT = Op->getValueType(ResNo: 0);
12361 return DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl: DL, MemVT: MII->getMemoryVT(), VT,
12362 Chain, Ptr, MMO: MII->getMemOperand());
12363 }
12364 case Intrinsic::amdgcn_av_load_b128: {
12365 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12366 SDValue Chain = Op->getOperand(Num: 0);
12367 SDValue Ptr = Op->getOperand(Num: 2);
12368 EVT VT = Op->getValueType(ResNo: 0);
12369 // Lower to a regular ISD::LOAD. The MachineMemOperand carries Monotonic
12370 // ordering and syncscope so that SIMemoryLegalizer sets cache policy bits.
12371 // Address space filtering in the load_global/load_flat PatFrags selects
12372 // the correct GLOBAL vs FLAT instruction.
12373 return DAG.getLoad(VT, dl: DL, Chain, Ptr, MMO: MII->getMemOperand());
12374 }
12375 case Intrinsic::amdgcn_flat_load_monitor_b32:
12376 case Intrinsic::amdgcn_flat_load_monitor_b64:
12377 case Intrinsic::amdgcn_flat_load_monitor_b128: {
12378 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12379 SDValue Chain = Op->getOperand(Num: 0);
12380 SDValue Ptr = Op->getOperand(Num: 2);
12381 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::FLAT_LOAD_MONITOR, dl: DL,
12382 VTList: Op->getVTList(), Ops: {Chain, Ptr},
12383 MemVT: MII->getMemoryVT(), MMO: MII->getMemOperand());
12384 }
12385 case Intrinsic::amdgcn_global_load_monitor_b32:
12386 case Intrinsic::amdgcn_global_load_monitor_b64:
12387 case Intrinsic::amdgcn_global_load_monitor_b128: {
12388 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12389 SDValue Chain = Op->getOperand(Num: 0);
12390 SDValue Ptr = Op->getOperand(Num: 2);
12391 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::GLOBAL_LOAD_MONITOR, dl: DL,
12392 VTList: Op->getVTList(), Ops: {Chain, Ptr},
12393 MemVT: MII->getMemoryVT(), MMO: MII->getMemOperand());
12394 }
12395 default:
12396
12397 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
12398 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrID))
12399 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: true);
12400
12401 return SDValue();
12402 }
12403}
12404
12405// Call DAG.getMemIntrinsicNode for a load, but first widen a dwordx3 type to
12406// dwordx4 if on SI and handle TFE loads.
12407SDValue SITargetLowering::getMemIntrinsicNode(unsigned Opcode, const SDLoc &DL,
12408 SDVTList VTList,
12409 ArrayRef<SDValue> Ops, EVT MemVT,
12410 MachineMemOperand *MMO,
12411 SelectionDAG &DAG) const {
12412 LLVMContext &C = *DAG.getContext();
12413 MachineFunction &MF = DAG.getMachineFunction();
12414 EVT VT = VTList.VTs[0];
12415
12416 assert(VTList.NumVTs == 2 || VTList.NumVTs == 3);
12417 bool IsTFE = VTList.NumVTs == 3;
12418 if (IsTFE) {
12419 unsigned NumValueDWords = divideCeil(Numerator: VT.getSizeInBits(), Denominator: 32);
12420 unsigned NumOpDWords = NumValueDWords + 1;
12421 EVT OpDWordsVT = EVT::getVectorVT(Context&: C, VT: MVT::i32, NumElements: NumOpDWords);
12422 SDVTList OpDWordsVTList = DAG.getVTList(VT1: OpDWordsVT, VT2: VTList.VTs[2]);
12423 MachineMemOperand *OpDWordsMMO =
12424 MF.getMachineMemOperand(MMO, Offset: 0, Size: NumOpDWords * 4);
12425 SDValue Op = getMemIntrinsicNode(Opcode, DL, VTList: OpDWordsVTList, Ops,
12426 MemVT: OpDWordsVT, MMO: OpDWordsMMO, DAG);
12427 SDValue Status = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
12428 N2: DAG.getVectorIdxConstant(Val: NumValueDWords, DL));
12429 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL);
12430 SDValue ValueDWords =
12431 NumValueDWords == 1
12432 ? DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op, N2: ZeroIdx)
12433 : DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL,
12434 VT: EVT::getVectorVT(Context&: C, VT: MVT::i32, NumElements: NumValueDWords), N1: Op,
12435 N2: ZeroIdx);
12436 SDValue Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: ValueDWords);
12437 return DAG.getMergeValues(Ops: {Value, Status, SDValue(Op.getNode(), 1)}, dl: DL);
12438 }
12439
12440 if (!Subtarget->hasDwordx3LoadStores() &&
12441 (VT == MVT::v3i32 || VT == MVT::v3f32)) {
12442 EVT WidenedVT = EVT::getVectorVT(Context&: C, VT: VT.getVectorElementType(), NumElements: 4);
12443 EVT WidenedMemVT = EVT::getVectorVT(Context&: C, VT: MemVT.getVectorElementType(), NumElements: 4);
12444 MachineMemOperand *WidenedMMO = MF.getMachineMemOperand(MMO, Offset: 0, Size: 16);
12445 SDVTList WidenedVTList = DAG.getVTList(VT1: WidenedVT, VT2: VTList.VTs[1]);
12446 SDValue Op = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: WidenedVTList, Ops,
12447 MemVT: WidenedMemVT, MMO: WidenedMMO);
12448 SDValue Value = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Op,
12449 N2: DAG.getVectorIdxConstant(Val: 0, DL));
12450 return DAG.getMergeValues(Ops: {Value, SDValue(Op.getNode(), 1)}, dl: DL);
12451 }
12452
12453 return DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList, Ops, MemVT, MMO);
12454}
12455
12456SDValue SITargetLowering::handleD16VData(SDValue VData, SelectionDAG &DAG,
12457 bool ImageStore) const {
12458 EVT StoreVT = VData.getValueType();
12459
12460 // No change for f16 and legal vector D16 types.
12461 if (!StoreVT.isVector())
12462 return VData;
12463
12464 SDLoc DL(VData);
12465 unsigned NumElements = StoreVT.getVectorNumElements();
12466
12467 if (Subtarget->hasUnpackedD16VMem()) {
12468 // We need to unpack the packed data to store.
12469 EVT IntStoreVT = StoreVT.changeTypeToInteger();
12470 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
12471
12472 EVT EquivStoreVT =
12473 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements);
12474 SDValue ZExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EquivStoreVT, Operand: IntVData);
12475 return DAG.UnrollVectorOp(N: ZExt.getNode());
12476 }
12477
12478 // The sq block of gfx8.1 does not estimate register use correctly for d16
12479 // image store instructions. The data operand is computed as if it were not a
12480 // d16 image instruction.
12481 if (ImageStore && Subtarget->hasImageStoreD16Bug()) {
12482 // Bitcast to i16
12483 EVT IntStoreVT = StoreVT.changeTypeToInteger();
12484 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
12485
12486 // Decompose into scalars
12487 SmallVector<SDValue, 4> Elts;
12488 DAG.ExtractVectorElements(Op: IntVData, Args&: Elts);
12489
12490 // Group pairs of i16 into v2i16 and bitcast to i32
12491 SmallVector<SDValue, 4> PackedElts;
12492 for (unsigned I = 0; I < Elts.size() / 2; I += 1) {
12493 SDValue Pair =
12494 DAG.getBuildVector(VT: MVT::v2i16, DL, Ops: {Elts[I * 2], Elts[I * 2 + 1]});
12495 SDValue IntPair = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Pair);
12496 PackedElts.push_back(Elt: IntPair);
12497 }
12498 if ((NumElements % 2) == 1) {
12499 // Handle v3i16
12500 unsigned I = Elts.size() / 2;
12501 SDValue Pair = DAG.getBuildVector(VT: MVT::v2i16, DL,
12502 Ops: {Elts[I * 2], DAG.getPOISON(VT: MVT::i16)});
12503 SDValue IntPair = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Pair);
12504 PackedElts.push_back(Elt: IntPair);
12505 }
12506
12507 // Pad using UNDEF
12508 PackedElts.resize(N: Elts.size(), NV: DAG.getPOISON(VT: MVT::i32));
12509
12510 // Build final vector
12511 EVT VecVT =
12512 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: PackedElts.size());
12513 return DAG.getBuildVector(VT: VecVT, DL, Ops: PackedElts);
12514 }
12515
12516 if (NumElements == 3) {
12517 EVT IntStoreVT =
12518 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StoreVT.getStoreSizeInBits());
12519 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
12520
12521 EVT WidenedStoreVT = EVT::getVectorVT(
12522 Context&: *DAG.getContext(), VT: StoreVT.getVectorElementType(), NumElements: NumElements + 1);
12523 EVT WidenedIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
12524 BitWidth: WidenedStoreVT.getStoreSizeInBits());
12525 SDValue ZExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenedIntVT, Operand: IntVData);
12526 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WidenedStoreVT, Operand: ZExt);
12527 }
12528
12529 assert(isTypeLegal(StoreVT));
12530 return VData;
12531}
12532
12533static bool isAsyncLDSDMA(Intrinsic::ID Intr) {
12534 switch (Intr) {
12535 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
12536 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
12537 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
12538 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds:
12539 case Intrinsic::amdgcn_load_async_to_lds:
12540 case Intrinsic::amdgcn_global_load_async_lds:
12541 return true;
12542 }
12543 return false;
12544}
12545
12546SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
12547 SelectionDAG &DAG) const {
12548 SDLoc DL(Op);
12549 SDValue Chain = Op.getOperand(i: 0);
12550 unsigned IntrinsicID = Op.getConstantOperandVal(i: 1);
12551
12552 switch (IntrinsicID) {
12553 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
12554 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
12555 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
12556 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
12557 if (Subtarget->hasGFX1250_STRICT())
12558 initializeM0ToZeroForClusterLoad(Op, DAG, DL);
12559 return SDValue();
12560 }
12561 case Intrinsic::amdgcn_exp_compr: {
12562 SDValue Src0 = Op.getOperand(i: 4);
12563 SDValue Src1 = Op.getOperand(i: 5);
12564 // Hack around illegal type on SI by directly selecting it.
12565 if (isTypeLegal(VT: Src0.getValueType()))
12566 return SDValue();
12567
12568 const ConstantSDNode *Done = cast<ConstantSDNode>(Val: Op.getOperand(i: 6));
12569 SDValue Undef = DAG.getPOISON(VT: MVT::f32);
12570 const SDValue Ops[] = {
12571 Op.getOperand(i: 2), // tgt
12572 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: Src0), // src0
12573 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: Src1), // src1
12574 Undef, // src2
12575 Undef, // src3
12576 Op.getOperand(i: 7), // vm
12577 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // compr
12578 Op.getOperand(i: 3), // en
12579 Op.getOperand(i: 0) // Chain
12580 };
12581
12582 unsigned Opc = Done->isZero() ? AMDGPU::EXP : AMDGPU::EXP_DONE;
12583 return SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops), 0);
12584 }
12585
12586 case Intrinsic::amdgcn_struct_tbuffer_store:
12587 case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
12588 SDValue VData = Op.getOperand(i: 2);
12589 bool IsD16 = (VData.getValueType().getScalarSizeInBits() == 16);
12590 if (IsD16)
12591 VData = handleD16VData(VData, DAG);
12592 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
12593 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
12594 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
12595 SDValue Ops[] = {
12596 Chain,
12597 VData, // vdata
12598 Rsrc, // rsrc
12599 Op.getOperand(i: 4), // vindex
12600 VOffset, // voffset
12601 SOffset, // soffset
12602 Offset, // offset
12603 Op.getOperand(i: 7), // format
12604 Op.getOperand(i: 8), // cachepolicy, swizzled buffer
12605 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
12606 };
12607 unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16
12608 : AMDGPUISD::TBUFFER_STORE_FORMAT;
12609 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12610 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12611 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12612 }
12613
12614 case Intrinsic::amdgcn_raw_tbuffer_store:
12615 case Intrinsic::amdgcn_raw_ptr_tbuffer_store: {
12616 SDValue VData = Op.getOperand(i: 2);
12617 bool IsD16 = (VData.getValueType().getScalarSizeInBits() == 16);
12618 if (IsD16)
12619 VData = handleD16VData(VData, DAG);
12620 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
12621 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
12622 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
12623 SDValue Ops[] = {
12624 Chain,
12625 VData, // vdata
12626 Rsrc, // rsrc
12627 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
12628 VOffset, // voffset
12629 SOffset, // soffset
12630 Offset, // offset
12631 Op.getOperand(i: 6), // format
12632 Op.getOperand(i: 7), // cachepolicy, swizzled buffer
12633 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
12634 };
12635 unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16
12636 : AMDGPUISD::TBUFFER_STORE_FORMAT;
12637 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12638 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12639 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12640 }
12641
12642 case Intrinsic::amdgcn_raw_buffer_store:
12643 case Intrinsic::amdgcn_raw_ptr_buffer_store:
12644 case Intrinsic::amdgcn_raw_buffer_store_format:
12645 case Intrinsic::amdgcn_raw_ptr_buffer_store_format: {
12646 const bool IsFormat =
12647 IntrinsicID == Intrinsic::amdgcn_raw_buffer_store_format ||
12648 IntrinsicID == Intrinsic::amdgcn_raw_ptr_buffer_store_format;
12649
12650 SDValue VData = Op.getOperand(i: 2);
12651 EVT VDataVT = VData.getValueType();
12652 EVT EltType = VDataVT.getScalarType();
12653 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
12654
12655 if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
12656 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
12657 DAG.getMachineFunction().getFunction(),
12658 "unsupported sub-dword format buffer store", DL.getDebugLoc()));
12659 return Chain;
12660 }
12661
12662 if (IsD16) {
12663 VData = handleD16VData(VData, DAG);
12664 VDataVT = VData.getValueType();
12665 }
12666
12667 if (!isTypeLegal(VT: VDataVT)) {
12668 VData =
12669 DAG.getNode(Opcode: ISD::BITCAST, DL,
12670 VT: getEquivalentMemType(Context&: *DAG.getContext(), VT: VDataVT), Operand: VData);
12671 }
12672
12673 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
12674 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
12675 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
12676 SDValue Ops[] = {
12677 Chain,
12678 VData,
12679 Rsrc,
12680 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
12681 VOffset, // voffset
12682 SOffset, // soffset
12683 Offset, // offset
12684 Op.getOperand(i: 6), // cachepolicy, swizzled buffer
12685 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
12686 };
12687 unsigned Opc =
12688 IsFormat ? AMDGPUISD::BUFFER_STORE_FORMAT : AMDGPUISD::BUFFER_STORE;
12689 Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 : Opc;
12690 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12691
12692 // Handle BUFFER_STORE_BYTE/SHORT overloaded intrinsics
12693 if (!IsD16 && !VDataVT.isVector() && EltType.getSizeInBits() < 32)
12694 return handleByteShortBufferStores(DAG, VDataType: VDataVT, DL, Ops, M);
12695
12696 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12697 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12698 }
12699
12700 case Intrinsic::amdgcn_struct_buffer_store:
12701 case Intrinsic::amdgcn_struct_ptr_buffer_store:
12702 case Intrinsic::amdgcn_struct_buffer_store_format:
12703 case Intrinsic::amdgcn_struct_ptr_buffer_store_format: {
12704 const bool IsFormat =
12705 IntrinsicID == Intrinsic::amdgcn_struct_buffer_store_format ||
12706 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_store_format;
12707
12708 SDValue VData = Op.getOperand(i: 2);
12709 EVT VDataVT = VData.getValueType();
12710 EVT EltType = VDataVT.getScalarType();
12711 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
12712
12713 if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
12714 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
12715 DAG.getMachineFunction().getFunction(),
12716 "unsupported sub-dword format buffer store", DL.getDebugLoc()));
12717 return Chain;
12718 }
12719
12720 if (IsD16) {
12721 VData = handleD16VData(VData, DAG);
12722 VDataVT = VData.getValueType();
12723 }
12724
12725 if (!isTypeLegal(VT: VDataVT)) {
12726 VData =
12727 DAG.getNode(Opcode: ISD::BITCAST, DL,
12728 VT: getEquivalentMemType(Context&: *DAG.getContext(), VT: VDataVT), Operand: VData);
12729 }
12730
12731 auto Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
12732 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
12733 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
12734 SDValue Ops[] = {
12735 Chain,
12736 VData,
12737 Rsrc,
12738 Op.getOperand(i: 4), // vindex
12739 VOffset, // voffset
12740 SOffset, // soffset
12741 Offset, // offset
12742 Op.getOperand(i: 7), // cachepolicy, swizzled buffer
12743 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
12744 };
12745 unsigned Opc =
12746 !IsFormat ? AMDGPUISD::BUFFER_STORE : AMDGPUISD::BUFFER_STORE_FORMAT;
12747 Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 : Opc;
12748 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12749
12750 // Handle BUFFER_STORE_BYTE/SHORT overloaded intrinsics
12751 EVT VDataType = VData.getValueType().getScalarType();
12752 if (!IsD16 && !VDataVT.isVector() && EltType.getSizeInBits() < 32)
12753 return handleByteShortBufferStores(DAG, VDataType, DL, Ops, M);
12754
12755 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12756 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12757 }
12758 case Intrinsic::amdgcn_raw_buffer_load_lds:
12759 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
12760 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
12761 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
12762 case Intrinsic::amdgcn_struct_buffer_load_lds:
12763 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
12764 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
12765 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
12766 unsigned Opc;
12767 bool HasVIndex =
12768 IntrinsicID == Intrinsic::amdgcn_struct_buffer_load_lds ||
12769 IntrinsicID == Intrinsic::amdgcn_struct_buffer_load_async_lds ||
12770 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_load_lds ||
12771 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds;
12772 unsigned OpOffset = HasVIndex ? 1 : 0;
12773 SDValue VOffset = Op.getOperand(i: 5 + OpOffset);
12774 bool HasVOffset = !isNullConstant(V: VOffset);
12775 unsigned Size = Op->getConstantOperandVal(Num: 4);
12776
12777 switch (Size) {
12778 default:
12779 return SDValue();
12780 case 1:
12781 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
12782 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
12783 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
12784 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
12785 break;
12786 case 2:
12787 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
12788 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
12789 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
12790 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
12791 break;
12792 case 4:
12793 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
12794 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
12795 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
12796 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
12797 break;
12798 case 12:
12799 if (!Subtarget->hasLDSLoadB96_B128())
12800 return SDValue();
12801 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
12802 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
12803 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
12804 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
12805 break;
12806 case 16:
12807 if (!Subtarget->hasLDSLoadB96_B128())
12808 return SDValue();
12809 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
12810 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
12811 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
12812 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
12813 break;
12814 }
12815
12816 SDValue M0Val = copyToM0(DAG, Chain, DL, V: Op.getOperand(i: 3));
12817
12818 SmallVector<SDValue, 8> Ops;
12819
12820 if (HasVIndex && HasVOffset)
12821 Ops.push_back(Elt: DAG.getBuildVector(VT: MVT::v2i32, DL,
12822 Ops: {Op.getOperand(i: 5), // VIndex
12823 VOffset}));
12824 else if (HasVIndex)
12825 Ops.push_back(Elt: Op.getOperand(i: 5));
12826 else if (HasVOffset)
12827 Ops.push_back(Elt: VOffset);
12828
12829 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
12830 Ops.push_back(Elt: Rsrc);
12831 Ops.push_back(Elt: Op.getOperand(i: 6 + OpOffset)); // soffset
12832 Ops.push_back(Elt: Op.getOperand(i: 7 + OpOffset)); // imm offset
12833 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
12834 unsigned Aux = Op.getConstantOperandVal(i: 8 + OpOffset);
12835 Ops.push_back(Elt: DAG.getTargetConstant(
12836 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12),
12837 DL, VT: MVT::i8)); // cpol
12838 Ops.push_back(Elt: DAG.getTargetConstant(
12839 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::SWZ : AMDGPU::CPol::SWZ_pregfx12)
12840 ? 1
12841 : 0,
12842 DL, VT: MVT::i8)); // swz
12843 Ops.push_back(
12844 Elt: DAG.getTargetConstant(Val: isAsyncLDSDMA(Intr: IntrinsicID), DL, VT: MVT::i8));
12845 Ops.push_back(Elt: M0Val.getValue(R: 0)); // Chain
12846 Ops.push_back(Elt: M0Val.getValue(R: 1)); // Glue
12847
12848 auto *M = cast<MemSDNode>(Val&: Op);
12849 auto *Load = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: M->getVTList(), Ops);
12850 DAG.setNodeMemRefs(N: Load, NewMemRefs: M->memoperands());
12851
12852 return SDValue(Load, 0);
12853 }
12854 // Buffers are handled by LowerBufferFatPointers, and we're going to go
12855 // for "trust me" that the remaining cases are global pointers until
12856 // such time as we can put two mem operands on an intrinsic.
12857 case Intrinsic::amdgcn_load_to_lds:
12858 case Intrinsic::amdgcn_load_async_to_lds:
12859 case Intrinsic::amdgcn_global_load_lds:
12860 case Intrinsic::amdgcn_global_load_async_lds: {
12861 if (!Subtarget->hasVMemToLDSLoad())
12862 return SDValue();
12863
12864 unsigned Opc;
12865 unsigned Size = Op->getConstantOperandVal(Num: 4);
12866 switch (Size) {
12867 default:
12868 return SDValue();
12869 case 1:
12870 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
12871 break;
12872 case 2:
12873 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
12874 break;
12875 case 4:
12876 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
12877 break;
12878 case 12:
12879 if (!Subtarget->hasLDSLoadB96_B128())
12880 return SDValue();
12881 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
12882 break;
12883 case 16:
12884 if (!Subtarget->hasLDSLoadB96_B128())
12885 return SDValue();
12886 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
12887 break;
12888 }
12889
12890 SDValue M0Val = copyToM0(DAG, Chain, DL, V: Op.getOperand(i: 3));
12891
12892 SmallVector<SDValue, 6> Ops;
12893
12894 SDValue Addr = Op.getOperand(i: 2); // Global ptr
12895 SDValue VOffset;
12896 // Try to split SAddr and VOffset. Global and LDS pointers share the same
12897 // immediate offset, so we cannot use a regular SelectGlobalSAddr().
12898 if (Addr->isDivergent() && Addr->isAnyAdd()) {
12899 SDValue LHS = Addr.getOperand(i: 0);
12900 SDValue RHS = Addr.getOperand(i: 1);
12901
12902 if (LHS->isDivergent())
12903 std::swap(a&: LHS, b&: RHS);
12904
12905 if (!LHS->isDivergent() && RHS.getOpcode() == ISD::ZERO_EXTEND &&
12906 RHS.getOperand(i: 0).getValueType() == MVT::i32) {
12907 // add (i64 sgpr), (zero_extend (i32 vgpr))
12908 Addr = LHS;
12909 VOffset = RHS.getOperand(i: 0);
12910 }
12911 }
12912
12913 Ops.push_back(Elt: Addr);
12914 if (!Addr->isDivergent()) {
12915 Opc = AMDGPU::getGlobalSaddrOp(Opcode: Opc);
12916 if (!VOffset)
12917 VOffset =
12918 SDValue(DAG.getMachineNode(Opcode: AMDGPU::V_MOV_B32_e32, dl: DL, VT: MVT::i32,
12919 Op1: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32)),
12920 0);
12921 Ops.push_back(Elt: VOffset);
12922 }
12923
12924 Ops.push_back(Elt: Op.getOperand(i: 5)); // Offset
12925
12926 unsigned Aux = Op.getConstantOperandVal(i: 6);
12927 Ops.push_back(Elt: DAG.getTargetConstant(Val: Aux & ~AMDGPU::CPol::VIRTUAL_BITS, DL,
12928 VT: MVT::i32)); // CPol
12929 Ops.push_back(
12930 Elt: DAG.getTargetConstant(Val: isAsyncLDSDMA(Intr: IntrinsicID), DL, VT: MVT::i8));
12931
12932 Ops.push_back(Elt: M0Val.getValue(R: 0)); // Chain
12933 Ops.push_back(Elt: M0Val.getValue(R: 1)); // Glue
12934
12935 auto *M = cast<MemSDNode>(Val&: Op);
12936 auto *Load = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
12937 DAG.setNodeMemRefs(N: Load, NewMemRefs: M->memoperands());
12938
12939 return SDValue(Load, 0);
12940 }
12941 case Intrinsic::amdgcn_end_cf:
12942 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::SI_END_CF, dl: DL, VT: MVT::Other,
12943 Op1: Op->getOperand(Num: 2), Op2: Chain),
12944 0);
12945 case Intrinsic::amdgcn_s_barrier_signal_var: {
12946 // Member count of 0 means to re-use a previous member count,
12947 // which, if the named barrier is statically chosen, means we can use
12948 // the immarg form. Otherwisee, fall through to constructiong M0 as for
12949 // s_barrier_init.
12950 SDValue CntOp = Op->getOperand(Num: 3);
12951 auto *CntC = dyn_cast<ConstantSDNode>(Val&: CntOp);
12952 if (CntC && CntC->isZero()) {
12953 SDValue Chain = Op->getOperand(Num: 0);
12954 SDValue BarOp = Op->getOperand(Num: 2);
12955 SmallVector<SDValue, 2> Ops;
12956
12957 std::optional<uint64_t> BarVal;
12958 if (auto *C = dyn_cast<ConstantSDNode>(Val&: BarOp))
12959 BarVal = C->getZExtValue();
12960 else if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: BarOp))
12961 if (auto Addr = AMDGPUMachineFunctionInfo::getLDSAbsoluteAddress(
12962 GV: *GA->getGlobal()))
12963 BarVal = *Addr + GA->getOffset();
12964
12965 if (BarVal) {
12966 unsigned BarID = (*BarVal >> 4) & 0x3F;
12967 Ops.push_back(Elt: DAG.getTargetConstant(Val: BarID, DL, VT: MVT::i32));
12968 Ops.push_back(Elt: Chain);
12969 auto *NewMI = DAG.getMachineNode(Opcode: AMDGPU::S_BARRIER_SIGNAL_IMM, dl: DL,
12970 VTs: Op->getVTList(), Ops);
12971 return SDValue(NewMI, 0);
12972 }
12973 }
12974 [[fallthrough]];
12975 }
12976 case Intrinsic::amdgcn_s_barrier_init: {
12977 // these two intrinsics have two operands: barrier pointer and member count
12978 SDValue Chain = Op->getOperand(Num: 0);
12979 SmallVector<SDValue, 2> Ops;
12980 SDValue BarOp = Op->getOperand(Num: 2);
12981 SDValue CntOp = Op->getOperand(Num: 3);
12982 SDValue M0Val;
12983 unsigned Opc = IntrinsicID == Intrinsic::amdgcn_s_barrier_init
12984 ? AMDGPU::S_BARRIER_INIT_M0
12985 : AMDGPU::S_BARRIER_SIGNAL_M0;
12986 // extract the BarrierID from bits 4-9 of BarOp
12987 SDValue BarID;
12988 BarID = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: BarOp,
12989 N2: DAG.getShiftAmountConstant(Val: 4, VT: MVT::i32, DL));
12990 BarID = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: BarID,
12991 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
12992 // Member count should be put into M0[ShAmt:+6]
12993 // Barrier ID should be put into M0[5:0]
12994 SDValue MemberCnt = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: CntOp,
12995 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
12996 constexpr unsigned ShAmt = 16;
12997 M0Val = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: MemberCnt,
12998 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT: MVT::i32, DL));
12999
13000 M0Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M0Val, N2: BarID);
13001
13002 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
13003
13004 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
13005 return SDValue(NewMI, 0);
13006 }
13007 case Intrinsic::amdgcn_s_wakeup_barrier: {
13008 if (!Subtarget->hasSWakeupBarrier())
13009 return SDValue();
13010 [[fallthrough]];
13011 }
13012 case Intrinsic::amdgcn_s_barrier_join: {
13013 // these three intrinsics have one operand: barrier pointer
13014 SDValue Chain = Op->getOperand(Num: 0);
13015 SmallVector<SDValue, 2> Ops;
13016 SDValue BarOp = Op->getOperand(Num: 2);
13017 unsigned Opc;
13018
13019 if (isa<ConstantSDNode>(Val: BarOp)) {
13020 uint64_t BarVal = cast<ConstantSDNode>(Val&: BarOp)->getZExtValue();
13021 switch (IntrinsicID) {
13022 default:
13023 return SDValue();
13024 case Intrinsic::amdgcn_s_barrier_join:
13025 Opc = AMDGPU::S_BARRIER_JOIN_IMM;
13026 break;
13027 case Intrinsic::amdgcn_s_wakeup_barrier:
13028 Opc = AMDGPU::S_WAKEUP_BARRIER_IMM;
13029 break;
13030 }
13031 // extract the BarrierID from bits 4-9 of the immediate
13032 unsigned BarID = (BarVal >> 4) & 0x3F;
13033 SDValue K = DAG.getTargetConstant(Val: BarID, DL, VT: MVT::i32);
13034 Ops.push_back(Elt: K);
13035 Ops.push_back(Elt: Chain);
13036 } else {
13037 switch (IntrinsicID) {
13038 default:
13039 return SDValue();
13040 case Intrinsic::amdgcn_s_barrier_join:
13041 Opc = AMDGPU::S_BARRIER_JOIN_M0;
13042 break;
13043 case Intrinsic::amdgcn_s_wakeup_barrier:
13044 Opc = AMDGPU::S_WAKEUP_BARRIER_M0;
13045 break;
13046 }
13047 // extract the BarrierID from bits 4-9 of BarOp, copy to M0[5:0]
13048 SDValue M0Val;
13049 M0Val = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: BarOp,
13050 N2: DAG.getShiftAmountConstant(Val: 4, VT: MVT::i32, DL));
13051 M0Val = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: M0Val,
13052 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
13053 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
13054 }
13055
13056 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
13057 return SDValue(NewMI, 0);
13058 }
13059 case Intrinsic::amdgcn_s_prefetch_data:
13060 case Intrinsic::amdgcn_s_prefetch_inst: {
13061 // For non-global address space preserve the chain and remove the call.
13062 if (!AMDGPU::isFlatGlobalAddrSpace(AS: cast<MemSDNode>(Val&: Op)->getAddressSpace()))
13063 return Op.getOperand(i: 0);
13064 return Op;
13065 }
13066 case Intrinsic::amdgcn_s_buffer_prefetch_data: {
13067 SDValue Ops[] = {
13068 Chain, bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG),
13069 Op.getOperand(i: 3), // offset
13070 Op.getOperand(i: 4), // length
13071 };
13072
13073 MemSDNode *M = cast<MemSDNode>(Val&: Op);
13074 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::SBUFFER_PREFETCH_DATA, dl: DL,
13075 VTList: Op->getVTList(), Ops, MemVT: M->getMemoryVT(),
13076 MMO: M->getMemOperand());
13077 }
13078 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
13079 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
13080 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
13081 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
13082 SDValue Chain = Op->getOperand(Num: 0);
13083 SDValue Ptr = Op->getOperand(Num: 2);
13084 SDValue Val = Op->getOperand(Num: 3);
13085 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MII->getMemoryVT(), Chain, Ptr: Val,
13086 Val: Ptr, MMO: MII->getMemOperand());
13087 }
13088 case Intrinsic::amdgcn_av_store_b128: {
13089 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
13090 SDValue Chain = Op->getOperand(Num: 0);
13091 SDValue Ptr = Op->getOperand(Num: 2);
13092 SDValue Val = Op->getOperand(Num: 3);
13093 return DAG.getStore(Chain, dl: DL, Val, Ptr, MMO: MII->getMemOperand());
13094 }
13095 default: {
13096 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
13097 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrinsicID))
13098 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: true);
13099
13100 return Op;
13101 }
13102 }
13103}
13104
13105// Return whether the operation has NoUnsignedWrap property.
13106static bool isNoUnsignedWrap(SDValue Addr) {
13107 return (Addr.getOpcode() == ISD::ADD &&
13108 Addr->getFlags().hasNoUnsignedWrap()) ||
13109 Addr->getOpcode() == ISD::OR;
13110}
13111
13112bool SITargetLowering::shouldPreservePtrArith(const Function &F,
13113 EVT PtrVT) const {
13114 return PtrVT == MVT::i64;
13115}
13116
13117bool SITargetLowering::canTransformPtrArithOutOfBounds(const Function &F,
13118 EVT PtrVT) const {
13119 return true;
13120}
13121
13122// The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
13123// offset (the offset that is included in bounds checking and swizzling, to be
13124// split between the instruction's voffset and immoffset fields) and soffset
13125// (the offset that is excluded from bounds checking and swizzling, to go in
13126// the instruction's soffset field). This function takes the first kind of
13127// offset and figures out how to split it between voffset and immoffset.
13128std::pair<SDValue, SDValue>
13129SITargetLowering::splitBufferOffsets(SDValue Offset, SelectionDAG &DAG) const {
13130 SDLoc DL(Offset);
13131 const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(ST: *Subtarget);
13132 SDValue N0 = Offset;
13133 ConstantSDNode *C1 = nullptr;
13134
13135 if ((C1 = dyn_cast<ConstantSDNode>(Val&: N0)))
13136 N0 = SDValue();
13137 else if (DAG.isBaseWithConstantOffset(Op: N0)) {
13138 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
13139 // being added, so we can only safely match a 32-bit addition with no
13140 // unsigned overflow.
13141 bool CheckNUW = Subtarget->hasGFX1250Insts();
13142 if (!CheckNUW || isNoUnsignedWrap(Addr: N0)) {
13143 C1 = cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
13144 N0 = N0.getOperand(i: 0);
13145 }
13146 }
13147
13148 if (C1) {
13149 unsigned ImmOffset = C1->getZExtValue();
13150 // If the immediate value is too big for the immoffset field, put only bits
13151 // that would normally fit in the immoffset field. The remaining value that
13152 // is copied/added for the voffset field is a large power of 2, and it
13153 // stands more chance of being CSEd with the copy/add for another similar
13154 // load/store.
13155 // However, do not do that rounding down if that is a negative
13156 // number, as it appears to be illegal to have a negative offset in the
13157 // vgpr, even if adding the immediate offset makes it positive.
13158 unsigned Overflow = ImmOffset & ~MaxImm;
13159 ImmOffset -= Overflow;
13160 if ((int32_t)Overflow < 0) {
13161 Overflow += ImmOffset;
13162 ImmOffset = 0;
13163 }
13164 C1 = cast<ConstantSDNode>(Val: DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32));
13165 if (Overflow) {
13166 auto OverflowVal = DAG.getConstant(Val: Overflow, DL, VT: MVT::i32);
13167 if (!N0)
13168 N0 = OverflowVal;
13169 else {
13170 SDValue Ops[] = {N0, OverflowVal};
13171 N0 = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, Ops);
13172 }
13173 }
13174 }
13175 if (!N0)
13176 N0 = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13177 if (!C1)
13178 C1 = cast<ConstantSDNode>(Val: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
13179 return {N0, SDValue(C1, 0)};
13180}
13181
13182// Analyze a combined offset from an amdgcn_s_buffer_load intrinsic and store
13183// the three offsets (voffset, soffset and instoffset) into the SDValue[3] array
13184// pointed to by Offsets.
13185void SITargetLowering::setBufferOffsets(SDValue CombinedOffset,
13186 SelectionDAG &DAG, SDValue *Offsets,
13187 Align Alignment) const {
13188 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
13189 SDLoc DL(CombinedOffset);
13190 if (auto *C = dyn_cast<ConstantSDNode>(Val&: CombinedOffset)) {
13191 uint32_t Imm = C->getZExtValue();
13192 uint32_t SOffset, ImmOffset;
13193 if (TII->splitMUBUFOffset(Imm, SOffset, ImmOffset, Alignment)) {
13194 Offsets[0] = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13195 Offsets[1] = DAG.getConstant(Val: SOffset, DL, VT: MVT::i32);
13196 Offsets[2] = DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32);
13197 return;
13198 }
13199 }
13200 if (DAG.isBaseWithConstantOffset(Op: CombinedOffset)) {
13201 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
13202 // being added, so we can only safely match a 32-bit addition with no
13203 // unsigned overflow.
13204 bool CheckNUW = Subtarget->hasGFX1250Insts();
13205 SDValue N0 = CombinedOffset.getOperand(i: 0);
13206 SDValue N1 = CombinedOffset.getOperand(i: 1);
13207 uint32_t SOffset, ImmOffset;
13208 int Offset = cast<ConstantSDNode>(Val&: N1)->getSExtValue();
13209 if (Offset >= 0 && (!CheckNUW || isNoUnsignedWrap(Addr: CombinedOffset)) &&
13210 TII->splitMUBUFOffset(Imm: Offset, SOffset, ImmOffset, Alignment)) {
13211 Offsets[0] = N0;
13212 Offsets[1] = DAG.getConstant(Val: SOffset, DL, VT: MVT::i32);
13213 Offsets[2] = DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32);
13214 return;
13215 }
13216 }
13217
13218 SDValue SOffsetZero = Subtarget->hasRestrictedSOffset()
13219 ? DAG.getRegister(Reg: AMDGPU::SGPR_NULL, VT: MVT::i32)
13220 : DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13221
13222 Offsets[0] = CombinedOffset;
13223 Offsets[1] = SOffsetZero;
13224 Offsets[2] = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32);
13225}
13226
13227SDValue SITargetLowering::bufferRsrcPtrToVector(SDValue MaybePointer,
13228 SelectionDAG &DAG) const {
13229 if (!MaybePointer.getValueType().isScalarInteger())
13230 return MaybePointer;
13231
13232 SDValue Rsrc = DAG.getBitcast(VT: MVT::v4i32, V: MaybePointer);
13233 return Rsrc;
13234}
13235
13236// Wrap a global or flat pointer into a buffer intrinsic using the flags
13237// specified in the intrinsic.
13238SDValue SITargetLowering::lowerPointerAsRsrcIntrin(SDNode *Op,
13239 SelectionDAG &DAG) const {
13240 SDLoc Loc(Op);
13241
13242 SDValue Pointer = Op->getOperand(Num: 1);
13243 SDValue Stride = Op->getOperand(Num: 2);
13244 SDValue NumRecords = Op->getOperand(Num: 3);
13245 SDValue Flags = Op->getOperand(Num: 4);
13246
13247 SDValue ExtStride = DAG.getAnyExtOrTrunc(Op: Stride, DL: Loc, VT: MVT::i32);
13248 SDValue Rsrc;
13249
13250 if (Subtarget->getBufferResourceNumRecordsWidth() == 45) {
13251 NumRecords = DAG.getZExtOrTrunc(Op: NumRecords, DL: Loc, VT: MVT::i64);
13252 NumRecords = DAG.getNode(Opcode: ISD::AND, DL: Loc, VT: MVT::i64, N1: NumRecords,
13253 N2: DAG.getConstant(Val: (1ULL << 45) - 1, DL: Loc, VT: MVT::i64));
13254 SDValue Zero = DAG.getConstant(Val: 0, DL: Loc, VT: MVT::i32);
13255 // Build the lower 64-bit value, which has a 57-bit base and the lower 7-bit
13256 // num_records.
13257 SDValue ExtPointer = DAG.getAnyExtOrTrunc(Op: Pointer, DL: Loc, VT: MVT::i64);
13258 SDValue NumRecordsLHS =
13259 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i64, N1: NumRecords,
13260 N2: DAG.getShiftAmountConstant(Val: 57, VT: MVT::i32, DL: Loc));
13261 SDValue LowHalf =
13262 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: ExtPointer, N2: NumRecordsLHS);
13263
13264 // Build the higher 64-bit value, which has the higher 38-bit num_records,
13265 // 6-bit zero (omit), 16-bit stride and scale and 4-bit flag.
13266 SDValue NumRecordsRHS =
13267 DAG.getNode(Opcode: ISD::SRL, DL: Loc, VT: MVT::i64, N1: NumRecords,
13268 N2: DAG.getShiftAmountConstant(Val: 7, VT: MVT::i32, DL: Loc));
13269 SDValue ShiftedStride =
13270 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: ExtStride,
13271 N2: DAG.getShiftAmountConstant(Val: 12, VT: MVT::i32, DL: Loc));
13272 SDValue ExtShiftedStrideVec =
13273 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i32, N1: Zero, N2: ShiftedStride);
13274 SDValue ExtShiftedStride =
13275 DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i64, Operand: ExtShiftedStrideVec);
13276 SDValue ShiftedFlags =
13277 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: Flags,
13278 N2: DAG.getShiftAmountConstant(Val: 28, VT: MVT::i32, DL: Loc));
13279 SDValue ExtShiftedFlagsVec =
13280 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i32, N1: Zero, N2: ShiftedFlags);
13281 SDValue ExtShiftedFlags =
13282 DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i64, Operand: ExtShiftedFlagsVec);
13283 SDValue CombinedFields =
13284 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: NumRecordsRHS, N2: ExtShiftedStride);
13285 SDValue HighHalf =
13286 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: CombinedFields, N2: ExtShiftedFlags);
13287
13288 Rsrc = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i64, N1: LowHalf, N2: HighHalf);
13289 } else {
13290 NumRecords = DAG.getZExtOrTrunc(Op: NumRecords, DL: Loc, VT: MVT::i32);
13291 auto [LowHalf, HighHalf] =
13292 DAG.SplitScalar(N: Pointer, DL: Loc, LoVT: MVT::i32, HiVT: MVT::i32);
13293 SDValue Mask = DAG.getConstant(Val: 0x0000ffff, DL: Loc, VT: MVT::i32);
13294 SDValue Masked = DAG.getNode(Opcode: ISD::AND, DL: Loc, VT: MVT::i32, N1: HighHalf, N2: Mask);
13295 SDValue ShiftedStride =
13296 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: ExtStride,
13297 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: Loc));
13298 SDValue NewHighHalf =
13299 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i32, N1: Masked, N2: ShiftedStride);
13300
13301 Rsrc = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v4i32, N1: LowHalf, N2: NewHighHalf,
13302 N3: NumRecords, N4: Flags);
13303 }
13304
13305 SDValue RsrcPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i128, Operand: Rsrc);
13306 return RsrcPtr;
13307}
13308
13309// Handle 8 bit and 16 bit buffer loads
13310SDValue SITargetLowering::handleByteShortBufferLoads(SelectionDAG &DAG,
13311 EVT LoadVT, SDLoc DL,
13312 ArrayRef<SDValue> Ops,
13313 MachineMemOperand *MMO,
13314 bool IsTFE) const {
13315 EVT IntVT = LoadVT.changeTypeToInteger();
13316
13317 if (IsTFE) {
13318 unsigned Opc = (LoadVT.getScalarType() == MVT::i8)
13319 ? AMDGPUISD::BUFFER_LOAD_UBYTE_TFE
13320 : AMDGPUISD::BUFFER_LOAD_USHORT_TFE;
13321 MachineFunction &MF = DAG.getMachineFunction();
13322 MachineMemOperand *OpMMO = MF.getMachineMemOperand(MMO, Offset: 0, Size: 8);
13323 SDVTList VTs = DAG.getVTList(VT1: MVT::v2i32, VT2: MVT::Other);
13324 SDValue Op = getMemIntrinsicNode(Opcode: Opc, DL, VTList: VTs, Ops, MemVT: MVT::v2i32, MMO: OpMMO, DAG);
13325 SDValue Status = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
13326 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
13327 SDValue Data = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
13328 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
13329 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: Data);
13330 SDValue Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: Trunc);
13331 return DAG.getMergeValues(Ops: {Value, Status, SDValue(Op.getNode(), 1)}, dl: DL);
13332 }
13333
13334 unsigned Opc = LoadVT.getScalarType() == MVT::i8
13335 ? AMDGPUISD::BUFFER_LOAD_UBYTE
13336 : AMDGPUISD::BUFFER_LOAD_USHORT;
13337
13338 SDVTList ResList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
13339 SDValue BufferLoad =
13340 DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: ResList, Ops, MemVT: IntVT, MMO);
13341 SDValue LoadVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: BufferLoad);
13342 LoadVal = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: LoadVal);
13343
13344 return DAG.getMergeValues(Ops: {LoadVal, BufferLoad.getValue(R: 1)}, dl: DL);
13345}
13346
13347// Handle 8 bit and 16 bit buffer stores
13348SDValue SITargetLowering::handleByteShortBufferStores(SelectionDAG &DAG,
13349 EVT VDataType, SDLoc DL,
13350 SDValue Ops[],
13351 MemSDNode *M) const {
13352 if (VDataType == MVT::f16 || VDataType == MVT::bf16)
13353 Ops[1] = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Ops[1]);
13354
13355 SDValue BufferStoreExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Ops[1]);
13356 Ops[1] = BufferStoreExt;
13357 unsigned Opc = (VDataType == MVT::i8) ? AMDGPUISD::BUFFER_STORE_BYTE
13358 : AMDGPUISD::BUFFER_STORE_SHORT;
13359 ArrayRef<SDValue> OpsRef = ArrayRef(&Ops[0], 9);
13360 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: M->getVTList(), Ops: OpsRef, MemVT: VDataType,
13361 MMO: M->getMemOperand());
13362}
13363
13364static SDValue getLoadExtOrTrunc(SelectionDAG &DAG, ISD::LoadExtType ExtType,
13365 SDValue Op, const SDLoc &SL, EVT VT) {
13366 if (VT.bitsLT(VT: Op.getValueType()))
13367 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
13368
13369 switch (ExtType) {
13370 case ISD::SEXTLOAD:
13371 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SL, VT, Operand: Op);
13372 case ISD::ZEXTLOAD:
13373 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT, Operand: Op);
13374 case ISD::EXTLOAD:
13375 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT, Operand: Op);
13376 case ISD::NON_EXTLOAD:
13377 return Op;
13378 }
13379
13380 llvm_unreachable("invalid ext type");
13381}
13382
13383// Try to turn 8 and 16-bit scalar loads into SMEM eligible 32-bit loads.
13384// TODO: Skip this on GFX12 which does have scalar sub-dword loads.
13385SDValue SITargetLowering::widenLoad(LoadSDNode *Ld,
13386 DAGCombinerInfo &DCI) const {
13387 SelectionDAG &DAG = DCI.DAG;
13388 if (Ld->getAlign() < Align(4) || Ld->isDivergent())
13389 return SDValue();
13390
13391 // FIXME: Constant loads should all be marked invariant.
13392 unsigned AS = Ld->getAddressSpace();
13393 if (AS != AMDGPUAS::CONSTANT_ADDRESS &&
13394 AS != AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
13395 (AS != AMDGPUAS::GLOBAL_ADDRESS || !Ld->isInvariant()))
13396 return SDValue();
13397
13398 // Don't do this early, since it may interfere with adjacent load merging for
13399 // illegal types. We can avoid losing alignment information for exotic types
13400 // pre-legalize.
13401 EVT MemVT = Ld->getMemoryVT();
13402 if ((MemVT.isSimple() && !DCI.isAfterLegalizeDAG()) ||
13403 MemVT.getSizeInBits() >= 32)
13404 return SDValue();
13405
13406 SDLoc SL(Ld);
13407
13408 assert((!MemVT.isVector() || Ld->getExtensionType() == ISD::NON_EXTLOAD) &&
13409 "unexpected vector extload");
13410
13411 // TODO: Drop only high part of range.
13412 SDValue Ptr = Ld->getBasePtr();
13413 SDValue NewLoad = DAG.getLoad(
13414 AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT: MVT::i32, dl: SL, Chain: Ld->getChain(), Ptr,
13415 Offset: Ld->getOffset(), PtrInfo: Ld->getPointerInfo(), MemVT: MVT::i32, Alignment: Ld->getAlign(),
13416 MMOFlags: Ld->getMemOperand()->getFlags(), Metadata: Ld->getAAInfo()); // Drop ranges
13417
13418 EVT TruncVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
13419 if (MemVT.isFloatingPoint()) {
13420 assert(Ld->getExtensionType() == ISD::NON_EXTLOAD &&
13421 "unexpected fp extload");
13422 TruncVT = MemVT.changeTypeToInteger();
13423 }
13424
13425 SDValue Cvt = NewLoad;
13426 if (Ld->getExtensionType() == ISD::SEXTLOAD) {
13427 Cvt = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SL, VT: MVT::i32, N1: NewLoad,
13428 N2: DAG.getValueType(TruncVT));
13429 } else if (Ld->getExtensionType() == ISD::ZEXTLOAD ||
13430 Ld->getExtensionType() == ISD::NON_EXTLOAD) {
13431 Cvt = DAG.getZeroExtendInReg(Op: NewLoad, DL: SL, VT: TruncVT);
13432 } else {
13433 assert(Ld->getExtensionType() == ISD::EXTLOAD);
13434 }
13435
13436 EVT VT = Ld->getValueType(ResNo: 0);
13437 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
13438
13439 DCI.AddToWorklist(N: Cvt.getNode());
13440
13441 // We may need to handle exotic cases, such as i16->i64 extloads, so insert
13442 // the appropriate extension from the 32-bit load.
13443 Cvt = getLoadExtOrTrunc(DAG, ExtType: Ld->getExtensionType(), Op: Cvt, SL, VT: IntVT);
13444 DCI.AddToWorklist(N: Cvt.getNode());
13445
13446 // Handle conversion back to floating point if necessary.
13447 Cvt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Cvt);
13448
13449 return DAG.getMergeValues(Ops: {Cvt, NewLoad.getValue(R: 1)}, dl: SL);
13450}
13451
13452static bool addressMayBeAccessedAsPrivate(const MachineMemOperand *MMO,
13453 const SIMachineFunctionInfo &Info) {
13454 // TODO: Should check if the address can definitely not access stack.
13455 if (Info.isEntryFunction())
13456 return Info.getUserSGPRInfo().hasFlatScratchInit();
13457 return true;
13458}
13459
13460SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
13461 SDLoc DL(Op);
13462 LoadSDNode *Load = cast<LoadSDNode>(Val&: Op);
13463 ISD::LoadExtType ExtType = Load->getExtensionType();
13464 EVT MemVT = Load->getMemoryVT();
13465 MachineMemOperand *MMO = Load->getMemOperand();
13466
13467 if (ExtType == ISD::NON_EXTLOAD && MemVT.getSizeInBits() < 32) {
13468 if (MemVT == MVT::i16 && isTypeLegal(VT: MVT::i16))
13469 return SDValue();
13470
13471 // FIXME: Copied from PPC
13472 // First, load into 32 bits, then truncate to 1 bit.
13473
13474 SDValue Chain = Load->getChain();
13475 SDValue BasePtr = Load->getBasePtr();
13476
13477 EVT RealMemVT = (MemVT == MVT::i1) ? MVT::i8 : MVT::i16;
13478
13479 SDValue NewLD = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: MVT::i32, Chain, Ptr: BasePtr,
13480 MemVT: RealMemVT, MMO);
13481
13482 if (!MemVT.isVector()) {
13483 SDValue Ops[] = {DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MemVT, Operand: NewLD),
13484 NewLD.getValue(R: 1)};
13485
13486 return DAG.getMergeValues(Ops, dl: DL);
13487 }
13488
13489 SmallVector<SDValue, 3> Elts;
13490 for (unsigned I = 0, N = MemVT.getVectorNumElements(); I != N; ++I) {
13491 SDValue Elt = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: NewLD,
13492 N2: DAG.getConstant(Val: I, DL, VT: MVT::i32));
13493
13494 Elts.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Elt));
13495 }
13496
13497 SDValue Ops[] = {DAG.getBuildVector(VT: MemVT, DL, Ops: Elts), NewLD.getValue(R: 1)};
13498
13499 return DAG.getMergeValues(Ops, dl: DL);
13500 }
13501
13502 if (!MemVT.isVector())
13503 return SDValue();
13504
13505 assert(Op.getValueType().getVectorElementType() == MVT::i32 &&
13506 "Custom lowering for non-i32 vectors hasn't been implemented.");
13507
13508 Align Alignment = Load->getAlign();
13509 unsigned AS = Load->getAddressSpace();
13510 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
13511 AS == AMDGPUAS::FLAT_ADDRESS &&
13512 Alignment.value() < MemVT.getStoreSize() && MemVT.getSizeInBits() > 32) {
13513 return SplitVectorLoad(Op, DAG);
13514 }
13515
13516 MachineFunction &MF = DAG.getMachineFunction();
13517 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
13518 // If there is a possibility that flat instruction access scratch memory
13519 // then we need to use the same legalization rules we use for private.
13520 if (AS == AMDGPUAS::FLAT_ADDRESS &&
13521 !Subtarget->hasMultiDwordFlatScratchAddressing())
13522 AS = addressMayBeAccessedAsPrivate(MMO: Load->getMemOperand(), Info: *MFI)
13523 ? AMDGPUAS::PRIVATE_ADDRESS
13524 : AMDGPUAS::GLOBAL_ADDRESS;
13525
13526 unsigned NumElements = MemVT.getVectorNumElements();
13527
13528 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
13529 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
13530 (AS == AMDGPUAS::GLOBAL_ADDRESS &&
13531 Subtarget->getScalarizeGlobalBehavior() && Load->isSimple() &&
13532 (Load->isInvariant() || isMemOpHasNoClobberedMemOperand(N: Load)))) {
13533 if ((!Op->isDivergent() || AMDGPU::isUniformMMO(MMO)) &&
13534 Alignment >= Align(4) && NumElements < 32) {
13535 if (MemVT.isPow2VectorType() ||
13536 (Subtarget->hasScalarDwordx3Loads() && NumElements == 3))
13537 return SDValue();
13538 return WidenOrSplitVectorLoad(Op, DAG);
13539 }
13540 // Non-uniform loads will be selected to MUBUF instructions, so they
13541 // have the same legalization requirements as global and private
13542 // loads.
13543 //
13544 }
13545 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
13546 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
13547 AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS) {
13548 if (NumElements > 4)
13549 return SplitVectorLoad(Op, DAG);
13550 // v3 loads not supported on SI.
13551 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
13552 return WidenOrSplitVectorLoad(Op, DAG);
13553
13554 // v3 and v4 loads are supported for private and global memory.
13555 return SDValue();
13556 }
13557 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
13558 // Depending on the setting of the private_element_size field in the
13559 // resource descriptor, we can only make private accesses up to a certain
13560 // size.
13561 switch (Subtarget->getMaxPrivateElementSize()) {
13562 case 4: {
13563 auto [Op0, Op1] = scalarizeVectorLoad(LD: Load, DAG);
13564 return DAG.getMergeValues(Ops: {Op0, Op1}, dl: DL);
13565 }
13566 case 8:
13567 if (NumElements > 2)
13568 return SplitVectorLoad(Op, DAG);
13569 return SDValue();
13570 case 16:
13571 // Same as global/flat
13572 if (NumElements > 4)
13573 return SplitVectorLoad(Op, DAG);
13574 // v3 loads not supported on SI.
13575 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
13576 return WidenOrSplitVectorLoad(Op, DAG);
13577
13578 return SDValue();
13579 default:
13580 llvm_unreachable("unsupported private_element_size");
13581 }
13582 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
13583 unsigned Fast = 0;
13584 auto Flags = Load->getMemOperand()->getFlags();
13585 if (allowsMisalignedMemoryAccessesImpl(Size: MemVT.getSizeInBits(), AddrSpace: AS,
13586 Alignment: Load->getAlign(), Flags, IsFast: &Fast) &&
13587 Fast > 1)
13588 return SDValue();
13589
13590 if (MemVT.isVector())
13591 return SplitVectorLoad(Op, DAG);
13592 }
13593
13594 if (!allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
13595 VT: MemVT, MMO: *Load->getMemOperand())) {
13596 auto [Op0, Op1] = expandUnalignedLoad(LD: Load, DAG);
13597 return DAG.getMergeValues(Ops: {Op0, Op1}, dl: DL);
13598 }
13599
13600 return SDValue();
13601}
13602
13603SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
13604 EVT VT = Op.getValueType();
13605 if (VT.getSizeInBits() == 128 || VT.getSizeInBits() == 256 ||
13606 VT.getSizeInBits() == 512)
13607 return splitTernaryVectorOp(Op, DAG);
13608
13609 assert(VT.getSizeInBits() == 64);
13610
13611 SDLoc DL(Op);
13612 SDValue Cond = DAG.getFreeze(V: Op.getOperand(i: 0));
13613
13614 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13615 SDValue One = DAG.getConstant(Val: 1, DL, VT: MVT::i32);
13616
13617 SDValue LHS = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
13618 SDValue RHS = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 2));
13619
13620 SDValue Lo0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: LHS, N2: Zero);
13621 SDValue Lo1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: RHS, N2: Zero);
13622
13623 SDValue Lo = DAG.getSelect(DL, VT: MVT::i32, Cond, LHS: Lo0, RHS: Lo1);
13624
13625 SDValue Hi0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: LHS, N2: One);
13626 SDValue Hi1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: RHS, N2: One);
13627
13628 SDValue Hi = DAG.getSelect(DL, VT: MVT::i32, Cond, LHS: Hi0, RHS: Hi1);
13629
13630 SDValue Res = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Lo, Hi});
13631 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Res);
13632}
13633
13634// Catch division cases where we can use shortcuts with rcp and rsq
13635// instructions.
13636SDValue SITargetLowering::lowerFastUnsafeFDIV(SDValue Op,
13637 SelectionDAG &DAG) const {
13638 SDLoc SL(Op);
13639 SDValue LHS = Op.getOperand(i: 0);
13640 SDValue RHS = Op.getOperand(i: 1);
13641 EVT VT = Op.getValueType();
13642 const SDNodeFlags Flags = Op->getFlags();
13643
13644 bool AllowInaccurateRcp = Flags.hasApproximateFuncs();
13645
13646 if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
13647 // Without !fpmath accuracy information, we can't do more because we don't
13648 // know exactly whether rcp is accurate enough to meet !fpmath requirement.
13649 // f16 is always accurate enough
13650 if (!AllowInaccurateRcp && VT != MVT::f16 && VT != MVT::bf16)
13651 return SDValue();
13652
13653 if (CLHS->isOne()) {
13654 // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
13655 // the CI documentation has a worst case error of 1 ulp.
13656 // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to
13657 // use it as long as we aren't trying to use denormals.
13658 //
13659 // v_rcp_f16 and v_rsq_f16 DO support denormals and 0.51ulp.
13660
13661 // 1.0 / sqrt(x) -> rsq(x)
13662
13663 // XXX - Is afn sufficient to do this for f64? The maximum ULP
13664 // error seems really high at 2^29 ULP.
13665 // 1.0 / x -> rcp(x)
13666 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: RHS);
13667 }
13668
13669 // Same as for 1.0, but expand the sign out of the constant.
13670 if (CLHS->isMinusOne()) {
13671 // -1.0 / x -> rcp (fneg x)
13672 SDValue FNegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
13673 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: FNegRHS);
13674 }
13675 }
13676
13677 // For f16 and bf16 require afn or arcp.
13678 // For f32 require afn.
13679 if (!AllowInaccurateRcp &&
13680 ((VT != MVT::f16 && VT != MVT::bf16) || !Flags.hasAllowReciprocal()))
13681 return SDValue();
13682
13683 // Turn into multiply by the reciprocal.
13684 // x / y -> x * (1.0 / y)
13685 SDValue Recip = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: RHS);
13686 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: LHS, N2: Recip, Flags);
13687}
13688
13689SDValue SITargetLowering::lowerFastUnsafeFDIV64(SDValue Op,
13690 SelectionDAG &DAG) const {
13691 SDLoc SL(Op);
13692 SDValue X = Op.getOperand(i: 0);
13693 SDValue Y = Op.getOperand(i: 1);
13694 EVT VT = Op.getValueType();
13695 const SDNodeFlags Flags = Op->getFlags();
13696
13697 bool AllowInaccurateDiv = Flags.hasApproximateFuncs();
13698 if (!AllowInaccurateDiv)
13699 return SDValue();
13700
13701 const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: X);
13702 bool IsNegRcp = CLHS && CLHS->isMinusOne();
13703
13704 // Pull out the negation so it folds for free into the source modifiers.
13705 if (IsNegRcp)
13706 X = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
13707
13708 SDValue NegY = IsNegRcp ? Y : DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Y);
13709 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
13710
13711 SDValue R = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: Y);
13712 if (IsNegRcp)
13713 R = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: R);
13714
13715 SDValue Tmp0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: R, N3: One);
13716
13717 R = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp0, N2: R, N3: R);
13718 SDValue Tmp1 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: R, N3: One);
13719 R = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp1, N2: R, N3: R);
13720
13721 // Skip the last 2 correction terms for reciprocal.
13722 if (IsNegRcp || (CLHS && CLHS->isOne()))
13723 return R;
13724
13725 SDValue Ret = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: R);
13726 SDValue Tmp2 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: Ret, N3: X);
13727 return DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp2, N2: R, N3: Ret);
13728}
13729
13730static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
13731 EVT VT, SDValue A, SDValue B, SDValue GlueChain,
13732 SDNodeFlags Flags) {
13733 if (GlueChain->getNumValues() <= 1) {
13734 return DAG.getNode(Opcode, DL: SL, VT, N1: A, N2: B, Flags);
13735 }
13736
13737 assert(GlueChain->getNumValues() == 3);
13738
13739 SDVTList VTList = DAG.getVTList(VT1: VT, VT2: MVT::Other, VT3: MVT::Glue);
13740 switch (Opcode) {
13741 default:
13742 llvm_unreachable("no chain equivalent for opcode");
13743 case ISD::FMUL:
13744 Opcode = AMDGPUISD::FMUL_W_CHAIN;
13745 break;
13746 }
13747
13748 return DAG.getNode(Opcode, DL: SL, VTList,
13749 Ops: {GlueChain.getValue(R: 1), A, B, GlueChain.getValue(R: 2)},
13750 Flags);
13751}
13752
13753static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
13754 EVT VT, SDValue A, SDValue B, SDValue C,
13755 SDValue GlueChain, SDNodeFlags Flags) {
13756 if (GlueChain->getNumValues() <= 1) {
13757 return DAG.getNode(Opcode, DL: SL, VT, Ops: {A, B, C}, Flags);
13758 }
13759
13760 assert(GlueChain->getNumValues() == 3);
13761
13762 SDVTList VTList = DAG.getVTList(VT1: VT, VT2: MVT::Other, VT3: MVT::Glue);
13763 switch (Opcode) {
13764 default:
13765 llvm_unreachable("no chain equivalent for opcode");
13766 case ISD::FMA:
13767 Opcode = AMDGPUISD::FMA_W_CHAIN;
13768 break;
13769 }
13770
13771 return DAG.getNode(Opcode, DL: SL, VTList,
13772 Ops: {GlueChain.getValue(R: 1), A, B, C, GlueChain.getValue(R: 2)},
13773 Flags);
13774}
13775
13776SDValue SITargetLowering::LowerFDIV16(SDValue Op, SelectionDAG &DAG) const {
13777 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
13778 return FastLowered;
13779
13780 SDLoc SL(Op);
13781 EVT VT = Op.getValueType();
13782 SDValue LHS = Op.getOperand(i: 0);
13783 SDValue RHS = Op.getOperand(i: 1);
13784
13785 SDValue LHSExt = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: LHS);
13786 SDValue RHSExt = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: RHS);
13787
13788 if (VT == MVT::bf16) {
13789 SDValue ExtDiv =
13790 DAG.getNode(Opcode: ISD::FDIV, DL: SL, VT: MVT::f32, N1: LHSExt, N2: RHSExt, Flags: Op->getFlags());
13791 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::bf16, N1: ExtDiv,
13792 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32));
13793 }
13794
13795 assert(VT == MVT::f16);
13796
13797 // a32.u = opx(V_CVT_F32_F16, a.u); // CVT to F32
13798 // b32.u = opx(V_CVT_F32_F16, b.u); // CVT to F32
13799 // r32.u = opx(V_RCP_F32, b32.u); // rcp = 1 / d
13800 // q32.u = opx(V_MUL_F32, a32.u, r32.u); // q = n * rcp
13801 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
13802 // q32.u = opx(V_MAD_F32, e32.u, r32.u, q32.u); // q = n * rcp
13803 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
13804 // tmp.u = opx(V_MUL_F32, e32.u, r32.u);
13805 // tmp.u = opx(V_AND_B32, tmp.u, 0xff800000)
13806 // q32.u = opx(V_ADD_F32, tmp.u, q32.u);
13807 // q16.u = opx(V_CVT_F16_F32, q32.u);
13808 // q16.u = opx(V_DIV_FIXUP_F16, q16.u, b.u, a.u); // q = touchup(q, d, n)
13809
13810 // We will use ISD::FMA on targets that don't support ISD::FMAD.
13811 unsigned FMADOpCode =
13812 isOperationLegal(Op: ISD::FMAD, VT: MVT::f32) ? ISD::FMAD : ISD::FMA;
13813 SDValue NegRHSExt = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: RHSExt);
13814 SDValue Rcp =
13815 DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: RHSExt, Flags: Op->getFlags());
13816 SDValue Quot =
13817 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: LHSExt, N2: Rcp, Flags: Op->getFlags());
13818 SDValue Err = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: NegRHSExt, N2: Quot, N3: LHSExt,
13819 Flags: Op->getFlags());
13820 Quot = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: Err, N2: Rcp, N3: Quot, Flags: Op->getFlags());
13821 Err = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: NegRHSExt, N2: Quot, N3: LHSExt,
13822 Flags: Op->getFlags());
13823 SDValue Tmp = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: Err, N2: Rcp, Flags: Op->getFlags());
13824 SDValue TmpCast = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Tmp);
13825 TmpCast = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: TmpCast,
13826 N2: DAG.getConstant(Val: 0xff800000, DL: SL, VT: MVT::i32));
13827 Tmp = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: TmpCast);
13828 Quot = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f32, N1: Tmp, N2: Quot, Flags: Op->getFlags());
13829 SDValue RDst = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::f16, N1: Quot,
13830 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32));
13831 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f16, N1: RDst, N2: RHS, N3: LHS,
13832 Flags: Op->getFlags());
13833}
13834
13835// Faster 2.5 ULP division that does not support denormals.
13836SDValue SITargetLowering::lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const {
13837 SDNodeFlags Flags = Op->getFlags();
13838 SDLoc SL(Op);
13839 SDValue LHS = Op.getOperand(i: 1);
13840 SDValue RHS = Op.getOperand(i: 2);
13841
13842 // TODO: The combiner should probably handle elimination of redundant fabs.
13843 SDValue r1 = DAG.SignBitIsZeroFP(Op: RHS)
13844 ? RHS
13845 : DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: MVT::f32, Operand: RHS, Flags);
13846
13847 const APFloat K0Val(0x1p+96f);
13848 const SDValue K0 = DAG.getConstantFP(Val: K0Val, DL: SL, VT: MVT::f32);
13849
13850 const APFloat K1Val(0x1p-32f);
13851 const SDValue K1 = DAG.getConstantFP(Val: K1Val, DL: SL, VT: MVT::f32);
13852
13853 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f32);
13854
13855 EVT SetCCVT =
13856 getSetCCResultType(DL: DAG.getDataLayout(), Ctx&: *DAG.getContext(), VT: MVT::f32);
13857
13858 SDValue r2 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: r1, RHS: K0, Cond: ISD::SETOGT);
13859
13860 SDValue r3 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f32, N1: r2, N2: K1, N3: One, Flags);
13861
13862 r1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: RHS, N2: r3, Flags);
13863
13864 // rcp does not support denormals.
13865 SDValue r0 = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: r1, Flags);
13866
13867 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: LHS, N2: r0, Flags);
13868
13869 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: r3, N2: Mul, Flags);
13870}
13871
13872// Returns immediate value for setting the F32 denorm mode when using the
13873// S_DENORM_MODE instruction.
13874static SDValue getSPDenormModeValue(uint32_t SPDenormMode, SelectionDAG &DAG,
13875 const SIMachineFunctionInfo *Info,
13876 const GCNSubtarget *ST) {
13877 assert(ST->hasDenormModeInst() && "Requires S_DENORM_MODE");
13878 uint32_t DPDenormModeDefault = Info->getMode().fpDenormModeDPValue();
13879 uint32_t Mode = SPDenormMode | (DPDenormModeDefault << 2);
13880 return DAG.getTargetConstant(Val: Mode, DL: SDLoc(), VT: MVT::i32);
13881}
13882
13883SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const {
13884 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
13885 return FastLowered;
13886
13887 // The selection matcher assumes anything with a chain selecting to a
13888 // mayRaiseFPException machine instruction. Since we're introducing a chain
13889 // here, we need to explicitly report nofpexcept for the regular fdiv
13890 // lowering.
13891 SDNodeFlags Flags = Op->getFlags();
13892 Flags.setNoFPExcept(true);
13893
13894 SDLoc SL(Op);
13895 SDValue LHS = Op.getOperand(i: 0);
13896 SDValue RHS = Op.getOperand(i: 1);
13897
13898 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f32);
13899
13900 SDVTList ScaleVT = DAG.getVTList(VT1: MVT::f32, VT2: MVT::i1);
13901
13902 SDValue DenominatorScaled =
13903 DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, Ops: {RHS, RHS, LHS}, Flags);
13904 SDValue NumeratorScaled =
13905 DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, Ops: {LHS, RHS, LHS}, Flags);
13906
13907 // Denominator is scaled to not be denormal, so using rcp is ok.
13908 SDValue ApproxRcp =
13909 DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: DenominatorScaled, Flags);
13910 SDValue NegDivScale0 =
13911 DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: DenominatorScaled, Flags);
13912
13913 using namespace AMDGPU::Hwreg;
13914 const unsigned Denorm32Reg = HwregEncoding::encode(Values: ID_MODE, Values: 4, Values: 2);
13915 const SDValue BitField = DAG.getTargetConstant(Val: Denorm32Reg, DL: SL, VT: MVT::i32);
13916
13917 const MachineFunction &MF = DAG.getMachineFunction();
13918 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
13919 const DenormalMode DenormMode = Info->getMode().FP32Denormals;
13920
13921 const bool PreservesDenormals = DenormMode == DenormalMode::getIEEE();
13922 const bool HasDynamicDenormals =
13923 (DenormMode.Input == DenormalMode::Dynamic) ||
13924 (DenormMode.Output == DenormalMode::Dynamic);
13925
13926 SDValue SavedDenormMode;
13927
13928 if (!PreservesDenormals) {
13929 // Note we can't use the STRICT_FMA/STRICT_FMUL for the non-strict FDIV
13930 // lowering. The chain dependence is insufficient, and we need glue. We do
13931 // not need the glue variants in a strictfp function.
13932
13933 SDVTList BindParamVTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
13934
13935 SDValue Glue = DAG.getEntryNode();
13936 if (HasDynamicDenormals) {
13937 SDNode *GetReg = DAG.getMachineNode(Opcode: AMDGPU::S_GETREG_B32, dl: SL,
13938 VTs: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Glue),
13939 Ops: {BitField, Glue});
13940 SavedDenormMode = SDValue(GetReg, 0);
13941
13942 Glue = DAG.getMergeValues(
13943 Ops: {DAG.getEntryNode(), SDValue(GetReg, 0), SDValue(GetReg, 1)}, dl: SL);
13944 }
13945
13946 SDNode *EnableDenorm;
13947 if (Subtarget->hasDenormModeInst()) {
13948 const SDValue EnableDenormValue =
13949 getSPDenormModeValue(FP_DENORM_FLUSH_NONE, DAG, Info, ST: Subtarget);
13950
13951 EnableDenorm = DAG.getNode(Opcode: AMDGPUISD::DENORM_MODE, DL: SL, VTList: BindParamVTs, N1: Glue,
13952 N2: EnableDenormValue)
13953 .getNode();
13954 } else {
13955 const SDValue EnableDenormValue =
13956 DAG.getConstant(FP_DENORM_FLUSH_NONE, DL: SL, VT: MVT::i32);
13957 EnableDenorm = DAG.getMachineNode(Opcode: AMDGPU::S_SETREG_B32, dl: SL, VTs: BindParamVTs,
13958 Ops: {EnableDenormValue, BitField, Glue});
13959 }
13960
13961 SDValue Ops[3] = {NegDivScale0, SDValue(EnableDenorm, 0),
13962 SDValue(EnableDenorm, 1)};
13963
13964 NegDivScale0 = DAG.getMergeValues(Ops, dl: SL);
13965 }
13966
13967 SDValue Fma0 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0,
13968 B: ApproxRcp, C: One, GlueChain: NegDivScale0, Flags);
13969
13970 SDValue Fma1 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: Fma0, B: ApproxRcp,
13971 C: ApproxRcp, GlueChain: Fma0, Flags);
13972
13973 SDValue Mul = getFPBinOp(DAG, Opcode: ISD::FMUL, SL, VT: MVT::f32, A: NumeratorScaled, B: Fma1,
13974 GlueChain: Fma1, Flags);
13975
13976 SDValue Fma2 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0, B: Mul,
13977 C: NumeratorScaled, GlueChain: Mul, Flags);
13978
13979 SDValue Fma3 =
13980 getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: Fma2, B: Fma1, C: Mul, GlueChain: Fma2, Flags);
13981
13982 SDValue Fma4 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0, B: Fma3,
13983 C: NumeratorScaled, GlueChain: Fma3, Flags);
13984
13985 if (!PreservesDenormals) {
13986 SDNode *DisableDenorm;
13987 if (!HasDynamicDenormals && Subtarget->hasDenormModeInst()) {
13988 const SDValue DisableDenormValue = getSPDenormModeValue(
13989 FP_DENORM_FLUSH_IN_FLUSH_OUT, DAG, Info, ST: Subtarget);
13990
13991 SDVTList BindParamVTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
13992 DisableDenorm =
13993 DAG.getNode(Opcode: AMDGPUISD::DENORM_MODE, DL: SL, VTList: BindParamVTs,
13994 N1: Fma4.getValue(R: 1), N2: DisableDenormValue, N3: Fma4.getValue(R: 2))
13995 .getNode();
13996 } else {
13997 assert(HasDynamicDenormals == (bool)SavedDenormMode);
13998 const SDValue DisableDenormValue =
13999 HasDynamicDenormals
14000 ? SavedDenormMode
14001 : DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, DL: SL, VT: MVT::i32);
14002
14003 DisableDenorm = DAG.getMachineNode(
14004 Opcode: AMDGPU::S_SETREG_B32, dl: SL, VT: MVT::Other,
14005 Ops: {DisableDenormValue, BitField, Fma4.getValue(R: 1), Fma4.getValue(R: 2)});
14006 }
14007
14008 SDValue OutputChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other,
14009 N1: SDValue(DisableDenorm, 0), N2: DAG.getRoot());
14010 DAG.setRoot(OutputChain);
14011 }
14012
14013 SDValue Scale = NumeratorScaled.getValue(R: 1);
14014 SDValue Fmas = DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL: SL, VT: MVT::f32,
14015 Ops: {Fma4, Fma1, Fma3, Scale}, Flags);
14016
14017 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f32, N1: Fmas, N2: RHS, N3: LHS, Flags);
14018}
14019
14020SDValue SITargetLowering::LowerFDIV64(SDValue Op, SelectionDAG &DAG) const {
14021 if (SDValue FastLowered = lowerFastUnsafeFDIV64(Op, DAG))
14022 return FastLowered;
14023
14024 SDLoc SL(Op);
14025 SDValue X = Op.getOperand(i: 0);
14026 SDValue Y = Op.getOperand(i: 1);
14027
14028 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f64);
14029
14030 SDVTList ScaleVT = DAG.getVTList(VT1: MVT::f64, VT2: MVT::i1);
14031
14032 SDValue DivScale0 = DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, N1: Y, N2: Y, N3: X);
14033
14034 SDValue NegDivScale0 = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f64, Operand: DivScale0);
14035
14036 SDValue Rcp = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f64, Operand: DivScale0);
14037
14038 SDValue Fma0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Rcp, N3: One);
14039
14040 SDValue Fma1 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: Rcp, N2: Fma0, N3: Rcp);
14041
14042 SDValue Fma2 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Fma1, N3: One);
14043
14044 SDValue DivScale1 = DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, N1: X, N2: Y, N3: X);
14045
14046 SDValue Fma3 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: Fma1, N2: Fma2, N3: Fma1);
14047 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f64, N1: DivScale1, N2: Fma3);
14048
14049 SDValue Fma4 =
14050 DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Mul, N3: DivScale1);
14051
14052 SDValue Scale;
14053
14054 if (!Subtarget->hasUsableDivScaleConditionOutput()) {
14055 // Workaround a hardware bug on SI where the condition output from div_scale
14056 // is not usable.
14057
14058 const SDValue Hi = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
14059
14060 // Figure out if the scale to use for div_fmas.
14061 SDValue NumBC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: X);
14062 SDValue DenBC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Y);
14063 SDValue Scale0BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: DivScale0);
14064 SDValue Scale1BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: DivScale1);
14065
14066 SDValue NumHi =
14067 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: NumBC, N2: Hi);
14068 SDValue DenHi =
14069 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: DenBC, N2: Hi);
14070
14071 SDValue Scale0Hi =
14072 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Scale0BC, N2: Hi);
14073 SDValue Scale1Hi =
14074 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Scale1BC, N2: Hi);
14075
14076 SDValue CmpDen = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: DenHi, RHS: Scale0Hi, Cond: ISD::SETEQ);
14077 SDValue CmpNum = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: NumHi, RHS: Scale1Hi, Cond: ISD::SETEQ);
14078 Scale = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i1, N1: CmpNum, N2: CmpDen);
14079 } else {
14080 Scale = DivScale1.getValue(R: 1);
14081 }
14082
14083 SDValue Fmas =
14084 DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL: SL, VT: MVT::f64, N1: Fma4, N2: Fma3, N3: Mul, N4: Scale);
14085
14086 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f64, N1: Fmas, N2: Y, N3: X);
14087}
14088
14089SDValue SITargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const {
14090 EVT VT = Op.getValueType();
14091
14092 if (VT == MVT::f32)
14093 return LowerFDIV32(Op, DAG);
14094
14095 if (VT == MVT::f64)
14096 return LowerFDIV64(Op, DAG);
14097
14098 if (VT == MVT::f16 || VT == MVT::bf16)
14099 return LowerFDIV16(Op, DAG);
14100
14101 llvm_unreachable("Unexpected type for fdiv");
14102}
14103
14104SDValue SITargetLowering::LowerFFREXP(SDValue Op, SelectionDAG &DAG) const {
14105 SDLoc dl(Op);
14106 SDValue Val = Op.getOperand(i: 0);
14107 EVT VT = Val.getValueType();
14108 EVT ResultExpVT = Op->getValueType(ResNo: 1);
14109 EVT InstrExpVT = VT == MVT::f16 ? MVT::i16 : MVT::i32;
14110
14111 SDValue Mant = DAG.getNode(
14112 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT,
14113 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_frexp_mant, DL: dl, VT: MVT::i32), N2: Val);
14114
14115 SDValue Exp = DAG.getNode(
14116 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: InstrExpVT,
14117 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_frexp_exp, DL: dl, VT: MVT::i32), N2: Val);
14118
14119 if (Subtarget->hasFractBug()) {
14120 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Val);
14121 SDValue Inf =
14122 DAG.getConstantFP(Val: APFloat::getInf(Sem: VT.getFltSemantics()), DL: dl, VT);
14123
14124 SDValue IsFinite = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: Fabs, RHS: Inf, Cond: ISD::SETOLT);
14125 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: InstrExpVT);
14126 Exp = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: InstrExpVT, N1: IsFinite, N2: Exp, N3: Zero);
14127 Mant = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: IsFinite, N2: Mant, N3: Val);
14128 }
14129
14130 SDValue CastExp = DAG.getSExtOrTrunc(Op: Exp, DL: dl, VT: ResultExpVT);
14131 return DAG.getMergeValues(Ops: {Mant, CastExp}, dl);
14132}
14133
14134SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
14135 SDLoc DL(Op);
14136 StoreSDNode *Store = cast<StoreSDNode>(Val&: Op);
14137 EVT VT = Store->getMemoryVT();
14138
14139 if (VT == MVT::i1) {
14140 return DAG.getTruncStore(
14141 Chain: Store->getChain(), dl: DL,
14142 Val: DAG.getSExtOrTrunc(Op: Store->getValue(), DL, VT: MVT::i32),
14143 Ptr: Store->getBasePtr(), SVT: MVT::i1, MMO: Store->getMemOperand());
14144 }
14145
14146 assert(VT.isVector() &&
14147 Store->getValue().getValueType().getScalarType() == MVT::i32);
14148
14149 unsigned AS = Store->getAddressSpace();
14150 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
14151 AS == AMDGPUAS::FLAT_ADDRESS &&
14152 Store->getAlign().value() < VT.getStoreSize() &&
14153 VT.getSizeInBits() > 32) {
14154 return SplitVectorStore(Op, DAG);
14155 }
14156
14157 MachineFunction &MF = DAG.getMachineFunction();
14158 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
14159 // If there is a possibility that flat instruction access scratch memory
14160 // then we need to use the same legalization rules we use for private.
14161 if (AS == AMDGPUAS::FLAT_ADDRESS &&
14162 !Subtarget->hasMultiDwordFlatScratchAddressing())
14163 AS = addressMayBeAccessedAsPrivate(MMO: Store->getMemOperand(), Info: *MFI)
14164 ? AMDGPUAS::PRIVATE_ADDRESS
14165 : AMDGPUAS::GLOBAL_ADDRESS;
14166
14167 unsigned NumElements = VT.getVectorNumElements();
14168 if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS) {
14169 if (NumElements > 4)
14170 return SplitVectorStore(Op, DAG);
14171 // v3 stores not supported on SI.
14172 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
14173 return SplitVectorStore(Op, DAG);
14174
14175 if (!allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
14176 VT, MMO: *Store->getMemOperand()))
14177 return expandUnalignedStore(ST: Store, DAG);
14178
14179 return SDValue();
14180 }
14181 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
14182 switch (Subtarget->getMaxPrivateElementSize()) {
14183 case 4:
14184 return scalarizeVectorStore(ST: Store, DAG);
14185 case 8:
14186 if (NumElements > 2)
14187 return SplitVectorStore(Op, DAG);
14188 return SDValue();
14189 case 16:
14190 if (NumElements > 4 ||
14191 (NumElements == 3 && !Subtarget->hasFlatScratchEnabled()))
14192 return SplitVectorStore(Op, DAG);
14193 return SDValue();
14194 default:
14195 llvm_unreachable("unsupported private_element_size");
14196 }
14197 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
14198 unsigned Fast = 0;
14199 auto Flags = Store->getMemOperand()->getFlags();
14200 if (allowsMisalignedMemoryAccessesImpl(Size: VT.getSizeInBits(), AddrSpace: AS,
14201 Alignment: Store->getAlign(), Flags, IsFast: &Fast) &&
14202 Fast > 1)
14203 return SDValue();
14204
14205 if (VT.isVector())
14206 return SplitVectorStore(Op, DAG);
14207
14208 return expandUnalignedStore(ST: Store, DAG);
14209 }
14210
14211 // Probably an invalid store. If so we'll end up emitting a selection error.
14212 return SDValue();
14213}
14214
14215// Avoid the full correct expansion for f32 sqrt when promoting from f16.
14216SDValue SITargetLowering::lowerFSQRTF16(SDValue Op, SelectionDAG &DAG) const {
14217 SDLoc SL(Op);
14218 assert(!Subtarget->has16BitInsts());
14219 SDNodeFlags Flags = Op->getFlags();
14220 SDValue Ext =
14221 DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op.getOperand(i: 0), Flags);
14222
14223 SDValue SqrtID = DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL: SL, VT: MVT::i32);
14224 SDValue Sqrt =
14225 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::f32, N1: SqrtID, N2: Ext, Flags);
14226
14227 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::f16, N1: Sqrt,
14228 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
14229}
14230
14231SDValue SITargetLowering::lowerFSQRTF32(SDValue Op, SelectionDAG &DAG) const {
14232 SDLoc DL(Op);
14233 SDNodeFlags Flags = Op->getFlags();
14234 MVT VT = Op.getValueType().getSimpleVT();
14235 const SDValue X = Op.getOperand(i: 0);
14236
14237 if (allowApproxFunc(DAG, Flags)) {
14238 // Instruction is 1ulp but ignores denormals.
14239 return DAG.getNode(
14240 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT,
14241 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL, VT: MVT::i32), N2: X, Flags);
14242 }
14243
14244 SDValue ScaleThreshold = DAG.getConstantFP(Val: 0x1.0p-96f, DL, VT);
14245 SDValue NeedScale = DAG.getSetCC(DL, VT: MVT::i1, LHS: X, RHS: ScaleThreshold, Cond: ISD::SETOLT);
14246
14247 SDValue ScaleUpFactor = DAG.getConstantFP(Val: 0x1.0p+32f, DL, VT);
14248
14249 SDValue ScaledX = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: X, N2: ScaleUpFactor, Flags);
14250
14251 SDValue SqrtX =
14252 DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: NeedScale, N2: ScaledX, N3: X, Flags);
14253
14254 SDValue SqrtS;
14255 if (needsDenormHandlingF32(DAG, Src: X, Flags)) {
14256 SDValue SqrtID =
14257 DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL, VT: MVT::i32);
14258 SqrtS = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT, N1: SqrtID, N2: SqrtX, Flags);
14259
14260 SDValue SqrtSAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: SqrtS);
14261 SDValue SqrtSNextDownInt =
14262 DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SqrtSAsInt,
14263 N2: DAG.getAllOnesConstant(DL, VT: MVT::i32));
14264 SDValue SqrtSNextDown = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: SqrtSNextDownInt);
14265
14266 SDValue NegSqrtSNextDown =
14267 DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtSNextDown, Flags);
14268
14269 SDValue SqrtVP =
14270 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtSNextDown, N2: SqrtS, N3: SqrtX, Flags);
14271
14272 SDValue SqrtSNextUpInt = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SqrtSAsInt,
14273 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
14274 SDValue SqrtSNextUp = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: SqrtSNextUpInt);
14275
14276 SDValue NegSqrtSNextUp = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtSNextUp, Flags);
14277 SDValue SqrtVS =
14278 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtSNextUp, N2: SqrtS, N3: SqrtX, Flags);
14279
14280 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL, VT);
14281 SDValue SqrtVPLE0 = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtVP, RHS: Zero, Cond: ISD::SETOLE);
14282
14283 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: SqrtVPLE0, N2: SqrtSNextDown, N3: SqrtS,
14284 Flags);
14285
14286 SDValue SqrtVPVSGT0 = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtVS, RHS: Zero, Cond: ISD::SETOGT);
14287 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: SqrtVPVSGT0, N2: SqrtSNextUp, N3: SqrtS,
14288 Flags);
14289 } else {
14290 SDValue SqrtR = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: SqrtX, Flags);
14291
14292 SqrtS = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtX, N2: SqrtR, Flags);
14293
14294 SDValue Half = DAG.getConstantFP(Val: 0.5f, DL, VT);
14295 SDValue SqrtH = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtR, N2: Half, Flags);
14296 SDValue NegSqrtH = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtH, Flags);
14297
14298 SDValue SqrtE = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtH, N2: SqrtS, N3: Half, Flags);
14299 SqrtH = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtH, N2: SqrtE, N3: SqrtH, Flags);
14300 SqrtS = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtS, N2: SqrtE, N3: SqrtS, Flags);
14301
14302 SDValue NegSqrtS = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtS, Flags);
14303 SDValue SqrtD =
14304 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtS, N2: SqrtS, N3: SqrtX, Flags);
14305 SqrtS = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtD, N2: SqrtH, N3: SqrtS, Flags);
14306 }
14307
14308 SDValue ScaleDownFactor = DAG.getConstantFP(Val: 0x1.0p-16f, DL, VT);
14309
14310 SDValue ScaledDown =
14311 DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtS, N2: ScaleDownFactor, Flags);
14312
14313 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: NeedScale, N2: ScaledDown, N3: SqrtS, Flags);
14314 SDValue IsZeroOrInf =
14315 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, N1: SqrtX,
14316 N2: DAG.getTargetConstant(Val: fcZero | fcPosInf, DL, VT: MVT::i32));
14317
14318 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsZeroOrInf, N2: SqrtX, N3: SqrtS, Flags);
14319}
14320
14321SDValue SITargetLowering::lowerFSQRTF64(SDValue Op, SelectionDAG &DAG) const {
14322 // For double type, the SQRT and RSQ instructions don't have required
14323 // precision, we apply Goldschmidt's algorithm to improve the result:
14324 //
14325 // y0 = rsq(x)
14326 // g0 = x * y0
14327 // h0 = 0.5 * y0
14328 //
14329 // r0 = 0.5 - h0 * g0
14330 // g1 = g0 * r0 + g0
14331 // h1 = h0 * r0 + h0
14332 //
14333 // r1 = 0.5 - h1 * g1 => d0 = x - g1 * g1
14334 // g2 = g1 * r1 + g1 g2 = d0 * h1 + g1
14335 // h2 = h1 * r1 + h1
14336 //
14337 // r2 = 0.5 - h2 * g2 => d1 = x - g2 * g2
14338 // g3 = g2 * r2 + g2 g3 = d1 * h1 + g2
14339 //
14340 // sqrt(x) = g3
14341
14342 SDNodeFlags Flags = Op->getFlags();
14343
14344 SDLoc DL(Op);
14345
14346 SDValue X = Op.getOperand(i: 0);
14347 SDValue ZeroInt = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
14348
14349 SDValue SqrtX = X;
14350 SDValue Scaling;
14351 if (!Flags.hasApproximateFuncs()) {
14352 SDValue ScaleConstant = DAG.getConstantFP(Val: 0x1.0p-767, DL, VT: MVT::f64);
14353 Scaling = DAG.getSetCC(DL, VT: MVT::i1, LHS: X, RHS: ScaleConstant, Cond: ISD::SETOLT);
14354
14355 // Scale up input if it is too small.
14356 SDValue ScaleUpFactor = DAG.getConstant(Val: 256, DL, VT: MVT::i32);
14357 SDValue ScaleUp =
14358 DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::i32, N1: Scaling, N2: ScaleUpFactor, N3: ZeroInt);
14359 SqrtX = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: X, N2: ScaleUp, Flags);
14360 }
14361
14362 SDValue SqrtY = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT: MVT::f64, Operand: SqrtX);
14363
14364 SDValue SqrtS0 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: SqrtX, N2: SqrtY);
14365
14366 SDValue Half = DAG.getConstantFP(Val: 0.5, DL, VT: MVT::f64);
14367 SDValue SqrtH0 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: SqrtY, N2: Half);
14368
14369 SDValue NegSqrtH0 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtH0);
14370 SDValue SqrtR0 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtH0, N2: SqrtS0, N3: Half);
14371
14372 SDValue SqrtH1 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtH0, N2: SqrtR0, N3: SqrtH0);
14373
14374 SDValue SqrtS1 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtS0, N2: SqrtR0, N3: SqrtS0);
14375
14376 SDValue NegSqrtS1 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtS1);
14377 SDValue SqrtD0 =
14378 DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtS1, N2: SqrtS1, N3: SqrtX);
14379
14380 SDValue SqrtS2 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtD0, N2: SqrtH1, N3: SqrtS1);
14381
14382 SDValue SqrtRet = SqrtS2;
14383 if (!Flags.hasApproximateFuncs()) {
14384 SDValue NegSqrtS2 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtS2);
14385 SDValue SqrtD1 =
14386 DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtS2, N2: SqrtS2, N3: SqrtX);
14387
14388 SqrtRet = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtD1, N2: SqrtH1, N3: SqrtS2);
14389
14390 SDValue ScaleDownFactor = DAG.getSignedConstant(Val: -128, DL, VT: MVT::i32);
14391 SDValue ScaleDown = DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::i32, N1: Scaling,
14392 N2: ScaleDownFactor, N3: ZeroInt);
14393 SqrtRet = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: SqrtRet, N2: ScaleDown, Flags);
14394 }
14395
14396 // TODO: Check for DAZ and expand to subnormals
14397
14398 SDValue IsZeroOrInf;
14399 if (Flags.hasNoInfs()) {
14400 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: MVT::f64);
14401 IsZeroOrInf = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtX, RHS: Zero, Cond: ISD::SETOEQ);
14402 } else {
14403 IsZeroOrInf =
14404 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, N1: SqrtX,
14405 N2: DAG.getTargetConstant(Val: fcZero | fcPosInf, DL, VT: MVT::i32));
14406 }
14407
14408 // If x is +INF, +0, or -0, use its original value
14409 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::f64, N1: IsZeroOrInf, N2: SqrtX, N3: SqrtRet,
14410 Flags);
14411}
14412
14413SDValue SITargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
14414 SDLoc DL(Op);
14415 EVT VT = Op.getValueType();
14416 SDValue Arg = Op.getOperand(i: 0);
14417 SDValue TrigVal;
14418
14419 // Propagate fast-math flags so that the multiply we introduce can be folded
14420 // if Arg is already the result of a multiply by constant.
14421 auto Flags = Op->getFlags();
14422
14423 // AMDGPUISD nodes of vector type must be unrolled here since
14424 // they will not be expanded elsewhere.
14425 auto UnrollIfVec = [&DAG](SDValue V) -> SDValue {
14426 if (!V.getValueType().isVector())
14427 return V;
14428
14429 return DAG.UnrollVectorOp(N: cast<SDNode>(Val&: V));
14430 };
14431
14432 SDValue OneOver2Pi = DAG.getConstantFP(Val: 0.5 * numbers::inv_pi, DL, VT);
14433
14434 if (Subtarget->hasTrigReducedRange()) {
14435 SDValue MulVal = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Arg, N2: OneOver2Pi, Flags);
14436 TrigVal = UnrollIfVec(DAG.getNode(Opcode: AMDGPUISD::FRACT, DL, VT, Operand: MulVal, Flags));
14437 } else {
14438 TrigVal = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Arg, N2: OneOver2Pi, Flags);
14439 }
14440
14441 switch (Op.getOpcode()) {
14442 case ISD::FCOS:
14443 TrigVal = DAG.getNode(Opcode: AMDGPUISD::COS_HW, DL: SDLoc(Op), VT, Operand: TrigVal, Flags);
14444 break;
14445 case ISD::FSIN:
14446 TrigVal = DAG.getNode(Opcode: AMDGPUISD::SIN_HW, DL: SDLoc(Op), VT, Operand: TrigVal, Flags);
14447 break;
14448 default:
14449 llvm_unreachable("Wrong trig opcode");
14450 }
14451
14452 return UnrollIfVec(TrigVal);
14453}
14454
14455SDValue SITargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op,
14456 SelectionDAG &DAG) const {
14457 AtomicSDNode *AtomicNode = cast<AtomicSDNode>(Val&: Op);
14458 assert(AtomicNode->isCompareAndSwap());
14459 unsigned AS = AtomicNode->getAddressSpace();
14460
14461 // No custom lowering required for local address space
14462 if (!AMDGPU::isFlatGlobalAddrSpace(AS))
14463 return Op;
14464
14465 // Non-local address space requires custom lowering for atomic compare
14466 // and swap; cmp and swap should be in a v2i32 or v2i64 in case of _X2
14467 SDLoc DL(Op);
14468 SDValue ChainIn = Op.getOperand(i: 0);
14469 SDValue Addr = Op.getOperand(i: 1);
14470 SDValue Old = Op.getOperand(i: 2);
14471 SDValue New = Op.getOperand(i: 3);
14472 EVT VT = Op.getValueType();
14473 MVT SimpleVT = VT.getSimpleVT();
14474 MVT VecType = MVT::getVectorVT(VT: SimpleVT, NumElements: 2);
14475
14476 SDValue NewOld = DAG.getBuildVector(VT: VecType, DL, Ops: {New, Old});
14477 SDValue Ops[] = {ChainIn, Addr, NewOld};
14478
14479 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::ATOMIC_CMP_SWAP, dl: DL,
14480 VTList: Op->getVTList(), Ops, MemVT: VT,
14481 MMO: AtomicNode->getMemOperand());
14482}
14483
14484//===----------------------------------------------------------------------===//
14485// Custom DAG optimizations
14486//===----------------------------------------------------------------------===//
14487
14488SDValue
14489SITargetLowering::performUCharToFloatCombine(SDNode *N,
14490 DAGCombinerInfo &DCI) const {
14491 EVT VT = N->getValueType(ResNo: 0);
14492 EVT ScalarVT = VT.getScalarType();
14493 if (ScalarVT != MVT::f32 && ScalarVT != MVT::f16)
14494 return SDValue();
14495
14496 SelectionDAG &DAG = DCI.DAG;
14497 SDLoc DL(N);
14498
14499 SDValue Src = N->getOperand(Num: 0);
14500 EVT SrcVT = Src.getValueType();
14501
14502 // TODO: We could try to match extracting the higher bytes, which would be
14503 // easier if i8 vectors weren't promoted to i32 vectors, particularly after
14504 // types are legalized. v4i8 -> v4f32 is probably the only case to worry
14505 // about in practice.
14506 if (DCI.isAfterLegalizeDAG() && SrcVT == MVT::i32) {
14507 if (DAG.MaskedValueIsZero(Op: Src, Mask: APInt::getHighBitsSet(numBits: 32, hiBitsSet: 24))) {
14508 SDValue Cvt = DAG.getNode(Opcode: AMDGPUISD::CVT_F32_UBYTE0, DL, VT: MVT::f32, Operand: Src);
14509 DCI.AddToWorklist(N: Cvt.getNode());
14510
14511 // For the f16 case, fold to a cast to f32 and then cast back to f16.
14512 if (ScalarVT != MVT::f32) {
14513 Cvt = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Cvt,
14514 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
14515 }
14516 return Cvt;
14517 }
14518 }
14519
14520 return SDValue();
14521}
14522
14523SDValue SITargetLowering::performFCopySignCombine(SDNode *N,
14524 DAGCombinerInfo &DCI) const {
14525 SDValue MagnitudeOp = N->getOperand(Num: 0);
14526 SDValue SignOp = N->getOperand(Num: 1);
14527
14528 // The generic combine for fcopysign + fp cast is too conservative with
14529 // vectors, and also gets confused by the splitting we will perform here, so
14530 // peek through FP casts.
14531 if (SignOp.getOpcode() == ISD::FP_EXTEND ||
14532 SignOp.getOpcode() == ISD::FP_ROUND)
14533 SignOp = SignOp.getOperand(i: 0);
14534
14535 SelectionDAG &DAG = DCI.DAG;
14536 SDLoc DL(N);
14537 EVT SignVT = SignOp.getValueType();
14538
14539 // f64 fcopysign is really an f32 copysign on the high bits, so replace the
14540 // lower half with a copy.
14541 // fcopysign f64:x, _:y -> x.lo32, (fcopysign (f32 x.hi32), _:y)
14542 EVT MagVT = MagnitudeOp.getValueType();
14543
14544 unsigned NumElts = MagVT.isVector() ? MagVT.getVectorNumElements() : 1;
14545
14546 if (MagVT.getScalarType() == MVT::f64) {
14547 EVT F32VT = MagVT.isVector()
14548 ? EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: 2 * NumElts)
14549 : MVT::v2f32;
14550
14551 SDValue MagAsVector = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: F32VT, Operand: MagnitudeOp);
14552
14553 SmallVector<SDValue, 8> NewElts;
14554 for (unsigned I = 0; I != NumElts; ++I) {
14555 SDValue MagLo =
14556 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: MagAsVector,
14557 N2: DAG.getConstant(Val: 2 * I, DL, VT: MVT::i32));
14558 SDValue MagHi =
14559 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: MagAsVector,
14560 N2: DAG.getConstant(Val: 2 * I + 1, DL, VT: MVT::i32));
14561
14562 SDValue SignOpElt =
14563 MagVT.isVector()
14564 ? DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: SignVT.getScalarType(),
14565 N1: SignOp, N2: DAG.getConstant(Val: I, DL, VT: MVT::i32))
14566 : SignOp;
14567
14568 SDValue HiOp =
14569 DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: MVT::f32, N1: MagHi, N2: SignOpElt);
14570
14571 SDValue Vector =
14572 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MVT::v2f32, N1: MagLo, N2: HiOp);
14573
14574 SDValue NewElt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: Vector);
14575 NewElts.push_back(Elt: NewElt);
14576 }
14577
14578 if (NewElts.size() == 1)
14579 return NewElts[0];
14580
14581 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MagVT, Ops: NewElts);
14582 }
14583
14584 if (SignVT.getScalarType() != MVT::f64)
14585 return SDValue();
14586
14587 // Reduce width of sign operand, we only need the highest bit.
14588 //
14589 // fcopysign f64:x, f64:y ->
14590 // fcopysign f64:x, (extract_vector_elt (bitcast f64:y to v2f32), 1)
14591 // TODO: In some cases it might make sense to go all the way to f16.
14592
14593 EVT F32VT = MagVT.isVector()
14594 ? EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: 2 * NumElts)
14595 : MVT::v2f32;
14596
14597 SDValue SignAsVector = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: F32VT, Operand: SignOp);
14598
14599 SmallVector<SDValue, 8> F32Signs;
14600 for (unsigned I = 0; I != NumElts; ++I) {
14601 // Take sign from odd elements of cast vector
14602 SDValue SignAsF32 =
14603 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: SignAsVector,
14604 N2: DAG.getConstant(Val: 2 * I + 1, DL, VT: MVT::i32));
14605 F32Signs.push_back(Elt: SignAsF32);
14606 }
14607
14608 SDValue NewSign =
14609 NumElts == 1
14610 ? F32Signs.back()
14611 : DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL,
14612 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: NumElts),
14613 Ops: F32Signs);
14614
14615 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
14616 N2: NewSign);
14617}
14618
14619// (shl (add x, c1), c2) -> add (shl x, c2), (shl c1, c2)
14620// (shl (or x, c1), c2) -> add (shl x, c2), (shl c1, c2) iff x and c1 share no
14621// bits
14622
14623// This is a variant of
14624// (mul (add x, c1), c2) -> add (mul x, c2), (mul c1, c2),
14625//
14626// The normal DAG combiner will do this, but only if the add has one use since
14627// that would increase the number of instructions.
14628//
14629// This prevents us from seeing a constant offset that can be folded into a
14630// memory instruction's addressing mode. If we know the resulting add offset of
14631// a pointer can be folded into an addressing offset, we can replace the pointer
14632// operand with the add of new constant offset. This eliminates one of the uses,
14633// and may allow the remaining use to also be simplified.
14634//
14635SDValue SITargetLowering::performSHLPtrCombine(SDNode *N, unsigned AddrSpace,
14636 EVT MemVT,
14637 DAGCombinerInfo &DCI) const {
14638 SDValue N0 = N->getOperand(Num: 0);
14639 SDValue N1 = N->getOperand(Num: 1);
14640
14641 // We only do this to handle cases where it's profitable when there are
14642 // multiple uses of the add, so defer to the standard combine.
14643 if ((!N0->isAnyAdd() && N0.getOpcode() != ISD::OR) || N0->hasOneUse())
14644 return SDValue();
14645
14646 const ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(Val&: N1);
14647 if (!CN1)
14648 return SDValue();
14649
14650 const ConstantSDNode *CAdd = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
14651 if (!CAdd)
14652 return SDValue();
14653
14654 SelectionDAG &DAG = DCI.DAG;
14655
14656 if (N0->getOpcode() == ISD::OR &&
14657 !DAG.haveNoCommonBitsSet(A: N0.getOperand(i: 0), B: N0.getOperand(i: 1)))
14658 return SDValue();
14659
14660 // If the resulting offset is too large, we can't fold it into the
14661 // addressing mode offset.
14662 APInt Offset = CAdd->getAPIntValue() << CN1->getAPIntValue();
14663 Type *Ty = MemVT.getTypeForEVT(Context&: *DCI.DAG.getContext());
14664
14665 AddrMode AM;
14666 AM.HasBaseReg = true;
14667 AM.BaseOffs = Offset.getSExtValue();
14668 if (!isLegalAddressingMode(DL: DCI.DAG.getDataLayout(), AM, Ty, AS: AddrSpace))
14669 return SDValue();
14670
14671 SDLoc SL(N);
14672 EVT VT = N->getValueType(ResNo: 0);
14673
14674 SDValue ShlX = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT, N1: N0.getOperand(i: 0), N2: N1);
14675 SDValue COffset = DAG.getConstant(Val: Offset, DL: SL, VT);
14676
14677 SDNodeFlags Flags;
14678 Flags.setNoUnsignedWrap(
14679 N->getFlags().hasNoUnsignedWrap() &&
14680 (N0.getOpcode() == ISD::OR || N0->getFlags().hasNoUnsignedWrap()));
14681
14682 // Use ISD::ADD even if the original operation was ISD::PTRADD, since we can't
14683 // be sure that the new left operand is a proper base pointer.
14684 return DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ShlX, N2: COffset, Flags);
14685}
14686
14687/// MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset
14688/// by the chain and intrinsic ID. Theoretically we would also need to check the
14689/// specific intrinsic, but they all place the pointer operand first.
14690static unsigned getBasePtrIndex(const MemSDNode *N) {
14691 switch (N->getOpcode()) {
14692 case ISD::STORE:
14693 case ISD::INTRINSIC_W_CHAIN:
14694 case ISD::INTRINSIC_VOID:
14695 return 2;
14696 default:
14697 return 1;
14698 }
14699}
14700
14701SDValue SITargetLowering::performMemSDNodeCombine(MemSDNode *N,
14702 DAGCombinerInfo &DCI) const {
14703 SelectionDAG &DAG = DCI.DAG;
14704
14705 unsigned PtrIdx = getBasePtrIndex(N);
14706 SDValue Ptr = N->getOperand(Num: PtrIdx);
14707
14708 // TODO: We could also do this for multiplies.
14709 if (Ptr.getOpcode() == ISD::SHL) {
14710 SDValue NewPtr = performSHLPtrCombine(N: Ptr.getNode(), AddrSpace: N->getAddressSpace(),
14711 MemVT: N->getMemoryVT(), DCI);
14712 if (NewPtr) {
14713 SmallVector<SDValue, 8> NewOps(N->ops());
14714
14715 NewOps[PtrIdx] = NewPtr;
14716 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
14717 }
14718 }
14719
14720 return SDValue();
14721}
14722
14723static bool bitOpWithConstantIsReducible(unsigned Opc, uint32_t Val) {
14724 return (Opc == ISD::AND && (Val == 0 || Val == 0xffffffff)) ||
14725 (Opc == ISD::OR && (Val == 0xffffffff || Val == 0)) ||
14726 (Opc == ISD::XOR && Val == 0);
14727}
14728
14729// Break up 64-bit bit operation of a constant into two 32-bit and/or/xor. This
14730// will typically happen anyway for a VALU 64-bit and. This exposes other 32-bit
14731// integer combine opportunities since most 64-bit operations are decomposed
14732// this way. TODO: We won't want this for SALU especially if it is an inline
14733// immediate.
14734SDValue SITargetLowering::splitBinaryBitConstantOp(
14735 DAGCombinerInfo &DCI, const SDLoc &SL, unsigned Opc, SDValue LHS,
14736 const ConstantSDNode *CRHS) const {
14737 uint64_t Val = CRHS->getZExtValue();
14738 uint32_t ValLo = Lo_32(Value: Val);
14739 uint32_t ValHi = Hi_32(Value: Val);
14740 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
14741
14742 if ((bitOpWithConstantIsReducible(Opc, Val: ValLo) ||
14743 bitOpWithConstantIsReducible(Opc, Val: ValHi)) ||
14744 (CRHS->hasOneUse() && !TII->isInlineConstant(Imm: CRHS->getAPIntValue()))) {
14745 // We have 64-bit scalar and/or/xor, but do not have vector forms.
14746 if (Subtarget->has64BitLiterals() && CRHS->hasOneUse() &&
14747 !CRHS->user_begin()->isDivergent())
14748 return SDValue();
14749
14750 // If we need to materialize a 64-bit immediate, it will be split up later
14751 // anyway. Avoid creating the harder to understand 64-bit immediate
14752 // materialization.
14753 return splitBinaryBitConstantOpImpl(DCI, SL, Opc, LHS, ValLo, ValHi);
14754 }
14755
14756 return SDValue();
14757}
14758
14759bool llvm::isBoolSGPR(SDValue V) {
14760 if (V.getValueType() != MVT::i1)
14761 return false;
14762 switch (V.getOpcode()) {
14763 default:
14764 break;
14765 case ISD::SETCC:
14766 case ISD::IS_FPCLASS:
14767 case AMDGPUISD::FP_CLASS:
14768 return true;
14769 case ISD::AND:
14770 case ISD::OR:
14771 case ISD::XOR:
14772 return isBoolSGPR(V: V.getOperand(i: 0)) && isBoolSGPR(V: V.getOperand(i: 1));
14773 case ISD::SADDO:
14774 case ISD::UADDO:
14775 case ISD::SSUBO:
14776 case ISD::USUBO:
14777 case ISD::SMULO:
14778 case ISD::UMULO:
14779 return V.getResNo() == 1;
14780 case ISD::INTRINSIC_WO_CHAIN: {
14781 unsigned IntrinsicID = V.getConstantOperandVal(i: 0);
14782 switch (IntrinsicID) {
14783 case Intrinsic::amdgcn_is_shared:
14784 case Intrinsic::amdgcn_is_private:
14785 return true;
14786 default:
14787 return false;
14788 }
14789
14790 return false;
14791 }
14792 }
14793 return false;
14794}
14795
14796// If a constant has all zeroes or all ones within each byte return it.
14797// Otherwise return 0.
14798static uint32_t getConstantPermuteMask(uint32_t C) {
14799 // 0xff for any zero byte in the mask
14800 uint32_t ZeroByteMask = 0;
14801 if (!(C & 0x000000ff))
14802 ZeroByteMask |= 0x000000ff;
14803 if (!(C & 0x0000ff00))
14804 ZeroByteMask |= 0x0000ff00;
14805 if (!(C & 0x00ff0000))
14806 ZeroByteMask |= 0x00ff0000;
14807 if (!(C & 0xff000000))
14808 ZeroByteMask |= 0xff000000;
14809 uint32_t NonZeroByteMask = ~ZeroByteMask; // 0xff for any non-zero byte
14810 if ((NonZeroByteMask & C) != NonZeroByteMask)
14811 return 0; // Partial bytes selected.
14812 return C;
14813}
14814
14815// Check if a node selects whole bytes from its operand 0 starting at a byte
14816// boundary while masking the rest. Returns select mask as in the v_perm_b32
14817// or -1 if not succeeded.
14818// Note byte select encoding:
14819// value 0-3 selects corresponding source byte;
14820// value 0xc selects zero;
14821// value 0xff selects 0xff.
14822static uint32_t getPermuteMask(SDValue V) {
14823 assert(V.getValueSizeInBits() == 32);
14824
14825 if (V.getNumOperands() != 2)
14826 return ~0;
14827
14828 ConstantSDNode *N1 = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1));
14829 if (!N1)
14830 return ~0;
14831
14832 uint32_t C = N1->getZExtValue();
14833
14834 switch (V.getOpcode()) {
14835 default:
14836 break;
14837 case ISD::AND:
14838 if (uint32_t ConstMask = getConstantPermuteMask(C))
14839 return (0x03020100 & ConstMask) | (0x0c0c0c0c & ~ConstMask);
14840 break;
14841
14842 case ISD::OR:
14843 if (uint32_t ConstMask = getConstantPermuteMask(C))
14844 return (0x03020100 & ~ConstMask) | ConstMask;
14845 break;
14846
14847 case ISD::SHL:
14848 if (C % 8)
14849 return ~0;
14850
14851 return uint32_t((0x030201000c0c0c0cull << C) >> 32);
14852
14853 case ISD::SRL:
14854 if (C % 8)
14855 return ~0;
14856
14857 return uint32_t(0x0c0c0c0c03020100ull >> C);
14858 }
14859
14860 return ~0;
14861}
14862
14863SDValue SITargetLowering::performAndCombine(SDNode *N,
14864 DAGCombinerInfo &DCI) const {
14865 if (DCI.isBeforeLegalize())
14866 return SDValue();
14867
14868 SelectionDAG &DAG = DCI.DAG;
14869 EVT VT = N->getValueType(ResNo: 0);
14870 SDValue LHS = N->getOperand(Num: 0);
14871 SDValue RHS = N->getOperand(Num: 1);
14872
14873 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
14874 if (VT == MVT::i64 && CRHS) {
14875 if (SDValue Split =
14876 splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::AND, LHS, CRHS))
14877 return Split;
14878 }
14879
14880 if (CRHS && VT == MVT::i32) {
14881 // and (srl x, c), mask => shl (bfe x, nb + c, mask >> nb), nb
14882 // nb = number of trailing zeroes in mask
14883 // It can be optimized out using SDWA for GFX8+ in the SDWA peephole pass,
14884 // given that we are selecting 8 or 16 bit fields starting at byte boundary.
14885 uint64_t Mask = CRHS->getZExtValue();
14886 unsigned Bits = llvm::popcount(Value: Mask);
14887 if (getSubtarget()->hasSDWA() && LHS->getOpcode() == ISD::SRL &&
14888 (Bits == 8 || Bits == 16) && isShiftedMask_64(Value: Mask) && !(Mask & 1)) {
14889 if (auto *CShift = dyn_cast<ConstantSDNode>(Val: LHS->getOperand(Num: 1))) {
14890 unsigned Shift = CShift->getZExtValue();
14891 unsigned NB = CRHS->getAPIntValue().countr_zero();
14892 unsigned Offset = NB + Shift;
14893 if ((Offset & (Bits - 1)) == 0) { // Starts at a byte or word boundary.
14894 SDLoc SL(N);
14895 SDValue BFE =
14896 DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT: MVT::i32, N1: LHS->getOperand(Num: 0),
14897 N2: DAG.getConstant(Val: Offset, DL: SL, VT: MVT::i32),
14898 N3: DAG.getConstant(Val: Bits, DL: SL, VT: MVT::i32));
14899 EVT NarrowVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: Bits);
14900 SDValue Ext = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT, N1: BFE,
14901 N2: DAG.getValueType(NarrowVT));
14902 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(LHS), VT, N1: Ext,
14903 N2: DAG.getConstant(Val: NB, DL: SDLoc(CRHS), VT: MVT::i32));
14904 return Shl;
14905 }
14906 }
14907 }
14908
14909 // and (perm x, y, c1), c2 -> perm x, y, permute_mask(c1, c2)
14910 if (LHS.hasOneUse() && LHS.getOpcode() == AMDGPUISD::PERM &&
14911 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2))) {
14912 uint32_t Sel = getConstantPermuteMask(C: Mask);
14913 if (!Sel)
14914 return SDValue();
14915
14916 // Select 0xc for all zero bytes
14917 Sel = (LHS.getConstantOperandVal(i: 2) & Sel) | (~Sel & 0x0c0c0c0c);
14918 SDLoc DL(N);
14919 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
14920 N2: LHS.getOperand(i: 1), N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
14921 }
14922 }
14923
14924 // (and (fcmp ord x, x), (fcmp une (fabs x), inf)) ->
14925 // fp_class x, ~(s_nan | q_nan | n_infinity | p_infinity)
14926 if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == ISD::SETCC) {
14927 ISD::CondCode LCC = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
14928 ISD::CondCode RCC = cast<CondCodeSDNode>(Val: RHS.getOperand(i: 2))->get();
14929
14930 SDValue X = LHS.getOperand(i: 0);
14931 SDValue Y = RHS.getOperand(i: 0);
14932 if (Y.getOpcode() != ISD::FABS || Y.getOperand(i: 0) != X ||
14933 !isTypeLegal(VT: X.getValueType()))
14934 return SDValue();
14935
14936 if (LCC == ISD::SETO) {
14937 if (X != LHS.getOperand(i: 1))
14938 return SDValue();
14939
14940 if (RCC == ISD::SETUNE) {
14941 const ConstantFPSDNode *C1 =
14942 dyn_cast<ConstantFPSDNode>(Val: RHS.getOperand(i: 1));
14943 if (!C1 || !C1->isInfinity() || C1->isNegative())
14944 return SDValue();
14945
14946 const uint32_t Mask = SIInstrFlags::N_NORMAL |
14947 SIInstrFlags::N_SUBNORMAL | SIInstrFlags::N_ZERO |
14948 SIInstrFlags::P_ZERO | SIInstrFlags::P_SUBNORMAL |
14949 SIInstrFlags::P_NORMAL;
14950
14951 static_assert(
14952 ((~(SIInstrFlags::S_NAN | SIInstrFlags::Q_NAN |
14953 SIInstrFlags::N_INFINITY | SIInstrFlags::P_INFINITY)) &
14954 0x3ff) == Mask,
14955 "mask not equal");
14956
14957 SDLoc DL(N);
14958 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: X,
14959 N2: DAG.getConstant(Val: Mask, DL, VT: MVT::i32));
14960 }
14961 }
14962 }
14963
14964 if (RHS.getOpcode() == ISD::SETCC && LHS.getOpcode() == AMDGPUISD::FP_CLASS)
14965 std::swap(a&: LHS, b&: RHS);
14966
14967 if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == AMDGPUISD::FP_CLASS &&
14968 RHS.hasOneUse()) {
14969 ISD::CondCode LCC = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
14970 // and (fcmp seto), (fp_class x, mask) -> fp_class x, mask & ~(p_nan |
14971 // n_nan) and (fcmp setuo), (fp_class x, mask) -> fp_class x, mask & (p_nan
14972 // | n_nan)
14973 const ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i: 1));
14974 if ((LCC == ISD::SETO || LCC == ISD::SETUO) && Mask &&
14975 (RHS.getOperand(i: 0) == LHS.getOperand(i: 0) &&
14976 LHS.getOperand(i: 0) == LHS.getOperand(i: 1))) {
14977 const unsigned OrdMask = SIInstrFlags::S_NAN | SIInstrFlags::Q_NAN;
14978 unsigned NewMask = LCC == ISD::SETO ? Mask->getZExtValue() & ~OrdMask
14979 : Mask->getZExtValue() & OrdMask;
14980
14981 SDLoc DL(N);
14982 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: RHS.getOperand(i: 0),
14983 N2: DAG.getConstant(Val: NewMask, DL, VT: MVT::i32));
14984 }
14985 }
14986
14987 if (VT == MVT::i32 && (RHS.getOpcode() == ISD::SIGN_EXTEND ||
14988 LHS.getOpcode() == ISD::SIGN_EXTEND)) {
14989 // and x, (sext cc from i1) => select cc, x, 0
14990 if (RHS.getOpcode() != ISD::SIGN_EXTEND)
14991 std::swap(a&: LHS, b&: RHS);
14992 if (isBoolSGPR(V: RHS.getOperand(i: 0)))
14993 return DAG.getSelect(DL: SDLoc(N), VT: MVT::i32, Cond: RHS.getOperand(i: 0), LHS,
14994 RHS: DAG.getConstant(Val: 0, DL: SDLoc(N), VT: MVT::i32));
14995 }
14996
14997 // and (op x, c1), (op y, c2) -> perm x, y, permute_mask(c1, c2)
14998 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
14999 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() &&
15000 N->isDivergent() && TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
15001 uint32_t LHSMask = getPermuteMask(V: LHS);
15002 uint32_t RHSMask = getPermuteMask(V: RHS);
15003 if (LHSMask != ~0u && RHSMask != ~0u) {
15004 // Canonicalize the expression in an attempt to have fewer unique masks
15005 // and therefore fewer registers used to hold the masks.
15006 if (LHSMask > RHSMask) {
15007 std::swap(a&: LHSMask, b&: RHSMask);
15008 std::swap(a&: LHS, b&: RHS);
15009 }
15010
15011 // Select 0xc for each lane used from source operand. Zero has 0xc mask
15012 // set, 0xff have 0xff in the mask, actual lanes are in the 0-3 range.
15013 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15014 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15015
15016 // Check of we need to combine values from two sources within a byte.
15017 if (!(LHSUsedLanes & RHSUsedLanes) &&
15018 // If we select high and lower word keep it for SDWA.
15019 // TODO: teach SDWA to work with v_perm_b32 and remove the check.
15020 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
15021 // Each byte in each mask is either selector mask 0-3, or has higher
15022 // bits set in either of masks, which can be 0xff for 0xff or 0x0c for
15023 // zero. If 0x0c is in either mask it shall always be 0x0c. Otherwise
15024 // mask which is not 0xff wins. By anding both masks we have a correct
15025 // result except that 0x0c shall be corrected to give 0x0c only.
15026 uint32_t Mask = LHSMask & RHSMask;
15027 for (unsigned I = 0; I < 32; I += 8) {
15028 uint32_t ByteSel = 0xff << I;
15029 if ((LHSMask & ByteSel) == 0x0c || (RHSMask & ByteSel) == 0x0c)
15030 Mask &= (0x0c << I) & 0xffffffff;
15031 }
15032
15033 // Add 4 to each active LHS lane. It will not affect any existing 0xff
15034 // or 0x0c.
15035 uint32_t Sel = Mask | (LHSUsedLanes & 0x04040404);
15036 SDLoc DL(N);
15037
15038 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15039 N2: RHS.getOperand(i: 0),
15040 N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15041 }
15042 }
15043 }
15044
15045 return SDValue();
15046}
15047
15048// A key component of v_perm is a mapping between byte position of the src
15049// operands, and the byte position of the dest. To provide such, we need: 1. the
15050// node that provides x byte of the dest of the OR, and 2. the byte of the node
15051// used to provide that x byte. calculateByteProvider finds which node provides
15052// a certain byte of the dest of the OR, and calculateSrcByte takes that node,
15053// and finds an ultimate src and byte position For example: The supported
15054// LoadCombine pattern for vector loads is as follows
15055// t1
15056// or
15057// / \
15058// t2 t3
15059// zext shl
15060// | | \
15061// t4 t5 16
15062// or anyext
15063// / \ |
15064// t6 t7 t8
15065// srl shl or
15066// / | / \ / \
15067// t9 t10 t11 t12 t13 t14
15068// trunc* 8 trunc* 8 and and
15069// | | / | | \
15070// t15 t16 t17 t18 t19 t20
15071// trunc* 255 srl -256
15072// | / \
15073// t15 t15 16
15074//
15075// *In this example, the truncs are from i32->i16
15076//
15077// calculateByteProvider would find t6, t7, t13, and t14 for bytes 0-3
15078// respectively. calculateSrcByte would find (given node) -> ultimate src &
15079// byteposition: t6 -> t15 & 1, t7 -> t16 & 0, t13 -> t15 & 0, t14 -> t15 & 3.
15080// After finding the mapping, we can combine the tree into vperm t15, t16,
15081// 0x05000407
15082
15083// Find the source and byte position from a node.
15084// \p DestByte is the byte position of the dest of the or that the src
15085// ultimately provides. \p SrcIndex is the byte of the src that maps to this
15086// dest of the or byte. \p Depth tracks how many recursive iterations we have
15087// performed.
15088static const std::optional<ByteProvider<SDValue>>
15089calculateSrcByte(const SDValue Op, uint64_t DestByte, uint64_t SrcIndex = 0,
15090 unsigned Depth = 0) {
15091 // We may need to recursively traverse a series of SRLs
15092 if (Depth >= 6)
15093 return std::nullopt;
15094
15095 if (Op.getValueSizeInBits() < 8)
15096 return std::nullopt;
15097
15098 if (Op.getValueType().isVector())
15099 return ByteProvider<SDValue>::getSrc(Val: Op, ByteOffset: DestByte, VectorOffset: SrcIndex);
15100
15101 switch (Op->getOpcode()) {
15102 case ISD::TRUNCATE: {
15103 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex, Depth: Depth + 1);
15104 }
15105
15106 case ISD::ANY_EXTEND:
15107 case ISD::SIGN_EXTEND:
15108 case ISD::ZERO_EXTEND:
15109 case ISD::SIGN_EXTEND_INREG: {
15110 SDValue NarrowOp = Op->getOperand(Num: 0);
15111 auto NarrowVT = NarrowOp.getValueType();
15112 if (Op->getOpcode() == ISD::SIGN_EXTEND_INREG) {
15113 auto *VTSign = cast<VTSDNode>(Val: Op->getOperand(Num: 1));
15114 NarrowVT = VTSign->getVT();
15115 }
15116 if (!NarrowVT.isByteSized())
15117 return std::nullopt;
15118 uint64_t NarrowByteWidth = NarrowVT.getStoreSize();
15119
15120 if (SrcIndex >= NarrowByteWidth)
15121 return std::nullopt;
15122 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex, Depth: Depth + 1);
15123 }
15124
15125 case ISD::SRA:
15126 case ISD::SRL: {
15127 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15128 if (!ShiftOp)
15129 return std::nullopt;
15130
15131 uint64_t BitShift = ShiftOp->getZExtValue();
15132
15133 if (BitShift % 8 != 0)
15134 return std::nullopt;
15135
15136 uint64_t NewSrcIndex = SrcIndex + BitShift / 8;
15137 if (NewSrcIndex >= Op.getScalarValueSizeInBits() / 8)
15138 return std::nullopt;
15139
15140 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex: NewSrcIndex,
15141 Depth: Depth + 1);
15142 }
15143
15144 default: {
15145 return ByteProvider<SDValue>::getSrc(Val: Op, ByteOffset: DestByte, VectorOffset: SrcIndex);
15146 }
15147 }
15148 llvm_unreachable("fully handled switch");
15149}
15150
15151// For a byte position in the result of an Or, traverse the tree and find the
15152// node (and the byte of the node) which ultimately provides this {Or,
15153// BytePosition}. \p Op is the operand we are currently examining. \p Index is
15154// the byte position of the Op that corresponds with the originally requested
15155// byte of the Or \p Depth tracks how many recursive iterations we have
15156// performed. \p StartingIndex is the originally requested byte of the Or
15157static const std::optional<ByteProvider<SDValue>>
15158calculateByteProvider(const SDValue &Op, unsigned Index, unsigned Depth,
15159 unsigned StartingIndex = 0) {
15160 // Finding Src tree of RHS of or typically requires at least 1 additional
15161 // depth
15162 if (Depth > 6)
15163 return std::nullopt;
15164
15165 unsigned BitWidth = Op.getScalarValueSizeInBits();
15166 if (BitWidth % 8 != 0)
15167 return std::nullopt;
15168 if (Index > BitWidth / 8 - 1)
15169 return std::nullopt;
15170
15171 bool IsVec = Op.getValueType().isVector();
15172 switch (Op.getOpcode()) {
15173 case ISD::OR: {
15174 if (IsVec)
15175 return std::nullopt;
15176
15177 auto RHS = calculateByteProvider(Op: Op.getOperand(i: 1), Index, Depth: Depth + 1,
15178 StartingIndex);
15179 if (!RHS)
15180 return std::nullopt;
15181 auto LHS = calculateByteProvider(Op: Op.getOperand(i: 0), Index, Depth: Depth + 1,
15182 StartingIndex);
15183 if (!LHS)
15184 return std::nullopt;
15185 // A well formed Or will have two ByteProviders for each byte, one of which
15186 // is constant zero
15187 if (!LHS->isConstantZero() && !RHS->isConstantZero())
15188 return std::nullopt;
15189 if (!LHS || LHS->isConstantZero())
15190 return RHS;
15191 if (!RHS || RHS->isConstantZero())
15192 return LHS;
15193 return std::nullopt;
15194 }
15195
15196 case ISD::AND: {
15197 if (IsVec)
15198 return std::nullopt;
15199
15200 auto *BitMaskOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15201 if (!BitMaskOp)
15202 return std::nullopt;
15203
15204 uint32_t BitMask = BitMaskOp->getZExtValue();
15205 // Bits we expect for our StartingIndex
15206 uint32_t IndexMask = 0xFF << (Index * 8);
15207
15208 if ((IndexMask & BitMask) != IndexMask) {
15209 // If the result of the and partially provides the byte, then it
15210 // is not well formatted
15211 if (IndexMask & BitMask)
15212 return std::nullopt;
15213 return ByteProvider<SDValue>::getConstantZero();
15214 }
15215
15216 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte: StartingIndex, SrcIndex: Index);
15217 }
15218
15219 case ISD::FSHR: {
15220 if (IsVec)
15221 return std::nullopt;
15222
15223 // fshr(X,Y,Z): (X << (BW - (Z % BW))) | (Y >> (Z % BW))
15224 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 2));
15225 if (!ShiftOp || Op.getValueType().isVector())
15226 return std::nullopt;
15227
15228 uint64_t BitsProvided = Op.getValueSizeInBits();
15229 if (BitsProvided % 8 != 0)
15230 return std::nullopt;
15231
15232 uint64_t BitShift = ShiftOp->getAPIntValue().urem(RHS: BitsProvided);
15233 if (BitShift % 8)
15234 return std::nullopt;
15235
15236 uint64_t ConcatSizeInBytes = BitsProvided / 4;
15237 uint64_t ByteShift = BitShift / 8;
15238
15239 uint64_t NewIndex = (Index + ByteShift) % ConcatSizeInBytes;
15240 uint64_t BytesProvided = BitsProvided / 8;
15241 SDValue NextOp = Op.getOperand(i: NewIndex >= BytesProvided ? 0 : 1);
15242 NewIndex %= BytesProvided;
15243 return calculateByteProvider(Op: NextOp, Index: NewIndex, Depth: Depth + 1, StartingIndex);
15244 }
15245
15246 case ISD::SRA:
15247 case ISD::SRL: {
15248 if (IsVec)
15249 return std::nullopt;
15250
15251 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15252 if (!ShiftOp)
15253 return std::nullopt;
15254
15255 uint64_t BitShift = ShiftOp->getZExtValue();
15256 if (BitShift % 8)
15257 return std::nullopt;
15258
15259 auto BitsProvided = Op.getScalarValueSizeInBits();
15260 if (BitsProvided % 8 != 0)
15261 return std::nullopt;
15262
15263 uint64_t BytesProvided = BitsProvided / 8;
15264 uint64_t ByteShift = BitShift / 8;
15265 if (Index + ByteShift < BytesProvided)
15266 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte: StartingIndex,
15267 SrcIndex: Index + ByteShift);
15268 // SRA's out-of-range bytes are sign bits, not constant zero.
15269 if (Op.getOpcode() == ISD::SRA)
15270 return std::nullopt;
15271 return ByteProvider<SDValue>::getConstantZero();
15272 }
15273
15274 case ISD::SHL: {
15275 if (IsVec)
15276 return std::nullopt;
15277
15278 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15279 if (!ShiftOp)
15280 return std::nullopt;
15281
15282 uint64_t BitShift = ShiftOp->getZExtValue();
15283 if (BitShift % 8 != 0)
15284 return std::nullopt;
15285 uint64_t ByteShift = BitShift / 8;
15286
15287 // If we are shifting by an amount greater than (or equal to)
15288 // the index we are trying to provide, then it provides 0s. If not,
15289 // then this bytes are not definitively 0s, and the corresponding byte
15290 // of interest is Index - ByteShift of the src
15291 return Index < ByteShift
15292 ? ByteProvider<SDValue>::getConstantZero()
15293 : calculateByteProvider(Op: Op.getOperand(i: 0), Index: Index - ByteShift,
15294 Depth: Depth + 1, StartingIndex);
15295 }
15296 case ISD::ANY_EXTEND:
15297 case ISD::SIGN_EXTEND:
15298 case ISD::ZERO_EXTEND:
15299 case ISD::SIGN_EXTEND_INREG:
15300 case ISD::AssertZext:
15301 case ISD::AssertSext: {
15302 if (IsVec)
15303 return std::nullopt;
15304
15305 SDValue NarrowOp = Op->getOperand(Num: 0);
15306 unsigned NarrowBitWidth = NarrowOp.getValueSizeInBits();
15307 if (Op->getOpcode() == ISD::SIGN_EXTEND_INREG ||
15308 Op->getOpcode() == ISD::AssertZext ||
15309 Op->getOpcode() == ISD::AssertSext) {
15310 auto *VTSign = cast<VTSDNode>(Val: Op->getOperand(Num: 1));
15311 NarrowBitWidth = VTSign->getVT().getSizeInBits();
15312 }
15313 if (NarrowBitWidth % 8 != 0)
15314 return std::nullopt;
15315 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
15316
15317 if (Index >= NarrowByteWidth)
15318 return Op.getOpcode() == ISD::ZERO_EXTEND
15319 ? std::optional<ByteProvider<SDValue>>(
15320 ByteProvider<SDValue>::getConstantZero())
15321 : std::nullopt;
15322 return calculateByteProvider(Op: NarrowOp, Index, Depth: Depth + 1, StartingIndex);
15323 }
15324
15325 case ISD::TRUNCATE: {
15326 if (IsVec)
15327 return std::nullopt;
15328
15329 uint64_t NarrowByteWidth = BitWidth / 8;
15330
15331 if (NarrowByteWidth >= Index) {
15332 return calculateByteProvider(Op: Op.getOperand(i: 0), Index, Depth: Depth + 1,
15333 StartingIndex);
15334 }
15335
15336 return std::nullopt;
15337 }
15338
15339 case ISD::CopyFromReg: {
15340 if (BitWidth / 8 > Index)
15341 return calculateSrcByte(Op, DestByte: StartingIndex, SrcIndex: Index);
15342
15343 return std::nullopt;
15344 }
15345
15346 case ISD::LOAD: {
15347 auto *L = cast<LoadSDNode>(Val: Op.getNode());
15348
15349 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
15350 if (NarrowBitWidth % 8 != 0)
15351 return std::nullopt;
15352 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
15353
15354 // If the width of the load does not reach byte we are trying to provide for
15355 // and it is not a ZEXTLOAD, then the load does not provide for the byte in
15356 // question
15357 if (Index >= NarrowByteWidth) {
15358 return L->getExtensionType() == ISD::ZEXTLOAD
15359 ? std::optional<ByteProvider<SDValue>>(
15360 ByteProvider<SDValue>::getConstantZero())
15361 : std::nullopt;
15362 }
15363
15364 if (NarrowByteWidth > Index) {
15365 return calculateSrcByte(Op, DestByte: StartingIndex, SrcIndex: Index);
15366 }
15367
15368 return std::nullopt;
15369 }
15370
15371 case ISD::BSWAP: {
15372 if (IsVec)
15373 return std::nullopt;
15374
15375 return calculateByteProvider(Op: Op->getOperand(Num: 0), Index: BitWidth / 8 - Index - 1,
15376 Depth: Depth + 1, StartingIndex);
15377 }
15378
15379 case ISD::EXTRACT_VECTOR_ELT: {
15380 auto *IdxOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15381 if (!IdxOp)
15382 return std::nullopt;
15383 auto VecIdx = IdxOp->getZExtValue();
15384 auto ScalarSize = Op.getScalarValueSizeInBits();
15385 if (ScalarSize < 32)
15386 Index = ScalarSize == 8 ? VecIdx : VecIdx * 2 + Index;
15387 return calculateSrcByte(Op: ScalarSize >= 32 ? Op : Op.getOperand(i: 0),
15388 DestByte: StartingIndex, SrcIndex: Index);
15389 }
15390
15391 case AMDGPUISD::PERM: {
15392 if (IsVec)
15393 return std::nullopt;
15394
15395 auto *PermMask = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 2));
15396 if (!PermMask)
15397 return std::nullopt;
15398
15399 auto IdxMask =
15400 (PermMask->getZExtValue() & (0xFF << (Index * 8))) >> (Index * 8);
15401 if (IdxMask > 0x07 && IdxMask != 0x0c)
15402 return std::nullopt;
15403
15404 auto NextOp = Op.getOperand(i: IdxMask > 0x03 ? 0 : 1);
15405 auto NextIndex = IdxMask > 0x03 ? IdxMask % 4 : IdxMask;
15406
15407 return IdxMask != 0x0c ? calculateSrcByte(Op: NextOp, DestByte: StartingIndex, SrcIndex: NextIndex)
15408 : ByteProvider<SDValue>(
15409 ByteProvider<SDValue>::getConstantZero());
15410 }
15411
15412 default: {
15413 return std::nullopt;
15414 }
15415 }
15416
15417 llvm_unreachable("fully handled switch");
15418}
15419
15420// Returns true if the Operand is a scalar and is 16 bits
15421static bool isExtendedFrom16Bits(SDValue &Operand) {
15422
15423 switch (Operand.getOpcode()) {
15424 case ISD::ANY_EXTEND:
15425 case ISD::SIGN_EXTEND:
15426 case ISD::ZERO_EXTEND: {
15427 auto OpVT = Operand.getOperand(i: 0).getValueType();
15428 return !OpVT.isVector() && OpVT.getSizeInBits() == 16;
15429 }
15430 case ISD::LOAD: {
15431 LoadSDNode *L = cast<LoadSDNode>(Val: Operand.getNode());
15432 auto ExtType = cast<LoadSDNode>(Val: L)->getExtensionType();
15433 if (ExtType == ISD::ZEXTLOAD || ExtType == ISD::SEXTLOAD ||
15434 ExtType == ISD::EXTLOAD) {
15435 auto MemVT = L->getMemoryVT();
15436 return !MemVT.isVector() && MemVT.getSizeInBits() == 16;
15437 }
15438 return L->getMemoryVT().getSizeInBits() == 16;
15439 }
15440 default:
15441 return false;
15442 }
15443}
15444
15445// Returns true if the mask matches consecutive bytes, and the first byte
15446// begins at a power of 2 byte offset from 0th byte
15447static bool addresses16Bits(int Mask) {
15448 int Low8 = Mask & 0xff;
15449 int Hi8 = (Mask & 0xff00) >> 8;
15450
15451 assert(Low8 < 8 && Hi8 < 8);
15452 // Are the bytes contiguous in the order of increasing addresses.
15453 bool IsConsecutive = (Hi8 - Low8 == 1);
15454 // Is the first byte at location that is aligned for 16 bit instructions.
15455 // A counter example is taking 2 consecutive bytes starting at the 8th bit.
15456 // In this case, we still need code to extract the 16 bit operand, so it
15457 // is better to use i8 v_perm
15458 bool Is16Aligned = !(Low8 % 2);
15459
15460 return IsConsecutive && Is16Aligned;
15461}
15462
15463// Do not lower into v_perm if the operands are actually 16 bit
15464// and the selected bits (based on PermMask) correspond with two
15465// easily addressable 16 bit operands.
15466static bool hasNon16BitAccesses(uint64_t PermMask, SDValue &Op,
15467 SDValue &OtherOp) {
15468 int Low16 = PermMask & 0xffff;
15469 int Hi16 = (PermMask & 0xffff0000) >> 16;
15470
15471 auto TempOp = peekThroughBitcasts(V: Op);
15472 auto TempOtherOp = peekThroughBitcasts(V: OtherOp);
15473
15474 auto OpIs16Bit =
15475 TempOp.getValueSizeInBits() == 16 || isExtendedFrom16Bits(Operand&: TempOp);
15476 if (!OpIs16Bit)
15477 return true;
15478
15479 auto OtherOpIs16Bit = TempOtherOp.getValueSizeInBits() == 16 ||
15480 isExtendedFrom16Bits(Operand&: TempOtherOp);
15481 if (!OtherOpIs16Bit)
15482 return true;
15483
15484 // Do we cleanly address both
15485 return !addresses16Bits(Mask: Low16) || !addresses16Bits(Mask: Hi16);
15486}
15487
15488static SDValue getDWordFromOffset(SelectionDAG &DAG, SDLoc SL, SDValue Src,
15489 unsigned DWordOffset) {
15490 SDValue Ret;
15491
15492 auto TypeSize = Src.getValueSizeInBits().getFixedValue();
15493 // ByteProvider must be at least 8 bits
15494 assert(Src.getValueSizeInBits().isKnownMultipleOf(8));
15495
15496 if (TypeSize <= 32)
15497 return DAG.getBitcastedAnyExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32);
15498
15499 if (Src.getValueType().isVector()) {
15500 auto ScalarTySize = Src.getScalarValueSizeInBits();
15501 auto ScalarTy = Src.getValueType().getScalarType();
15502 if (ScalarTySize == 32) {
15503 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Src,
15504 N2: DAG.getConstant(Val: DWordOffset, DL: SL, VT: MVT::i32));
15505 }
15506 if (ScalarTySize > 32) {
15507 Ret = DAG.getNode(
15508 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ScalarTy, N1: Src,
15509 N2: DAG.getConstant(Val: DWordOffset / (ScalarTySize / 32), DL: SL, VT: MVT::i32));
15510 auto ShiftVal = 32 * (DWordOffset % (ScalarTySize / 32));
15511 if (ShiftVal)
15512 Ret = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: Ret.getValueType(), N1: Ret,
15513 N2: DAG.getConstant(Val: ShiftVal, DL: SL, VT: MVT::i32));
15514 return DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
15515 }
15516
15517 assert(ScalarTySize < 32);
15518 if (TypeSize % 32 == 0) {
15519 assert(DWordOffset < TypeSize / 32);
15520 SDValue Cast = DAG.getBitcast(
15521 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: TypeSize / 32), V: Src);
15522 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Cast,
15523 N2: DAG.getConstant(Val: DWordOffset, DL: SL, VT: MVT::i32));
15524 }
15525
15526 auto NumElements = TypeSize / ScalarTySize;
15527 auto Trunc32Elements = (ScalarTySize * NumElements) / 32;
15528 auto NormalizedTrunc = Trunc32Elements * 32 / ScalarTySize;
15529 auto NumElementsIn32 = 32 / ScalarTySize;
15530 auto NumAvailElements = DWordOffset < Trunc32Elements
15531 ? NumElementsIn32
15532 : NumElements - NormalizedTrunc;
15533
15534 SmallVector<SDValue, 4> VecSrcs;
15535 DAG.ExtractVectorElements(Op: Src, Args&: VecSrcs, Start: DWordOffset * NumElementsIn32,
15536 Count: NumAvailElements);
15537
15538 Ret = DAG.getBuildVector(
15539 VT: MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ScalarTySize), NumElements: NumAvailElements), DL: SL,
15540 Ops: VecSrcs);
15541 return Ret = DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
15542 }
15543
15544 /// Scalar Type
15545 auto ShiftVal = 32 * DWordOffset;
15546 Ret = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: Src.getValueType(), N1: Src,
15547 N2: DAG.getConstant(Val: ShiftVal, DL: SL, VT: MVT::i32));
15548 return DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
15549}
15550
15551static SDValue matchPERM(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15552 SelectionDAG &DAG = DCI.DAG;
15553 [[maybe_unused]] EVT VT = N->getValueType(ResNo: 0);
15554 SmallVector<ByteProvider<SDValue>, 8> PermNodes;
15555
15556 // VT is known to be MVT::i32, so we need to provide 4 bytes.
15557 assert(VT == MVT::i32);
15558 for (int i = 0; i < 4; i++) {
15559 // Find the ByteProvider that provides the ith byte of the result of OR
15560 std::optional<ByteProvider<SDValue>> P =
15561 calculateByteProvider(Op: SDValue(N, 0), Index: i, Depth: 0, /*StartingIndex = */ i);
15562 // TODO support constantZero
15563 if (!P || P->isConstantZero())
15564 return SDValue();
15565
15566 PermNodes.push_back(Elt: *P);
15567 }
15568 if (PermNodes.size() != 4)
15569 return SDValue();
15570
15571 std::pair<unsigned, unsigned> FirstSrc(0, PermNodes[0].SrcOffset / 4);
15572 std::optional<std::pair<unsigned, unsigned>> SecondSrc;
15573 uint64_t PermMask = 0x00000000;
15574 for (size_t i = 0; i < PermNodes.size(); i++) {
15575 auto PermOp = PermNodes[i];
15576 // Since the mask is applied to Src1:Src2, Src1 bytes must be offset
15577 // by sizeof(Src2) = 4
15578 int SrcByteAdjust = 4;
15579
15580 // If the Src uses a byte from a different DWORD, then it corresponds
15581 // with a difference source
15582 if (!PermOp.hasSameSrc(Other: PermNodes[FirstSrc.first]) ||
15583 ((PermOp.SrcOffset / 4) != FirstSrc.second)) {
15584 if (SecondSrc)
15585 if (!PermOp.hasSameSrc(Other: PermNodes[SecondSrc->first]) ||
15586 ((PermOp.SrcOffset / 4) != SecondSrc->second))
15587 return SDValue();
15588
15589 // Set the index of the second distinct Src node
15590 SecondSrc = {i, PermNodes[i].SrcOffset / 4};
15591 assert(!(PermNodes[SecondSrc->first].Src->getValueSizeInBits() % 8));
15592 SrcByteAdjust = 0;
15593 }
15594 assert((PermOp.SrcOffset % 4) + SrcByteAdjust < 8);
15595 assert(!DAG.getDataLayout().isBigEndian());
15596 PermMask |= ((PermOp.SrcOffset % 4) + SrcByteAdjust) << (i * 8);
15597 }
15598 SDLoc DL(N);
15599 SDValue Op = *PermNodes[FirstSrc.first].Src;
15600 Op = getDWordFromOffset(DAG, SL: DL, Src: Op, DWordOffset: FirstSrc.second);
15601 assert(Op.getValueSizeInBits() == 32);
15602
15603 // Check that we are not just extracting the bytes in order from an op
15604 if (!SecondSrc) {
15605 int Low16 = PermMask & 0xffff;
15606 int Hi16 = (PermMask & 0xffff0000) >> 16;
15607
15608 bool WellFormedLow = (Low16 == 0x0504) || (Low16 == 0x0100);
15609 bool WellFormedHi = (Hi16 == 0x0706) || (Hi16 == 0x0302);
15610
15611 // The perm op would really just produce Op. So combine into Op
15612 if (WellFormedLow && WellFormedHi)
15613 return DAG.getBitcast(VT: MVT::getIntegerVT(BitWidth: 32), V: Op);
15614 }
15615
15616 SDValue OtherOp = SecondSrc ? *PermNodes[SecondSrc->first].Src : Op;
15617
15618 if (SecondSrc) {
15619 OtherOp = getDWordFromOffset(DAG, SL: DL, Src: OtherOp, DWordOffset: SecondSrc->second);
15620 assert(OtherOp.getValueSizeInBits() == 32);
15621 }
15622
15623 // Check that we haven't just recreated the same FSHR node.
15624 if (N->getOpcode() == ISD::FSHR &&
15625 (N->getOperand(Num: 0) == Op || N->getOperand(Num: 0) == OtherOp) &&
15626 (N->getOperand(Num: 1) == Op || N->getOperand(Num: 1) == OtherOp))
15627 return SDValue();
15628
15629 if (hasNon16BitAccesses(PermMask, Op, OtherOp)) {
15630
15631 assert(Op.getValueType().isByteSized() &&
15632 OtherOp.getValueType().isByteSized());
15633
15634 // If the ultimate src is less than 32 bits, then we will only be
15635 // using bytes 0: Op.getValueSizeInBytes() - 1 in the or.
15636 // CalculateByteProvider would not have returned Op as source if we
15637 // used a byte that is outside its ValueType. Thus, we are free to
15638 // ANY_EXTEND as the extended bits are dont-cares.
15639 Op = DAG.getBitcastedAnyExtOrTrunc(Op, DL, VT: MVT::i32);
15640 OtherOp = DAG.getBitcastedAnyExtOrTrunc(Op: OtherOp, DL, VT: MVT::i32);
15641
15642 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: Op, N2: OtherOp,
15643 N3: DAG.getConstant(Val: PermMask, DL, VT: MVT::i32));
15644 }
15645 return SDValue();
15646}
15647
15648SDValue SITargetLowering::performOrCombine(SDNode *N,
15649 DAGCombinerInfo &DCI) const {
15650 SelectionDAG &DAG = DCI.DAG;
15651 SDValue LHS = N->getOperand(Num: 0);
15652 SDValue RHS = N->getOperand(Num: 1);
15653
15654 EVT VT = N->getValueType(ResNo: 0);
15655 if (VT == MVT::i1) {
15656 // or (fp_class x, c1), (fp_class x, c2) -> fp_class x, (c1 | c2)
15657 if (LHS.getOpcode() == AMDGPUISD::FP_CLASS &&
15658 RHS.getOpcode() == AMDGPUISD::FP_CLASS) {
15659 SDValue Src = LHS.getOperand(i: 0);
15660 if (Src != RHS.getOperand(i: 0))
15661 return SDValue();
15662
15663 const ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
15664 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i: 1));
15665 if (!CLHS || !CRHS)
15666 return SDValue();
15667
15668 // Only 10 bits are used.
15669 static const uint32_t MaxMask = 0x3ff;
15670
15671 uint32_t NewMask =
15672 (CLHS->getZExtValue() | CRHS->getZExtValue()) & MaxMask;
15673 SDLoc DL(N);
15674 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: Src,
15675 N2: DAG.getConstant(Val: NewMask, DL, VT: MVT::i32));
15676 }
15677
15678 return SDValue();
15679 }
15680
15681 // or (perm x, y, c1), c2 -> perm x, y, permute_mask(c1, c2)
15682 if (isa<ConstantSDNode>(Val: RHS) && LHS.hasOneUse() &&
15683 LHS.getOpcode() == AMDGPUISD::PERM &&
15684 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2))) {
15685 uint32_t Sel = getConstantPermuteMask(C: N->getConstantOperandVal(Num: 1));
15686 if (!Sel)
15687 return SDValue();
15688
15689 Sel |= LHS.getConstantOperandVal(i: 2);
15690 SDLoc DL(N);
15691 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15692 N2: LHS.getOperand(i: 1), N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15693 }
15694
15695 // or (op x, c1), (op y, c2) -> perm x, y, permute_mask(c1, c2)
15696 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
15697 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() &&
15698 N->isDivergent() && TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
15699
15700 // If all the uses of an or need to extract the individual elements, do not
15701 // attempt to lower into v_perm
15702 auto usesCombinedOperand = [](SDNode *OrUse) {
15703 // If we have any non-vectorized use, then it is a candidate for v_perm
15704 if (OrUse->getOpcode() != ISD::BITCAST ||
15705 !OrUse->getValueType(ResNo: 0).isVector())
15706 return true;
15707
15708 // If we have any non-vectorized use, then it is a candidate for v_perm
15709 for (auto *VUser : OrUse->users()) {
15710 if (!VUser->getValueType(ResNo: 0).isVector())
15711 return true;
15712
15713 // If the use of a vector is a store, then combining via a v_perm
15714 // is beneficial.
15715 // TODO -- whitelist more uses
15716 for (auto VectorwiseOp : {ISD::STORE, ISD::CopyToReg, ISD::CopyFromReg})
15717 if (VUser->getOpcode() == VectorwiseOp)
15718 return true;
15719 }
15720 return false;
15721 };
15722
15723 if (!any_of(Range: N->users(), P: usesCombinedOperand))
15724 return SDValue();
15725
15726 uint32_t LHSMask = getPermuteMask(V: LHS);
15727 uint32_t RHSMask = getPermuteMask(V: RHS);
15728
15729 if (LHSMask != ~0u && RHSMask != ~0u) {
15730 // Canonicalize the expression in an attempt to have fewer unique masks
15731 // and therefore fewer registers used to hold the masks.
15732 if (LHSMask > RHSMask) {
15733 std::swap(a&: LHSMask, b&: RHSMask);
15734 std::swap(a&: LHS, b&: RHS);
15735 }
15736
15737 // Select 0xc for each lane used from source operand. Zero has 0xc mask
15738 // set, 0xff have 0xff in the mask, actual lanes are in the 0-3 range.
15739 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15740 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15741
15742 // Check of we need to combine values from two sources within a byte.
15743 if (!(LHSUsedLanes & RHSUsedLanes) &&
15744 // If we select high and lower word keep it for SDWA.
15745 // TODO: teach SDWA to work with v_perm_b32 and remove the check.
15746 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
15747 // Kill zero bytes selected by other mask. Zero value is 0xc.
15748 LHSMask &= ~RHSUsedLanes;
15749 RHSMask &= ~LHSUsedLanes;
15750 // Add 4 to each active LHS lane
15751 LHSMask |= LHSUsedLanes & 0x04040404;
15752 // Combine masks
15753 uint32_t Sel = LHSMask | RHSMask;
15754 SDLoc DL(N);
15755
15756 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15757 N2: RHS.getOperand(i: 0),
15758 N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15759 }
15760 }
15761 if (LHSMask == ~0u || RHSMask == ~0u) {
15762 if (SDValue Perm = matchPERM(N, DCI))
15763 return Perm;
15764 }
15765 }
15766
15767 // Detect identity v2i32 OR and replace with identity source node.
15768 // Specifically an Or that has operands constructed from the same source node
15769 // via extract_vector_elt and build_vector. I.E.
15770 // v2i32 or(
15771 // v2i32 build_vector(
15772 // i32 extract_elt(%IdentitySrc, 0),
15773 // i32 0
15774 // ),
15775 // v2i32 build_vector(
15776 // i32 0,
15777 // i32 extract_elt(%IdentitySrc, 1)
15778 // ) )
15779 // =>
15780 // v2i32 %IdentitySrc
15781
15782 if (VT == MVT::v2i32 && LHS->getOpcode() == ISD::BUILD_VECTOR &&
15783 RHS->getOpcode() == ISD::BUILD_VECTOR) {
15784
15785 ConstantSDNode *LC = dyn_cast<ConstantSDNode>(Val: LHS->getOperand(Num: 1));
15786 ConstantSDNode *RC = dyn_cast<ConstantSDNode>(Val: RHS->getOperand(Num: 0));
15787
15788 // Test for and normalise build vectors.
15789 if (LC && RC && LC->getZExtValue() == 0 && RC->getZExtValue() == 0) {
15790
15791 // Get the extract_vector_element operands.
15792 SDValue LEVE = LHS->getOperand(Num: 0);
15793 SDValue REVE = RHS->getOperand(Num: 1);
15794
15795 if (LEVE->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15796 REVE->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
15797 // Check that different elements from the same vector are
15798 // extracted.
15799 if (LEVE->getOperand(Num: 0) == REVE->getOperand(Num: 0) &&
15800 LEVE->getOperand(Num: 1) != REVE->getOperand(Num: 1)) {
15801 SDValue IdentitySrc = LEVE.getOperand(i: 0);
15802 return IdentitySrc;
15803 }
15804 }
15805 }
15806 }
15807
15808 if (VT != MVT::i64 || DCI.isBeforeLegalizeOps())
15809 return SDValue();
15810
15811 // TODO: This could be a generic combine with a predicate for extracting the
15812 // high half of an integer being free.
15813
15814 // (or i64:x, (zero_extend i32:y)) ->
15815 // i64 (bitcast (v2i32 build_vector (or i32:y, lo_32(x)), hi_32(x)))
15816 if (LHS.getOpcode() == ISD::ZERO_EXTEND &&
15817 RHS.getOpcode() != ISD::ZERO_EXTEND)
15818 std::swap(a&: LHS, b&: RHS);
15819
15820 if (RHS.getOpcode() == ISD::ZERO_EXTEND) {
15821 SDValue ExtSrc = RHS.getOperand(i: 0);
15822 EVT SrcVT = ExtSrc.getValueType();
15823 if (SrcVT == MVT::i32) {
15824 SDLoc SL(N);
15825 auto [LowLHS, HiBits] = split64BitValue(Op: LHS, DAG);
15826 SDValue LowOr = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: LowLHS, N2: ExtSrc);
15827
15828 DCI.AddToWorklist(N: LowOr.getNode());
15829 DCI.AddToWorklist(N: HiBits.getNode());
15830
15831 SDValue Vec =
15832 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: LowOr, N2: HiBits);
15833 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
15834 }
15835 }
15836
15837 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
15838 if (CRHS) {
15839 if (SDValue Split = splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::OR,
15840 LHS: N->getOperand(Num: 0), CRHS))
15841 return Split;
15842 }
15843
15844 return SDValue();
15845}
15846
15847SDValue SITargetLowering::performXorCombine(SDNode *N,
15848 DAGCombinerInfo &DCI) const {
15849 if (SDValue RV = reassociateScalarOps(N, DAG&: DCI.DAG))
15850 return RV;
15851
15852 SDValue LHS = N->getOperand(Num: 0);
15853 SDValue RHS = N->getOperand(Num: 1);
15854
15855 const ConstantSDNode *CRHS = isConstOrConstSplat(N: RHS);
15856 SelectionDAG &DAG = DCI.DAG;
15857
15858 EVT VT = N->getValueType(ResNo: 0);
15859 if (CRHS && VT == MVT::i64) {
15860 if (SDValue Split =
15861 splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::XOR, LHS, CRHS))
15862 return Split;
15863 }
15864
15865 // v2i32 (xor (vselect cc, x, y), K) ->
15866 // (v2i32 svelect cc, (xor x, K), (xor y, K)) This enables the xor to be
15867 // replaced with source modifiers when the select is lowered to CNDMASK.
15868 unsigned Opc = LHS.getOpcode();
15869 if (((Opc == ISD::VSELECT && VT == MVT::v2i32) ||
15870 (Opc == ISD::SELECT && VT == MVT::i64)) &&
15871 CRHS && CRHS->getAPIntValue().isSignMask()) {
15872 SDValue CC = LHS->getOperand(Num: 0);
15873 SDValue TRUE = LHS->getOperand(Num: 1);
15874 SDValue FALSE = LHS->getOperand(Num: 2);
15875 SDValue XTrue = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N), VT, N1: TRUE, N2: RHS);
15876 SDValue XFalse = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N), VT, N1: FALSE, N2: RHS);
15877 SDValue XSelect =
15878 DAG.getNode(Opcode: ISD::VSELECT, DL: SDLoc(N), VT, N1: CC, N2: XTrue, N3: XFalse);
15879 return XSelect;
15880 }
15881
15882 // Make sure to apply the 64-bit constant splitting fold before trying to fold
15883 // fneg-like xors into 64-bit select.
15884 if (LHS.getOpcode() == ISD::SELECT && VT == MVT::i32) {
15885 // This looks like an fneg, try to fold as a source modifier.
15886 if (CRHS && CRHS->getAPIntValue().isSignMask() &&
15887 shouldFoldFNegIntoSrc(FNeg: N, FNegSrc: LHS)) {
15888 // xor (select c, a, b), 0x80000000 ->
15889 // bitcast (select c, (fneg (bitcast a)), (fneg (bitcast b)))
15890 SDLoc DL(N);
15891 SDValue CastLHS =
15892 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: LHS->getOperand(Num: 1));
15893 SDValue CastRHS =
15894 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: LHS->getOperand(Num: 2));
15895 SDValue FNegLHS = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f32, Operand: CastLHS);
15896 SDValue FNegRHS = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f32, Operand: CastRHS);
15897 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::f32,
15898 N1: LHS->getOperand(Num: 0), N2: FNegLHS, N3: FNegRHS);
15899 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: NewSelect);
15900 }
15901 }
15902
15903 return SDValue();
15904}
15905
15906SDValue
15907SITargetLowering::performZeroOrAnyExtendCombine(SDNode *N,
15908 DAGCombinerInfo &DCI) const {
15909 if (!Subtarget->has16BitInsts() ||
15910 DCI.getDAGCombineLevel() < AfterLegalizeTypes)
15911 return SDValue();
15912
15913 EVT VT = N->getValueType(ResNo: 0);
15914 if (VT != MVT::i32)
15915 return SDValue();
15916
15917 SDValue Src = N->getOperand(Num: 0);
15918 if (Src.getValueType() != MVT::i16)
15919 return SDValue();
15920
15921 if (!Src->hasOneUse())
15922 return SDValue();
15923
15924 // TODO: We bail out below if SrcOffset is not in the first dword (>= 4). It's
15925 // possible we're missing out on some combine opportunities, but we'd need to
15926 // weigh the cost of extracting the byte from the upper dwords.
15927
15928 std::optional<ByteProvider<SDValue>> BP0 =
15929 calculateByteProvider(Op: SDValue(N, 0), Index: 0, Depth: 0, StartingIndex: 0);
15930 if (!BP0 || BP0->SrcOffset >= 4 || !BP0->Src)
15931 return SDValue();
15932 SDValue V0 = *BP0->Src;
15933
15934 std::optional<ByteProvider<SDValue>> BP1 =
15935 calculateByteProvider(Op: SDValue(N, 0), Index: 1, Depth: 0, StartingIndex: 1);
15936 if (!BP1 || BP1->SrcOffset >= 4 || !BP1->Src)
15937 return SDValue();
15938
15939 SDValue V1 = *BP1->Src;
15940
15941 if (V0 == V1)
15942 return SDValue();
15943
15944 SelectionDAG &DAG = DCI.DAG;
15945 SDLoc DL(N);
15946 uint32_t PermMask = 0x0c0c0c0c;
15947 if (V0) {
15948 V0 = DAG.getBitcastedAnyExtOrTrunc(Op: V0, DL, VT: MVT::i32);
15949 PermMask = (PermMask & ~0xFF) | (BP0->SrcOffset + 4);
15950 }
15951
15952 if (V1) {
15953 V1 = DAG.getBitcastedAnyExtOrTrunc(Op: V1, DL, VT: MVT::i32);
15954 PermMask = (PermMask & ~(0xFF << 8)) | (BP1->SrcOffset << 8);
15955 }
15956
15957 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: V0, N2: V1,
15958 N3: DAG.getConstant(Val: PermMask, DL, VT: MVT::i32));
15959}
15960
15961SDValue
15962SITargetLowering::performSignExtendInRegCombine(SDNode *N,
15963 DAGCombinerInfo &DCI) const {
15964 SDValue Src = N->getOperand(Num: 0);
15965 auto *VTSign = cast<VTSDNode>(Val: N->getOperand(Num: 1));
15966
15967 // Combine s_buffer_load_u8 or s_buffer_load_u16 with sext and replace them
15968 // with s_buffer_load_i8 and s_buffer_load_i16 respectively.
15969 if (((Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE &&
15970 VTSign->getVT() == MVT::i8) ||
15971 (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_USHORT &&
15972 VTSign->getVT() == MVT::i16))) {
15973 assert(Subtarget->hasScalarSubwordLoads() &&
15974 "s_buffer_load_{u8, i8} are supported "
15975 "in GFX12 (or newer) architectures.");
15976 unsigned Opc = (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE)
15977 ? AMDGPUISD::SBUFFER_LOAD_BYTE
15978 : AMDGPUISD::SBUFFER_LOAD_SHORT;
15979 SDLoc DL(N);
15980 SDVTList ResList =
15981 DCI.DAG.getVTList(VT1: MVT::i32, VT2: Src.getOperand(i: 0).getValueType());
15982 SDValue Ops[] = {
15983 Src.getOperand(i: 0), // Chain
15984 Src.getOperand(i: 1), // source register
15985 Src.getOperand(i: 2), // offset
15986 Src.getOperand(i: 3) // cachePolicy
15987 };
15988 auto *M = cast<MemSDNode>(Val&: Src);
15989 SDValue BufferLoad = DCI.DAG.getMemIntrinsicNode(
15990 Opcode: Opc, dl: DL, VTList: ResList, Ops, MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
15991 return DCI.DAG.getMergeValues(Ops: {BufferLoad, BufferLoad.getValue(R: 1)}, dl: DL);
15992 }
15993 if (((Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE &&
15994 VTSign->getVT() == MVT::i8) ||
15995 (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_USHORT &&
15996 VTSign->getVT() == MVT::i16)) &&
15997 Src.hasOneUse()) {
15998 auto *M = cast<MemSDNode>(Val&: Src);
15999 SDValue Ops[] = {Src.getOperand(i: 0), // Chain
16000 Src.getOperand(i: 1), // rsrc
16001 Src.getOperand(i: 2), // vindex
16002 Src.getOperand(i: 3), // voffset
16003 Src.getOperand(i: 4), // soffset
16004 Src.getOperand(i: 5), // offset
16005 Src.getOperand(i: 6), Src.getOperand(i: 7)};
16006 // replace with BUFFER_LOAD_BYTE/SHORT
16007 SDVTList ResList =
16008 DCI.DAG.getVTList(VT1: MVT::i32, VT2: Src.getOperand(i: 0).getValueType());
16009 unsigned Opc = (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE)
16010 ? AMDGPUISD::BUFFER_LOAD_BYTE
16011 : AMDGPUISD::BUFFER_LOAD_SHORT;
16012 SDValue BufferLoadSignExt = DCI.DAG.getMemIntrinsicNode(
16013 Opcode: Opc, dl: SDLoc(N), VTList: ResList, Ops, MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
16014 return DCI.DAG.getMergeValues(
16015 Ops: {BufferLoadSignExt, BufferLoadSignExt.getValue(R: 1)}, dl: SDLoc(N));
16016 }
16017 return SDValue();
16018}
16019
16020SDValue SITargetLowering::performClassCombine(SDNode *N,
16021 DAGCombinerInfo &DCI) const {
16022 SelectionDAG &DAG = DCI.DAG;
16023 SDValue Mask = N->getOperand(Num: 1);
16024
16025 // fp_class x, 0 -> false
16026 if (isNullConstant(V: Mask))
16027 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT: MVT::i1);
16028
16029 if (N->getOperand(Num: 0).isUndef())
16030 return DAG.getUNDEF(VT: MVT::i1);
16031
16032 return SDValue();
16033}
16034
16035SDValue SITargetLowering::performRcpCombine(SDNode *N,
16036 DAGCombinerInfo &DCI) const {
16037 EVT VT = N->getValueType(ResNo: 0);
16038 SDValue N0 = N->getOperand(Num: 0);
16039
16040 if (N0.isUndef()) {
16041 return DCI.DAG.getConstantFP(Val: APFloat::getQNaN(Sem: VT.getFltSemantics()),
16042 DL: SDLoc(N), VT);
16043 }
16044
16045 // TODO: Could handle f32 + amdgcn.sqrt but probably never reaches here.
16046 if ((VT == MVT::f16 && N0.getOpcode() == ISD::FSQRT) &&
16047 N->getFlags().hasAllowContract() && N0->getFlags().hasAllowContract()) {
16048 return DCI.DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SDLoc(N), VT, Operand: N0.getOperand(i: 0),
16049 Flags: N->getFlags());
16050 }
16051
16052 return AMDGPUTargetLowering::performRcpCombine(N, DCI);
16053}
16054
16055bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
16056 SDNodeFlags UserFlags,
16057 unsigned MaxDepth) const {
16058 EVT VT = Op.getValueType();
16059 assert(VT.isFloatingPoint() &&
16060 "expected a floating-point value to query canonicality of");
16061 return isCanonicalized(DAG, Op, QueryVT: VT.getScalarType(), UserFlags, MaxDepth);
16062}
16063
16064bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
16065 EVT QueryVT, SDNodeFlags UserFlags,
16066 unsigned MaxDepth) const {
16067 assert(QueryVT.isFloatingPoint() && !QueryVT.isVector() &&
16068 "QueryVT must be a floating-point scalar type");
16069 EVT VT = Op.getValueType();
16070 if (VT.isFloatingPoint() && VT.getScalarType() != QueryVT)
16071 return false;
16072
16073 unsigned Opcode = Op.getOpcode();
16074 if (Opcode == ISD::FCANONICALIZE)
16075 return true;
16076
16077 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
16078 const auto &F = CFP->getValueAPF();
16079 if (F.isNaN() && F.isSignaling())
16080 return false;
16081 if (!F.isDenormal())
16082 return true;
16083
16084 DenormalMode Mode =
16085 DAG.getMachineFunction().getDenormalMode(FPType: F.getSemantics());
16086 return Mode == DenormalMode::getIEEE();
16087 }
16088
16089 // If source is a result of another standard FP operation it is already in
16090 // canonical form.
16091 if (MaxDepth == 0)
16092 return false;
16093
16094 switch (Opcode) {
16095 // These will flush denorms if required.
16096 case ISD::FADD:
16097 case ISD::FSUB:
16098 case ISD::FMUL:
16099 case ISD::FCEIL:
16100 case ISD::FFLOOR:
16101 case ISD::FMA:
16102 case ISD::FMAD:
16103 case ISD::FSQRT:
16104 case ISD::FDIV:
16105 case ISD::FREM:
16106 case ISD::FP_ROUND:
16107 case ISD::FP_EXTEND:
16108 case ISD::FP16_TO_FP:
16109 case ISD::FP_TO_FP16:
16110 case ISD::BF16_TO_FP:
16111 case ISD::FP_TO_BF16:
16112 case ISD::FLDEXP:
16113 case AMDGPUISD::FMUL_LEGACY:
16114 case AMDGPUISD::FMAD_FTZ:
16115 case AMDGPUISD::RCP:
16116 case AMDGPUISD::RSQ:
16117 case AMDGPUISD::RSQ_CLAMP:
16118 case AMDGPUISD::RCP_LEGACY:
16119 case AMDGPUISD::RCP_IFLAG:
16120 case AMDGPUISD::LOG:
16121 case AMDGPUISD::EXP:
16122 case AMDGPUISD::DIV_SCALE:
16123 case AMDGPUISD::DIV_FMAS:
16124 case AMDGPUISD::DIV_FIXUP:
16125 case AMDGPUISD::FRACT:
16126 case AMDGPUISD::CVT_PKRTZ_F16_F32:
16127 case AMDGPUISD::CVT_F32_UBYTE0:
16128 case AMDGPUISD::CVT_F32_UBYTE1:
16129 case AMDGPUISD::CVT_F32_UBYTE2:
16130 case AMDGPUISD::CVT_F32_UBYTE3:
16131 case AMDGPUISD::FP_TO_FP16:
16132 case AMDGPUISD::SIN_HW:
16133 case AMDGPUISD::COS_HW:
16134 return true;
16135
16136 // It can/will be lowered or combined as a bit operation.
16137 // Need to check their input recursively to handle.
16138 case ISD::FNEG:
16139 case ISD::FABS:
16140 case ISD::FCOPYSIGN:
16141 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), QueryVT, UserFlags,
16142 MaxDepth: MaxDepth - 1);
16143
16144 case ISD::AND:
16145 if (Op.getValueType() == MVT::i32) {
16146 // Be careful as we only know it is a bitcast floating point type. It
16147 // could be f32, v2f16, we have no way of knowing. Luckily the constant
16148 // value that we optimize for, which comes up in fp32 to bf16 conversions,
16149 // is valid to optimize for all types.
16150 if (auto *RHS = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
16151 if (RHS->getZExtValue() == 0xffff0000) {
16152 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), QueryVT, UserFlags,
16153 MaxDepth: MaxDepth - 1);
16154 }
16155 }
16156 }
16157 break;
16158
16159 case ISD::FSIN:
16160 case ISD::FCOS:
16161 case ISD::FSINCOS:
16162 return Op.getValueType().getScalarType() != MVT::f16;
16163
16164 case ISD::FMINNUM:
16165 case ISD::FMAXNUM:
16166 case ISD::FMINNUM_IEEE:
16167 case ISD::FMAXNUM_IEEE:
16168 case ISD::FMINIMUM:
16169 case ISD::FMAXIMUM:
16170 case ISD::FMINIMUMNUM:
16171 case ISD::FMAXIMUMNUM:
16172 case AMDGPUISD::CLAMP:
16173 case AMDGPUISD::FMED3:
16174 case AMDGPUISD::FMAX3:
16175 case AMDGPUISD::FMIN3:
16176 case AMDGPUISD::FMAXIMUM3:
16177 case AMDGPUISD::FMINIMUM3: {
16178 // FIXME: Shouldn't treat the generic operations different based these.
16179 // However, we aren't really required to flush the result from
16180 // minnum/maxnum..
16181
16182 // snans will be quieted, so we only need to worry about denormals.
16183 if (Subtarget->supportsMinMaxDenormModes() ||
16184 // FIXME: denormalsEnabledForType is broken for dynamic
16185 denormalsEnabledForType(DAG, VT: Op.getValueType()))
16186 return true;
16187
16188 // Flushing may be required.
16189 // In pre-GFX9 targets V_MIN_F32 and others do not flush denorms. For such
16190 // targets need to check their input recursively.
16191
16192 // FIXME: Does this apply with clamp? It's implemented with max.
16193 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
16194 if (!isCanonicalized(DAG, Op: Op.getOperand(i: I), QueryVT, UserFlags,
16195 MaxDepth: MaxDepth - 1))
16196 return false;
16197 }
16198
16199 return true;
16200 }
16201 case ISD::SELECT: {
16202 return isCanonicalized(DAG, Op: Op.getOperand(i: 1), QueryVT, UserFlags,
16203 MaxDepth: MaxDepth - 1) &&
16204 isCanonicalized(DAG, Op: Op.getOperand(i: 2), QueryVT, UserFlags,
16205 MaxDepth: MaxDepth - 1);
16206 }
16207 case ISD::BUILD_VECTOR: {
16208 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
16209 SDValue SrcOp = Op.getOperand(i);
16210 if (!isCanonicalized(DAG, Op: SrcOp, QueryVT, UserFlags, MaxDepth: MaxDepth - 1))
16211 return false;
16212 }
16213
16214 return true;
16215 }
16216 case ISD::EXTRACT_VECTOR_ELT:
16217 case ISD::EXTRACT_SUBVECTOR: {
16218 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), QueryVT, UserFlags,
16219 MaxDepth: MaxDepth - 1);
16220 }
16221 case ISD::INSERT_VECTOR_ELT: {
16222 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), QueryVT, UserFlags,
16223 MaxDepth: MaxDepth - 1) &&
16224 isCanonicalized(DAG, Op: Op.getOperand(i: 1), QueryVT, UserFlags,
16225 MaxDepth: MaxDepth - 1);
16226 }
16227 case ISD::POISON:
16228 return true;
16229 case ISD::UNDEF:
16230 // Could be anything.
16231 return false;
16232
16233 case ISD::BITCAST: {
16234 // Carry QueryVT through the bitcast unchanged. The top-of-function guard
16235 // rejects a source whose FP format differs from the consumed type, so a
16236 // value canonical in one FP format is not assumed canonical in another.
16237 SDValue Src = peekThroughBitcasts(V: Op.getOperand(i: 0));
16238 return isCanonicalized(DAG, Op: Src, QueryVT, UserFlags, MaxDepth: MaxDepth - 1);
16239 }
16240 case ISD::TRUNCATE: {
16241 // Hack round the mess we make when legalizing extract_vector_elt
16242 if (Op.getValueType() == MVT::i16) {
16243 SDValue TruncSrc = Op.getOperand(i: 0);
16244 if (TruncSrc.getValueType() == MVT::i32 &&
16245 TruncSrc.getOpcode() == ISD::BITCAST &&
16246 TruncSrc.getOperand(i: 0).getValueType() == MVT::v2f16) {
16247 return isCanonicalized(DAG, Op: TruncSrc.getOperand(i: 0), QueryVT, UserFlags,
16248 MaxDepth: MaxDepth - 1);
16249 }
16250 }
16251 return false;
16252 }
16253 case ISD::INTRINSIC_WO_CHAIN: {
16254 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
16255 // TODO: Handle more intrinsics
16256 switch (IntrinsicID) {
16257 case Intrinsic::amdgcn_cvt_pkrtz:
16258 case Intrinsic::amdgcn_cubeid:
16259 case Intrinsic::amdgcn_frexp_mant:
16260 case Intrinsic::amdgcn_fdot2:
16261 case Intrinsic::amdgcn_rcp:
16262 case Intrinsic::amdgcn_rsq:
16263 case Intrinsic::amdgcn_rsq_clamp:
16264 case Intrinsic::amdgcn_rcp_legacy:
16265 case Intrinsic::amdgcn_rsq_legacy:
16266 case Intrinsic::amdgcn_trig_preop:
16267 case Intrinsic::amdgcn_tanh:
16268 case Intrinsic::amdgcn_log:
16269 case Intrinsic::amdgcn_exp2:
16270 case Intrinsic::amdgcn_sqrt:
16271 return true;
16272 default:
16273 break;
16274 }
16275
16276 break;
16277 }
16278 default:
16279 break;
16280 }
16281
16282 // FIXME: denormalsEnabledForType is broken for dynamic
16283 return denormalsEnabledForType(DAG, VT: Op.getValueType()) &&
16284 (UserFlags.hasNoNaNs() || DAG.isKnownNeverSNaN(Op));
16285}
16286
16287bool SITargetLowering::isCanonicalized(Register Reg, const MachineFunction &MF,
16288 unsigned MaxDepth) const {
16289 const MachineRegisterInfo &MRI = MF.getRegInfo();
16290 MachineInstr *MI = MRI.getVRegDef(Reg);
16291 unsigned Opcode = MI->getOpcode();
16292
16293 if (Opcode == AMDGPU::G_FCANONICALIZE)
16294 return true;
16295
16296 std::optional<FPValueAndVReg> FCR;
16297 // Constant splat (can be padded with undef) or scalar constant.
16298 if (mi_match(R: Reg, MRI, P: MIPatternMatch::m_GFCstOrSplat(FPValReg&: FCR))) {
16299 if (FCR->Value.isSignaling())
16300 return false;
16301 if (!FCR->Value.isDenormal())
16302 return true;
16303
16304 DenormalMode Mode = MF.getDenormalMode(FPType: FCR->Value.getSemantics());
16305 return Mode == DenormalMode::getIEEE();
16306 }
16307
16308 if (MaxDepth == 0)
16309 return false;
16310
16311 switch (Opcode) {
16312 case AMDGPU::G_FADD:
16313 case AMDGPU::G_FSUB:
16314 case AMDGPU::G_FMUL:
16315 case AMDGPU::G_FCEIL:
16316 case AMDGPU::G_FFLOOR:
16317 case AMDGPU::G_FRINT:
16318 case AMDGPU::G_FNEARBYINT:
16319 case AMDGPU::G_INTRINSIC_FPTRUNC_ROUND:
16320 case AMDGPU::G_INTRINSIC_TRUNC:
16321 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
16322 case AMDGPU::G_FMA:
16323 case AMDGPU::G_FMAD:
16324 case AMDGPU::G_FSQRT:
16325 case AMDGPU::G_FDIV:
16326 case AMDGPU::G_FREM:
16327 case AMDGPU::G_FPOW:
16328 case AMDGPU::G_FPEXT:
16329 case AMDGPU::G_FLOG:
16330 case AMDGPU::G_FLOG2:
16331 case AMDGPU::G_FLOG10:
16332 case AMDGPU::G_FPTRUNC:
16333 case AMDGPU::G_AMDGPU_RCP_IFLAG:
16334 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
16335 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
16336 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
16337 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
16338 return true;
16339 case AMDGPU::G_FNEG:
16340 case AMDGPU::G_FABS:
16341 case AMDGPU::G_FCOPYSIGN:
16342 return isCanonicalized(Reg: MI->getOperand(i: 1).getReg(), MF, MaxDepth: MaxDepth - 1);
16343 case AMDGPU::G_FMINNUM:
16344 case AMDGPU::G_FMAXNUM:
16345 case AMDGPU::G_FMINNUM_IEEE:
16346 case AMDGPU::G_FMAXNUM_IEEE:
16347 case AMDGPU::G_FMINIMUM:
16348 case AMDGPU::G_FMAXIMUM:
16349 case AMDGPU::G_FMINIMUMNUM:
16350 case AMDGPU::G_FMAXIMUMNUM: {
16351 if (Subtarget->supportsMinMaxDenormModes() ||
16352 // FIXME: denormalsEnabledForType is broken for dynamic
16353 denormalsEnabledForType(Ty: MRI.getType(Reg), MF))
16354 return true;
16355
16356 [[fallthrough]];
16357 }
16358 case AMDGPU::G_BUILD_VECTOR:
16359 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands()))
16360 if (!isCanonicalized(Reg: MO.getReg(), MF, MaxDepth: MaxDepth - 1))
16361 return false;
16362 return true;
16363 case AMDGPU::G_INTRINSIC:
16364 case AMDGPU::G_INTRINSIC_CONVERGENT:
16365 switch (cast<GIntrinsic>(Val: MI)->getIntrinsicID()) {
16366 case Intrinsic::amdgcn_fmul_legacy:
16367 case Intrinsic::amdgcn_fmad_ftz:
16368 case Intrinsic::amdgcn_sqrt:
16369 case Intrinsic::amdgcn_fmed3:
16370 case Intrinsic::amdgcn_sin:
16371 case Intrinsic::amdgcn_cos:
16372 case Intrinsic::amdgcn_log:
16373 case Intrinsic::amdgcn_exp2:
16374 case Intrinsic::amdgcn_log_clamp:
16375 case Intrinsic::amdgcn_rcp:
16376 case Intrinsic::amdgcn_rcp_legacy:
16377 case Intrinsic::amdgcn_rsq:
16378 case Intrinsic::amdgcn_rsq_clamp:
16379 case Intrinsic::amdgcn_rsq_legacy:
16380 case Intrinsic::amdgcn_div_scale:
16381 case Intrinsic::amdgcn_div_fmas:
16382 case Intrinsic::amdgcn_div_fixup:
16383 case Intrinsic::amdgcn_fract:
16384 case Intrinsic::amdgcn_cvt_pkrtz:
16385 case Intrinsic::amdgcn_cubeid:
16386 case Intrinsic::amdgcn_cubema:
16387 case Intrinsic::amdgcn_cubesc:
16388 case Intrinsic::amdgcn_cubetc:
16389 case Intrinsic::amdgcn_frexp_mant:
16390 case Intrinsic::amdgcn_fdot2:
16391 case Intrinsic::amdgcn_trig_preop:
16392 case Intrinsic::amdgcn_tanh:
16393 return true;
16394 default:
16395 break;
16396 }
16397
16398 [[fallthrough]];
16399 default:
16400 return false;
16401 }
16402
16403 llvm_unreachable("invalid operation");
16404}
16405
16406// Constant fold canonicalize.
16407SDValue SITargetLowering::getCanonicalConstantFP(SelectionDAG &DAG,
16408 const SDLoc &SL, EVT VT,
16409 const APFloat &C) const {
16410 // Flush denormals to 0 if not enabled.
16411 if (C.isDenormal()) {
16412 DenormalMode Mode =
16413 DAG.getMachineFunction().getDenormalMode(FPType: C.getSemantics());
16414 if (Mode == DenormalMode::getPreserveSign()) {
16415 return DAG.getConstantFP(
16416 Val: APFloat::getZero(Sem: C.getSemantics(), Negative: C.isNegative()), DL: SL, VT);
16417 }
16418
16419 if (Mode != DenormalMode::getIEEE())
16420 return SDValue();
16421 }
16422
16423 if (C.isNaN()) {
16424 if (C.isSignaling()) {
16425 // Quiet a signaling NaN.
16426 return DAG.getConstantFP(Val: C.makeQuiet(), DL: SL, VT);
16427 }
16428 }
16429
16430 // Already canonical.
16431 return DAG.getConstantFP(Val: C, DL: SL, VT);
16432}
16433
16434static bool vectorEltWillFoldAway(SDValue Op) {
16435 return Op.isUndef() || isa<ConstantFPSDNode>(Val: Op);
16436}
16437
16438SDValue
16439SITargetLowering::performFCanonicalizeCombine(SDNode *N,
16440 DAGCombinerInfo &DCI) const {
16441 SelectionDAG &DAG = DCI.DAG;
16442 SDValue N0 = N->getOperand(Num: 0);
16443 EVT VT = N->getValueType(ResNo: 0);
16444
16445 // fcanonicalize undef -> qnan
16446 if (N0.isUndef()) {
16447 APFloat QNaN = APFloat::getQNaN(Sem: VT.getFltSemantics());
16448 return DAG.getConstantFP(Val: QNaN, DL: SDLoc(N), VT);
16449 }
16450
16451 if (ConstantFPSDNode *CFP = isConstOrConstSplatFP(N: N0))
16452 return getCanonicalConstantFP(DAG, SL: SDLoc(N), VT, C: CFP->getValueAPF());
16453
16454 // fcanonicalize (build_vector x, k) -> build_vector (fcanonicalize x),
16455 // (fcanonicalize k)
16456 //
16457 // fcanonicalize (build_vector x, undef) -> build_vector (fcanonicalize x), 0
16458
16459 // TODO: This could be better with wider vectors that will be split to v2f16,
16460 // and to consider uses since there aren't that many packed operations.
16461 if (N0.getOpcode() == ISD::BUILD_VECTOR && N0.getNumOperands() == 2 &&
16462 isTypeLegal(VT)) {
16463 SDLoc SL(N);
16464 SDValue NewElts[2];
16465 SDValue Lo = N0.getOperand(i: 0);
16466 SDValue Hi = N0.getOperand(i: 1);
16467 EVT EltVT = Lo.getValueType();
16468
16469 // Only apply this optimization if scalar canonicalize is legal for the
16470 // element type. Otherwise, scalarizing may require widening the scalar back
16471 // to a vector, adding overhead (e.g., bf16 has no scalar instructions).
16472 if (getOperationAction(Op: ISD::FCANONICALIZE, VT: EltVT) != Legal)
16473 return SDValue();
16474
16475 if (vectorEltWillFoldAway(Op: Lo) || vectorEltWillFoldAway(Op: Hi)) {
16476 for (unsigned I = 0; I != 2; ++I) {
16477 SDValue Op = N0.getOperand(i: I);
16478 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
16479 NewElts[I] =
16480 getCanonicalConstantFP(DAG, SL, VT: EltVT, C: CFP->getValueAPF());
16481 } else if (Op.isUndef()) {
16482 // Handled below based on what the other operand is.
16483 NewElts[I] = Op;
16484 } else {
16485 NewElts[I] = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: SL, VT: EltVT, Operand: Op);
16486 }
16487 }
16488
16489 // If one half is undef, and one is constant, prefer a splat vector.
16490 // Otherwise, convert the undef to 0.0 since that's cheaper to use and may
16491 // be free with a packed operation.
16492 if (NewElts[0].isUndef()) {
16493 NewElts[0] = isa<ConstantFPSDNode>(Val: NewElts[1])
16494 ? NewElts[1]
16495 : DAG.getConstantFP(Val: 0.0f, DL: SL, VT: EltVT);
16496 }
16497
16498 if (NewElts[1].isUndef()) {
16499 NewElts[1] = isa<ConstantFPSDNode>(Val: NewElts[0])
16500 ? NewElts[0]
16501 : DAG.getConstantFP(Val: 0.0f, DL: SL, VT: EltVT);
16502 }
16503
16504 return DAG.getBuildVector(VT, DL: SL, Ops: NewElts);
16505 }
16506 }
16507
16508 return SDValue();
16509}
16510
16511static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc) {
16512 switch (Opc) {
16513 case ISD::FMAXNUM:
16514 case ISD::FMAXNUM_IEEE:
16515 case ISD::FMAXIMUMNUM:
16516 return AMDGPUISD::FMAX3;
16517 case ISD::FMAXIMUM:
16518 return AMDGPUISD::FMAXIMUM3;
16519 case ISD::SMAX:
16520 return AMDGPUISD::SMAX3;
16521 case ISD::UMAX:
16522 return AMDGPUISD::UMAX3;
16523 case ISD::FMINNUM:
16524 case ISD::FMINNUM_IEEE:
16525 case ISD::FMINIMUMNUM:
16526 return AMDGPUISD::FMIN3;
16527 case ISD::FMINIMUM:
16528 return AMDGPUISD::FMINIMUM3;
16529 case ISD::SMIN:
16530 return AMDGPUISD::SMIN3;
16531 case ISD::UMIN:
16532 return AMDGPUISD::UMIN3;
16533 default:
16534 llvm_unreachable("Not a min/max opcode");
16535 }
16536}
16537
16538SDValue SITargetLowering::performIntMed3ImmCombine(SelectionDAG &DAG,
16539 const SDLoc &SL, SDValue Src,
16540 SDValue MinVal,
16541 SDValue MaxVal,
16542 bool Signed) const {
16543
16544 // med3 comes from
16545 // min(max(x, K0), K1), K0 < K1
16546 // max(min(x, K0), K1), K1 < K0
16547 //
16548 // "MinVal" and "MaxVal" respectively refer to the rhs of the
16549 // min/max op.
16550 ConstantSDNode *MinK = dyn_cast<ConstantSDNode>(Val&: MinVal);
16551 ConstantSDNode *MaxK = dyn_cast<ConstantSDNode>(Val&: MaxVal);
16552
16553 if (!MinK || !MaxK)
16554 return SDValue();
16555
16556 if (Signed) {
16557 if (MaxK->getAPIntValue().sge(RHS: MinK->getAPIntValue()))
16558 return SDValue();
16559 } else {
16560 if (MaxK->getAPIntValue().uge(RHS: MinK->getAPIntValue()))
16561 return SDValue();
16562 }
16563
16564 EVT VT = MinK->getValueType(ResNo: 0);
16565 unsigned Med3Opc = Signed ? AMDGPUISD::SMED3 : AMDGPUISD::UMED3;
16566 if (VT == MVT::i32 || (VT == MVT::i16 && Subtarget->hasMed3_16()))
16567 return DAG.getNode(Opcode: Med3Opc, DL: SL, VT, N1: Src, N2: MaxVal, N3: MinVal);
16568
16569 // Note: we could also extend to i32 and use i32 med3 if i16 med3 is
16570 // not available, but this is unlikely to be profitable as constants
16571 // will often need to be materialized & extended, especially on
16572 // pre-GFX10 where VOP3 instructions couldn't take literal operands.
16573 return SDValue();
16574}
16575
16576static ConstantFPSDNode *getSplatConstantFP(SDValue Op) {
16577 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op))
16578 return C;
16579
16580 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: Op)) {
16581 if (ConstantFPSDNode *C = BV->getConstantFPSplatNode())
16582 return C;
16583 }
16584
16585 return nullptr;
16586}
16587
16588SDValue SITargetLowering::performFPMed3ImmCombine(SelectionDAG &DAG,
16589 const SDLoc &SL, SDValue Op0,
16590 SDValue Op1,
16591 bool IsKnownNoNaNs) const {
16592 ConstantFPSDNode *K1 = getSplatConstantFP(Op: Op1);
16593 if (!K1)
16594 return SDValue();
16595
16596 ConstantFPSDNode *K0 = getSplatConstantFP(Op: Op0.getOperand(i: 1));
16597 if (!K0)
16598 return SDValue();
16599
16600 // Ordered >= (although NaN inputs should have folded away by now).
16601 if (K0->getValueAPF() > K1->getValueAPF())
16602 return SDValue();
16603
16604 // med3 with a nan input acts like
16605 // v_min_f32(v_min_f32(S0.f32, S1.f32), S2.f32)
16606 //
16607 // So the result depends on whether the IEEE mode bit is enabled or not with a
16608 // signaling nan input.
16609 // ieee=1
16610 // s0 snan: yields s2
16611 // s1 snan: yields s2
16612 // s2 snan: qnan
16613
16614 // s0 qnan: min(s1, s2)
16615 // s1 qnan: min(s0, s2)
16616 // s2 qnan: min(s0, s1)
16617
16618 // ieee=0
16619 // s0 snan: min(s1, s2)
16620 // s1 snan: min(s0, s2)
16621 // s2 snan: qnan
16622
16623 // s0 qnan: min(s1, s2)
16624 // s1 qnan: min(s0, s2)
16625 // s2 qnan: min(s0, s1)
16626 const MachineFunction &MF = DAG.getMachineFunction();
16627 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
16628
16629 // TODO: Check IEEE bit enabled. We can form fmed3 with IEEE=0 regardless of
16630 // whether the input is a signaling nan if op0 is fmaximum or fmaximumnum. We
16631 // can only form if op0 is fmaxnum_ieee if IEEE=1.
16632 EVT VT = Op0.getValueType();
16633 if (Info->getMode().DX10Clamp) {
16634 // If dx10_clamp is enabled, NaNs clamp to 0.0. This is the same as the
16635 // hardware fmed3 behavior converting to a min.
16636 // FIXME: Should this be allowing -0.0?
16637 if (K1->isOne() && K0->isPosZero())
16638 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Op0.getOperand(i: 0));
16639 }
16640
16641 // med3 for f16 is only available on gfx9+, and not available for v2f16.
16642 if (VT == MVT::f32 || (VT == MVT::f16 && Subtarget->hasMed3_16())) {
16643 // This isn't safe with signaling NaNs because in IEEE mode, min/max on a
16644 // signaling NaN gives a quiet NaN. The quiet NaN input to the min would
16645 // then give the other result, which is different from med3 with a NaN
16646 // input.
16647 SDValue Var = Op0.getOperand(i: 0);
16648 if (!IsKnownNoNaNs && !DAG.isKnownNeverSNaN(Op: Var))
16649 return SDValue();
16650
16651 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
16652
16653 if ((!K0->hasOneUse() || TII->isInlineConstant(Imm: K0->getValueAPF())) &&
16654 (!K1->hasOneUse() || TII->isInlineConstant(Imm: K1->getValueAPF()))) {
16655 return DAG.getNode(Opcode: AMDGPUISD::FMED3, DL: SL, VT: K0->getValueType(ResNo: 0), N1: Var,
16656 N2: SDValue(K0, 0), N3: SDValue(K1, 0));
16657 }
16658 }
16659
16660 return SDValue();
16661}
16662
16663/// \return true if the subtarget supports minimum3 and maximum3 with the given
16664/// base min/max opcode \p Opc for type \p VT.
16665static bool supportsMin3Max3(const GCNSubtarget &Subtarget, unsigned Opc,
16666 EVT VT) {
16667 switch (Opc) {
16668 case ISD::FMINNUM:
16669 case ISD::FMAXNUM:
16670 case ISD::FMINNUM_IEEE:
16671 case ISD::FMAXNUM_IEEE:
16672 case ISD::FMINIMUMNUM:
16673 case ISD::FMAXIMUMNUM:
16674 case AMDGPUISD::FMIN_LEGACY:
16675 case AMDGPUISD::FMAX_LEGACY:
16676 return (VT == MVT::f32) || (VT == MVT::f16 && Subtarget.hasMin3Max3_16()) ||
16677 (VT == MVT::v2f16 && Subtarget.hasMin3Max3PKF16());
16678 case ISD::FMINIMUM:
16679 case ISD::FMAXIMUM:
16680 return (VT == MVT::f32 && Subtarget.hasMinimum3Maximum3F32()) ||
16681 (VT == MVT::f16 && Subtarget.hasMinimum3Maximum3F16()) ||
16682 (VT == MVT::v2f16 && Subtarget.hasMinimum3Maximum3PKF16());
16683 case ISD::SMAX:
16684 case ISD::SMIN:
16685 case ISD::UMAX:
16686 case ISD::UMIN:
16687 return (VT == MVT::i32) || (VT == MVT::i16 && Subtarget.hasMin3Max3_16());
16688 default:
16689 return false;
16690 }
16691
16692 llvm_unreachable("not a min/max opcode");
16693}
16694
16695SDValue SITargetLowering::performMinMaxCombine(SDNode *N,
16696 DAGCombinerInfo &DCI) const {
16697 SelectionDAG &DAG = DCI.DAG;
16698
16699 EVT VT = N->getValueType(ResNo: 0);
16700 unsigned Opc = N->getOpcode();
16701 SDValue Op0 = N->getOperand(Num: 0);
16702 SDValue Op1 = N->getOperand(Num: 1);
16703
16704 // Only do this if the inner op has one use since this will just increases
16705 // register pressure for no benefit.
16706
16707 if (supportsMin3Max3(Subtarget: *Subtarget, Opc, VT)) {
16708 auto IsTreeWithCombinableChildren = [Opc](SDValue Op) {
16709 return (Op.getOperand(i: 0).getOpcode() == Opc &&
16710 Op.getOperand(i: 0).hasOneUse()) ||
16711 (Op.getOperand(i: 1).getOpcode() == Opc &&
16712 Op.getOperand(i: 1).hasOneUse());
16713 };
16714
16715 bool CanTreeCombineApply = Op0.getOpcode() == Opc && Op0.hasOneUse() &&
16716 Op1.getOpcode() == Opc && Op1.hasOneUse();
16717 bool HasCombinableTreeChild =
16718 CanTreeCombineApply && (IsTreeWithCombinableChildren(Op0) ||
16719 IsTreeWithCombinableChildren(Op1));
16720
16721 // Tree reduction: when both operands are the same min/max op, restructure
16722 // to keep a 2-op node on top so higher tree levels can still combine.
16723 //
16724 // max(max(a, b), max(c, d)) -> max(max3(a, b, c), d)
16725 // min(min(a, b), min(c, d)) -> min(min3(a, b, c), d)
16726 //
16727 // Defer when either inner op is a tree node with combinable children.
16728 if (CanTreeCombineApply && !HasCombinableTreeChild) {
16729 SDLoc DL(N);
16730 SDValue Inner =
16731 DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT, N1: Op0.getOperand(i: 0),
16732 N2: Op0.getOperand(i: 1), N3: Op1.getOperand(i: 0));
16733 return DAG.getNode(Opcode: Opc, DL, VT, N1: Inner, N2: Op1.getOperand(i: 1));
16734 }
16735
16736 // max(max(a, b), c) -> max3(a, b, c)
16737 // min(min(a, b), c) -> min3(a, b, c)
16738 // Deferred when Op0 is a tree node with combinable children.
16739 if (Op0.getOpcode() == Opc && Op0.hasOneUse() && !HasCombinableTreeChild) {
16740 SDLoc DL(N);
16741 return DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT: N->getValueType(ResNo: 0),
16742 N1: Op0.getOperand(i: 0), N2: Op0.getOperand(i: 1), N3: Op1);
16743 }
16744
16745 // Try commuted.
16746 // max(a, max(b, c)) -> max3(a, b, c)
16747 // min(a, min(b, c)) -> min3(a, b, c)
16748 // Deferred when Op1 is a tree node with combinable children.
16749 if (Op1.getOpcode() == Opc && Op1.hasOneUse() && !HasCombinableTreeChild) {
16750 SDLoc DL(N);
16751 return DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT: N->getValueType(ResNo: 0),
16752 N1: Op0, N2: Op1.getOperand(i: 0), N3: Op1.getOperand(i: 1));
16753 }
16754 }
16755
16756 // umin(sffbh(x), bitwidth) -> sffbh(x) if x is known to be not 0 or -1.
16757 SDValue FfbhSrc;
16758 uint64_t Clamp = 0;
16759 if (Opc == ISD::UMIN &&
16760 sd_match(N: Op0,
16761 P: m_IntrinsicWOChain<Intrinsic::amdgcn_sffbh>(Opnds: m_Value(N&: FfbhSrc))) &&
16762 sd_match(N: Op1, P: m_ConstInt(V&: Clamp))) {
16763 unsigned BitWidth = FfbhSrc.getValueType().getScalarSizeInBits();
16764 if (Clamp >= BitWidth) {
16765 KnownBits Known = DAG.computeKnownBits(Op: FfbhSrc);
16766 if (Known.isNonZero() && Known.Zero.getBoolValue())
16767 return Op0;
16768 }
16769 }
16770
16771 // min(max(x, K0), K1), K0 < K1 -> med3(x, K0, K1)
16772 // max(min(x, K0), K1), K1 < K0 -> med3(x, K1, K0)
16773 if (Opc == ISD::SMIN && Op0.getOpcode() == ISD::SMAX && Op0.hasOneUse()) {
16774 if (SDValue Med3 = performIntMed3ImmCombine(
16775 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op1, MaxVal: Op0->getOperand(Num: 1), Signed: true))
16776 return Med3;
16777 }
16778 if (Opc == ISD::SMAX && Op0.getOpcode() == ISD::SMIN && Op0.hasOneUse()) {
16779 if (SDValue Med3 = performIntMed3ImmCombine(
16780 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op0->getOperand(Num: 1), MaxVal: Op1, Signed: true))
16781 return Med3;
16782 }
16783
16784 if (Opc == ISD::UMIN && Op0.getOpcode() == ISD::UMAX && Op0.hasOneUse()) {
16785 if (SDValue Med3 = performIntMed3ImmCombine(
16786 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op1, MaxVal: Op0->getOperand(Num: 1), Signed: false))
16787 return Med3;
16788 }
16789 if (Opc == ISD::UMAX && Op0.getOpcode() == ISD::UMIN && Op0.hasOneUse()) {
16790 if (SDValue Med3 = performIntMed3ImmCombine(
16791 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op0->getOperand(Num: 1), MaxVal: Op1, Signed: false))
16792 return Med3;
16793 }
16794
16795 // if !is_snan(x):
16796 // fminnum(fmaxnum(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16797 // fminnum_ieee(fmaxnum_ieee(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16798 // fminnumnum(fmaxnumnum(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16799 // fmin_legacy(fmax_legacy(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16800 if (((Opc == ISD::FMINNUM && Op0.getOpcode() == ISD::FMAXNUM) ||
16801 (Opc == ISD::FMINNUM_IEEE && Op0.getOpcode() == ISD::FMAXNUM_IEEE) ||
16802 (Opc == ISD::FMINIMUMNUM && Op0.getOpcode() == ISD::FMAXIMUMNUM) ||
16803 (Opc == AMDGPUISD::FMIN_LEGACY &&
16804 Op0.getOpcode() == AMDGPUISD::FMAX_LEGACY)) &&
16805 (VT == MVT::f32 || VT == MVT::f64 ||
16806 (VT == MVT::f16 && Subtarget->has16BitInsts()) ||
16807 (VT == MVT::bf16 && Subtarget->hasBF16PackedInsts()) ||
16808 (VT == MVT::v2bf16 && Subtarget->hasBF16PackedInsts()) ||
16809 (VT == MVT::v2f16 && Subtarget->hasVOP3PInsts())) &&
16810 Op0.hasOneUse()) {
16811 if (SDValue Res = performFPMed3ImmCombine(DAG, SL: SDLoc(N), Op0, Op1,
16812 IsKnownNoNaNs: N->getFlags().hasNoNaNs()))
16813 return Res;
16814 }
16815
16816 // Prefer fminnum_ieee over fminimum. For gfx950, minimum/maximum are legal
16817 // for some types, but at a higher cost since it's implemented with a 3
16818 // operand form.
16819 const SDNodeFlags Flags = N->getFlags();
16820 if ((Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM) && Flags.hasNoNaNs() &&
16821 !Subtarget->hasIEEEMinimumMaximumInsts() &&
16822 isOperationLegal(Op: ISD::FMINNUM_IEEE, VT: VT.getScalarType())) {
16823 unsigned NewOpc =
16824 Opc == ISD::FMINIMUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
16825 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(N), VT, N1: Op0, N2: Op1, Flags);
16826 }
16827
16828 return SDValue();
16829}
16830
16831static bool isClampZeroToOne(SDValue A, SDValue B) {
16832 if (ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A)) {
16833 if (ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B)) {
16834 // FIXME: Should this be allowing -0.0?
16835 return (CA->isPosZero() && CB->isOne()) ||
16836 (CA->isOne() && CB->isPosZero());
16837 }
16838 }
16839
16840 return false;
16841}
16842
16843// FIXME: Should only worry about snans for version with chain.
16844SDValue SITargetLowering::performFMed3Combine(SDNode *N,
16845 DAGCombinerInfo &DCI) const {
16846 EVT VT = N->getValueType(ResNo: 0);
16847 // v_med3_f32 and v_max_f32 behave identically wrt denorms, exceptions and
16848 // NaNs. With a NaN input, the order of the operands may change the result.
16849
16850 SelectionDAG &DAG = DCI.DAG;
16851 SDLoc SL(N);
16852
16853 SDValue Src0 = N->getOperand(Num: 0);
16854 SDValue Src1 = N->getOperand(Num: 1);
16855 SDValue Src2 = N->getOperand(Num: 2);
16856
16857 if (isClampZeroToOne(A: Src0, B: Src1)) {
16858 // const_a, const_b, x -> clamp is safe in all cases including signaling
16859 // nans.
16860 // FIXME: Should this be allowing -0.0?
16861 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Src2);
16862 }
16863
16864 const MachineFunction &MF = DAG.getMachineFunction();
16865 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
16866
16867 // FIXME: dx10_clamp behavior assumed in instcombine. Should we really bother
16868 // handling no dx10-clamp?
16869 if (Info->getMode().DX10Clamp) {
16870 // If NaNs is clamped to 0, we are free to reorder the inputs.
16871
16872 if (isa<ConstantFPSDNode>(Val: Src0) && !isa<ConstantFPSDNode>(Val: Src1))
16873 std::swap(a&: Src0, b&: Src1);
16874
16875 if (isa<ConstantFPSDNode>(Val: Src1) && !isa<ConstantFPSDNode>(Val: Src2))
16876 std::swap(a&: Src1, b&: Src2);
16877
16878 if (isa<ConstantFPSDNode>(Val: Src0) && !isa<ConstantFPSDNode>(Val: Src1))
16879 std::swap(a&: Src0, b&: Src1);
16880
16881 if (isClampZeroToOne(A: Src1, B: Src2))
16882 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Src0);
16883 }
16884
16885 return SDValue();
16886}
16887
16888SDValue SITargetLowering::performCvtPkRTZCombine(SDNode *N,
16889 DAGCombinerInfo &DCI) const {
16890 SDValue Src0 = N->getOperand(Num: 0);
16891 SDValue Src1 = N->getOperand(Num: 1);
16892 if (Src0.isUndef() && Src1.isUndef())
16893 return DCI.DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
16894 return SDValue();
16895}
16896
16897// Check if EXTRACT_VECTOR_ELT/INSERT_VECTOR_ELT (<n x e>, var-idx) should be
16898// expanded into a set of cmp/select instructions.
16899bool SITargetLowering::shouldExpandVectorDynExt(unsigned EltSize,
16900 unsigned NumElem,
16901 bool IsDivergentIdx,
16902 const GCNSubtarget *Subtarget) {
16903 if (UseDivergentRegisterIndexing)
16904 return false;
16905
16906 unsigned VecSize = EltSize * NumElem;
16907
16908 // Sub-dword vectors of size 2 dword or less have better implementation.
16909 if (VecSize <= 64 && EltSize < 32)
16910 return false;
16911
16912 // Always expand the rest of sub-dword instructions, otherwise it will be
16913 // lowered via memory.
16914 if (EltSize < 32)
16915 return true;
16916
16917 // Always do this if var-idx is divergent, otherwise it will become a loop.
16918 if (IsDivergentIdx)
16919 return true;
16920
16921 // Large vectors would yield too many compares and v_cndmask_b32 instructions.
16922 unsigned NumInsts = NumElem /* Number of compares */ +
16923 ((EltSize + 31) / 32) * NumElem /* Number of cndmasks */;
16924
16925 // On some architectures (GFX9) movrel is not available and it's better
16926 // to expand.
16927 if (Subtarget->useVGPRIndexMode())
16928 return NumInsts <= 16;
16929
16930 // If movrel is available, use it instead of expanding for vector of 8
16931 // elements.
16932 if (Subtarget->hasMovrel())
16933 return NumInsts <= 15;
16934
16935 return true;
16936}
16937
16938bool SITargetLowering::shouldExpandVectorDynExt(SDNode *N) const {
16939 SDValue Idx = N->getOperand(Num: N->getNumOperands() - 1);
16940 if (isa<ConstantSDNode>(Val: Idx))
16941 return false;
16942
16943 SDValue Vec = N->getOperand(Num: 0);
16944 EVT VecVT = Vec.getValueType();
16945 EVT EltVT = VecVT.getVectorElementType();
16946 unsigned EltSize = EltVT.getSizeInBits();
16947 unsigned NumElem = VecVT.getVectorNumElements();
16948
16949 return SITargetLowering::shouldExpandVectorDynExt(
16950 EltSize, NumElem, IsDivergentIdx: Idx->isDivergent(), Subtarget: getSubtarget());
16951}
16952
16953SDValue
16954SITargetLowering::performExtractVectorEltCombine(SDNode *N,
16955 DAGCombinerInfo &DCI) const {
16956 SDValue Vec = N->getOperand(Num: 0);
16957 SelectionDAG &DAG = DCI.DAG;
16958
16959 EVT VecVT = Vec.getValueType();
16960 EVT VecEltVT = VecVT.getVectorElementType();
16961 EVT ResVT = N->getValueType(ResNo: 0);
16962
16963 unsigned VecSize = VecVT.getSizeInBits();
16964 unsigned VecEltSize = VecEltVT.getSizeInBits();
16965
16966 if ((Vec.getOpcode() == ISD::FNEG || Vec.getOpcode() == ISD::FABS) &&
16967 allUsesHaveSourceMods(N)) {
16968 SDLoc SL(N);
16969 SDValue Idx = N->getOperand(Num: 1);
16970 SDValue Elt =
16971 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT, N1: Vec.getOperand(i: 0), N2: Idx);
16972 return DAG.getNode(Opcode: Vec.getOpcode(), DL: SL, VT: ResVT, Operand: Elt);
16973 }
16974
16975 // (extract_vector_element (and {y0, y1}, (build_vector 0x1f, 0x1f)), index)
16976 // -> (and (extract_vector_element {y0, y1}, index), 0x1f)
16977 // There are optimisations to transform 64-bit shifts into 32-bit shifts
16978 // depending on the shift operand. See e.g. performSraCombine().
16979 // This combine ensures that the optimisation is compatible with v2i32
16980 // legalised AND.
16981 if (VecVT == MVT::v2i32 && Vec->getOpcode() == ISD::AND &&
16982 Vec->getOperand(Num: 1)->getOpcode() == ISD::BUILD_VECTOR) {
16983
16984 const ConstantSDNode *C = isConstOrConstSplat(N: Vec.getOperand(i: 1));
16985 if (!C || C->getZExtValue() != 0x1f)
16986 return SDValue();
16987
16988 SDLoc SL(N);
16989 SDValue AndMask = DAG.getConstant(Val: 0x1f, DL: SL, VT: MVT::i32);
16990 SDValue EVE = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32,
16991 N1: Vec->getOperand(Num: 0), N2: N->getOperand(Num: 1));
16992 SDValue A = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: EVE, N2: AndMask);
16993 DAG.ReplaceAllUsesWith(From: N, To: A.getNode());
16994 }
16995
16996 // ScalarRes = EXTRACT_VECTOR_ELT ((vector-BINOP Vec1, Vec2), Idx)
16997 // =>
16998 // Vec1Elt = EXTRACT_VECTOR_ELT(Vec1, Idx)
16999 // Vec2Elt = EXTRACT_VECTOR_ELT(Vec2, Idx)
17000 // ScalarRes = scalar-BINOP Vec1Elt, Vec2Elt
17001 if (Vec.hasOneUse() && DCI.isBeforeLegalize() && VecEltVT == ResVT) {
17002 SDLoc SL(N);
17003 SDValue Idx = N->getOperand(Num: 1);
17004 unsigned Opc = Vec.getOpcode();
17005
17006 switch (Opc) {
17007 default:
17008 break;
17009 // TODO: Support other binary operations.
17010 case ISD::FADD:
17011 case ISD::FSUB:
17012 case ISD::FMUL:
17013 case ISD::ADD:
17014 case ISD::UMIN:
17015 case ISD::UMAX:
17016 case ISD::SMIN:
17017 case ISD::SMAX:
17018 case ISD::FMAXNUM:
17019 case ISD::FMINNUM:
17020 case ISD::FMAXNUM_IEEE:
17021 case ISD::FMINNUM_IEEE:
17022 case ISD::FMAXIMUM:
17023 case ISD::FMINIMUM: {
17024 SDValue Elt0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT,
17025 N1: Vec.getOperand(i: 0), N2: Idx);
17026 SDValue Elt1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT,
17027 N1: Vec.getOperand(i: 1), N2: Idx);
17028
17029 DCI.AddToWorklist(N: Elt0.getNode());
17030 DCI.AddToWorklist(N: Elt1.getNode());
17031 return DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT, N1: Elt0, N2: Elt1, Flags: Vec->getFlags());
17032 }
17033 }
17034 }
17035
17036 // EXTRACT_VECTOR_ELT (<n x e>, var-idx) => n x select (e, const-idx)
17037 if (shouldExpandVectorDynExt(N)) {
17038 SDLoc SL(N);
17039 SDValue Idx = N->getOperand(Num: 1);
17040 SDValue V;
17041 for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
17042 SDValue IC = DAG.getVectorIdxConstant(Val: I, DL: SL);
17043 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT, N1: Vec, N2: IC);
17044 if (I == 0)
17045 V = Elt;
17046 else
17047 V = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IC, True: Elt, False: V, Cond: ISD::SETEQ);
17048 }
17049 return V;
17050 }
17051
17052 // EXTRACT_VECTOR_ELT (v2i32 bitcast (i64/f64:k), Idx)
17053 // =>
17054 // i32:Lo(k) if Idx == 0, or
17055 // i32:Hi(k) if Idx == 1
17056 auto *Idx = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
17057 if (Vec.getOpcode() == ISD::BITCAST && VecVT == MVT::v2i32 && Idx) {
17058 SDLoc SL(N);
17059 SDValue PeekThrough = Vec.getOperand(i: 0);
17060 auto *KImm = dyn_cast<ConstantSDNode>(Val&: PeekThrough);
17061 if (KImm && KImm->getValueType(ResNo: 0).getSizeInBits() == 64) {
17062 uint64_t KImmValue = KImm->getZExtValue();
17063 return DAG.getConstant(
17064 Val: (KImmValue >> (32 * Idx->getZExtValue())) & 0xffffffff, DL: SL, VT: MVT::i32);
17065 }
17066 auto *KFPImm = dyn_cast<ConstantFPSDNode>(Val&: PeekThrough);
17067 if (KFPImm && KFPImm->getValueType(ResNo: 0).getSizeInBits() == 64) {
17068 uint64_t KFPImmValue =
17069 KFPImm->getValueAPF().bitcastToAPInt().getZExtValue();
17070 return DAG.getConstant(Val: (KFPImmValue >> (32 * Idx->getZExtValue())) &
17071 0xffffffff,
17072 DL: SL, VT: MVT::i32);
17073 }
17074 }
17075
17076 if (!DCI.isBeforeLegalize())
17077 return SDValue();
17078
17079 // Try to turn sub-dword accesses of vectors into accesses of the same 32-bit
17080 // elements. This exposes more load reduction opportunities by replacing
17081 // multiple small extract_vector_elements with a single 32-bit extract.
17082 if (isa<MemSDNode>(Val: Vec) && VecEltSize <= 16 && VecEltVT.isByteSized() &&
17083 VecSize > 32 && VecSize % 32 == 0 && Idx) {
17084 EVT NewVT = getEquivalentMemType(Context&: *DAG.getContext(), VT: VecVT);
17085
17086 unsigned BitIndex = Idx->getZExtValue() * VecEltSize;
17087 unsigned EltIdx = BitIndex / 32;
17088 unsigned LeftoverBitIdx = BitIndex % 32;
17089 SDLoc SL(N);
17090
17091 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: Vec);
17092 DCI.AddToWorklist(N: Cast.getNode());
17093
17094 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Cast,
17095 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
17096 DCI.AddToWorklist(N: Elt.getNode());
17097 SDValue Srl = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: Elt,
17098 N2: DAG.getConstant(Val: LeftoverBitIdx, DL: SL, VT: MVT::i32));
17099 DCI.AddToWorklist(N: Srl.getNode());
17100
17101 EVT VecEltAsIntVT = VecEltVT.changeTypeToInteger();
17102 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: VecEltAsIntVT, Operand: Srl);
17103 DCI.AddToWorklist(N: Trunc.getNode());
17104
17105 if (VecEltVT == ResVT) {
17106 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecEltVT, Operand: Trunc);
17107 }
17108
17109 assert(ResVT.isScalarInteger());
17110 return DAG.getAnyExtOrTrunc(Op: Trunc, DL: SL, VT: ResVT);
17111 }
17112
17113 return SDValue();
17114}
17115
17116SDValue
17117SITargetLowering::performInsertVectorEltCombine(SDNode *N,
17118 DAGCombinerInfo &DCI) const {
17119 SDValue Vec = N->getOperand(Num: 0);
17120 SDValue Idx = N->getOperand(Num: 2);
17121 EVT VecVT = Vec.getValueType();
17122 EVT EltVT = VecVT.getVectorElementType();
17123
17124 // INSERT_VECTOR_ELT (<n x e>, var-idx)
17125 // => BUILD_VECTOR n x select (e, const-idx)
17126 if (!shouldExpandVectorDynExt(N))
17127 return SDValue();
17128
17129 SelectionDAG &DAG = DCI.DAG;
17130 SDLoc SL(N);
17131 SDValue Ins = N->getOperand(Num: 1);
17132 EVT IdxVT = Idx.getValueType();
17133
17134 SmallVector<SDValue, 16> Ops;
17135 for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
17136 SDValue IC = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
17137 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec, N2: IC);
17138 SDValue V = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IC, True: Ins, False: Elt, Cond: ISD::SETEQ);
17139 Ops.push_back(Elt: V);
17140 }
17141
17142 return DAG.getBuildVector(VT: VecVT, DL: SL, Ops);
17143}
17144
17145/// Return the source of an fp_extend from f16 to f32, or a converted FP
17146/// constant.
17147static SDValue strictFPExtFromF16(SelectionDAG &DAG, SDValue Src) {
17148 if (Src.getOpcode() == ISD::FP_EXTEND &&
17149 Src.getOperand(i: 0).getValueType() == MVT::f16) {
17150 return Src.getOperand(i: 0);
17151 }
17152
17153 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Src)) {
17154 APFloat Val = CFP->getValueAPF();
17155 bool LosesInfo = true;
17156 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
17157 if (!LosesInfo)
17158 return DAG.getConstantFP(Val, DL: SDLoc(Src), VT: MVT::f16);
17159 }
17160
17161 return SDValue();
17162}
17163
17164SDValue SITargetLowering::performFPRoundCombine(SDNode *N,
17165 DAGCombinerInfo &DCI) const {
17166 assert(Subtarget->has16BitInsts() && !Subtarget->hasMed3_16() &&
17167 "combine only useful on gfx8");
17168
17169 SDValue TruncSrc = N->getOperand(Num: 0);
17170 EVT VT = N->getValueType(ResNo: 0);
17171 if (VT != MVT::f16)
17172 return SDValue();
17173
17174 if (TruncSrc.getOpcode() != AMDGPUISD::FMED3 ||
17175 TruncSrc.getValueType() != MVT::f32 || !TruncSrc.hasOneUse())
17176 return SDValue();
17177
17178 SelectionDAG &DAG = DCI.DAG;
17179 SDLoc SL(N);
17180
17181 // Optimize f16 fmed3 pattern performed on f32. On gfx8 there is no f16 fmed3,
17182 // and expanding it with min/max saves 1 instruction vs. casting to f32 and
17183 // casting back.
17184
17185 // fptrunc (f32 (fmed3 (fpext f16:a, fpext f16:b, fpext f16:c))) =>
17186 // fmin(fmax(a, b), fmax(fmin(a, b), c))
17187 SDValue A = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 0));
17188 if (!A)
17189 return SDValue();
17190
17191 SDValue B = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 1));
17192 if (!B)
17193 return SDValue();
17194
17195 SDValue C = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 2));
17196 if (!C)
17197 return SDValue();
17198
17199 // This changes signaling nan behavior. If an input is a signaling nan, it
17200 // would have been quieted by the fpext originally. We don't care because
17201 // these are unconstrained ops. If we needed to insert quieting canonicalizes
17202 // we would be worse off than just doing the promotion.
17203 SDValue A1 = DAG.getNode(Opcode: ISD::FMINNUM_IEEE, DL: SL, VT, N1: A, N2: B);
17204 SDValue B1 = DAG.getNode(Opcode: ISD::FMAXNUM_IEEE, DL: SL, VT, N1: A, N2: B);
17205 SDValue C1 = DAG.getNode(Opcode: ISD::FMAXNUM_IEEE, DL: SL, VT, N1: A1, N2: C);
17206 return DAG.getNode(Opcode: ISD::FMINNUM_IEEE, DL: SL, VT, N1: B1, N2: C1);
17207}
17208
17209unsigned SITargetLowering::getFusedOpcode(const SelectionDAG &DAG,
17210 const SDNode *N0,
17211 const SDNode *N1) const {
17212 EVT VT = N0->getValueType(ResNo: 0);
17213
17214 // Only do this if we are not trying to support denormals. v_mad_f32 does not
17215 // support denormals ever.
17216 if (((VT == MVT::f32 &&
17217 denormalModeIsFlushAllF32(MF: DAG.getMachineFunction())) ||
17218 (VT == MVT::f16 && Subtarget->hasMadF16() &&
17219 denormalModeIsFlushAllF64F16(MF: DAG.getMachineFunction()))) &&
17220 isOperationLegal(Op: ISD::FMAD, VT))
17221 return ISD::FMAD;
17222
17223 const TargetOptions &Options = DAG.getTarget().Options;
17224 if ((Options.AllowFPOpFusion == FPOpFusion::Fast ||
17225 (N0->getFlags().hasAllowContract() &&
17226 N1->getFlags().hasAllowContract())) &&
17227 isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
17228 return ISD::FMA;
17229 }
17230
17231 return 0;
17232}
17233
17234// For a reassociatable opcode perform:
17235// op x, (op y, z) -> op (op x, z), y, if x and z are uniform
17236SDValue SITargetLowering::reassociateScalarOps(SDNode *N,
17237 SelectionDAG &DAG) const {
17238 EVT VT = N->getValueType(ResNo: 0);
17239 if (VT != MVT::i32 && VT != MVT::i64)
17240 return SDValue();
17241
17242 if (DAG.isBaseWithConstantOffset(Op: SDValue(N, 0)))
17243 return SDValue();
17244
17245 unsigned Opc = N->getOpcode();
17246 SDValue Op0 = N->getOperand(Num: 0);
17247 SDValue Op1 = N->getOperand(Num: 1);
17248
17249 if (!(Op0->isDivergent() ^ Op1->isDivergent()))
17250 return SDValue();
17251
17252 if (Op0->isDivergent())
17253 std::swap(a&: Op0, b&: Op1);
17254
17255 if (Op1.getOpcode() != Opc || !Op1.hasOneUse())
17256 return SDValue();
17257
17258 SDValue Op2 = Op1.getOperand(i: 1);
17259 Op1 = Op1.getOperand(i: 0);
17260 if (!(Op1->isDivergent() ^ Op2->isDivergent()))
17261 return SDValue();
17262
17263 if (Op1->isDivergent())
17264 std::swap(a&: Op1, b&: Op2);
17265
17266 SDLoc SL(N);
17267 SDValue Add1 = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: Op0, N2: Op1);
17268 return DAG.getNode(Opcode: Opc, DL: SL, VT, N1: Add1, N2: Op2);
17269}
17270
17271static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL, EVT VT,
17272 SDValue N0, SDValue N1, SDValue N2, bool Signed) {
17273 unsigned MadOpc = Signed ? AMDGPUISD::MAD_I64_I32 : AMDGPUISD::MAD_U64_U32;
17274 SDVTList VTs = DAG.getVTList(VT1: MVT::i64, VT2: MVT::i1);
17275 SDValue Mad = DAG.getNode(Opcode: MadOpc, DL: SL, VTList: VTs, N1: N0, N2: N1, N3: N2);
17276 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Mad);
17277}
17278
17279// Fold
17280// y = lshr i64 x, 32
17281// res = add (mul i64 y, Const), x where "Const" is a 64-bit constant
17282// with Const.hi == -1
17283// To
17284// res = mad_u64_u32 y.lo ,Const.lo, x.lo
17285static SDValue tryFoldMADwithSRL(SelectionDAG &DAG, const SDLoc &SL,
17286 SDValue MulLHS, SDValue MulRHS,
17287 SDValue AddRHS) {
17288 if (MulRHS.getOpcode() == ISD::SRL)
17289 std::swap(a&: MulLHS, b&: MulRHS);
17290
17291 if (MulLHS.getValueType() != MVT::i64 || MulLHS.getOpcode() != ISD::SRL)
17292 return SDValue();
17293
17294 ConstantSDNode *ShiftVal = dyn_cast<ConstantSDNode>(Val: MulLHS.getOperand(i: 1));
17295 if (!ShiftVal || ShiftVal->getAsZExtVal() != 32 ||
17296 MulLHS.getOperand(i: 0) != AddRHS)
17297 return SDValue();
17298
17299 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val: MulRHS.getNode());
17300 if (!Const || Hi_32(Value: Const->getZExtValue()) != uint32_t(-1))
17301 return SDValue();
17302
17303 SDValue ConstMul =
17304 DAG.getConstant(Val: Lo_32(Value: Const->getZExtValue()), DL: SL, VT: MVT::i32);
17305 return getMad64_32(DAG, SL, VT: MVT::i64,
17306 N0: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulLHS), N1: ConstMul,
17307 N2: DAG.getZeroExtendInReg(Op: AddRHS, DL: SL, VT: MVT::i32), Signed: false);
17308}
17309
17310// Fold (add (mul x, y), z) --> (mad_[iu]64_[iu]32 x, y, z) plus high
17311// multiplies, if any.
17312//
17313// Full 64-bit multiplies that feed into an addition are lowered here instead
17314// of using the generic expansion. The generic expansion ends up with
17315// a tree of ADD nodes that prevents us from using the "add" part of the
17316// MAD instruction. The expansion produced here results in a chain of ADDs
17317// instead of a tree.
17318SDValue SITargetLowering::tryFoldToMad64_32(SDNode *N,
17319 DAGCombinerInfo &DCI) const {
17320 assert(N->isAnyAdd());
17321
17322 SelectionDAG &DAG = DCI.DAG;
17323 EVT VT = N->getValueType(ResNo: 0);
17324 SDLoc SL(N);
17325 SDValue LHS = N->getOperand(Num: 0);
17326 SDValue RHS = N->getOperand(Num: 1);
17327
17328 if (VT.isVector())
17329 return SDValue();
17330
17331 // S_MUL_HI_[IU]32 was added in gfx9, which allows us to keep the overall
17332 // result in scalar registers for uniform values.
17333 if (!N->isDivergent() && Subtarget->hasSMulHi())
17334 return SDValue();
17335
17336 unsigned NumBits = VT.getScalarSizeInBits();
17337 if (NumBits <= 32 || NumBits > 64)
17338 return SDValue();
17339
17340 if (LHS.getOpcode() != ISD::MUL) {
17341 assert(RHS.getOpcode() == ISD::MUL);
17342 std::swap(a&: LHS, b&: RHS);
17343 }
17344
17345 // Avoid the fold if it would unduly increase the number of multiplies due to
17346 // multiple uses, except on hardware with full-rate multiply-add (which is
17347 // part of full-rate 64-bit ops).
17348 if (!Subtarget->hasFullRate64Ops()) {
17349 unsigned NumUsers = 0;
17350 for (SDNode *User : LHS->users()) {
17351 // There is a use that does not feed into addition, so the multiply can't
17352 // be removed. We prefer MUL + ADD + ADDC over MAD + MUL.
17353 if (!User->isAnyAdd())
17354 return SDValue();
17355
17356 // We prefer 2xMAD over MUL + 2xADD + 2xADDC (code density), and prefer
17357 // MUL + 3xADD + 3xADDC over 3xMAD.
17358 ++NumUsers;
17359 if (NumUsers >= 3)
17360 return SDValue();
17361 }
17362 }
17363
17364 SDValue MulLHS = LHS.getOperand(i: 0);
17365 SDValue MulRHS = LHS.getOperand(i: 1);
17366 SDValue AddRHS = RHS;
17367
17368 if (SDValue FoldedMAD = tryFoldMADwithSRL(DAG, SL, MulLHS, MulRHS, AddRHS))
17369 return FoldedMAD;
17370
17371 // Always check whether operands are small unsigned values, since that
17372 // knowledge is useful in more cases. Check for small signed values only if
17373 // doing so can unlock a shorter code sequence.
17374 bool MulLHSUnsigned32 = numBitsUnsigned(Op: MulLHS, DAG) <= 32;
17375 bool MulRHSUnsigned32 = numBitsUnsigned(Op: MulRHS, DAG) <= 32;
17376
17377 bool MulSignedLo = false;
17378 if (!MulLHSUnsigned32 || !MulRHSUnsigned32) {
17379 MulSignedLo =
17380 numBitsSigned(Op: MulLHS, DAG) <= 32 && numBitsSigned(Op: MulRHS, DAG) <= 32;
17381 }
17382
17383 // The operands and final result all have the same number of bits. If
17384 // operands need to be extended, they can be extended with garbage. The
17385 // resulting garbage in the high bits of the mad_[iu]64_[iu]32 result is
17386 // truncated away in the end.
17387 if (VT != MVT::i64) {
17388 MulLHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: MulLHS);
17389 MulRHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: MulRHS);
17390 AddRHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: AddRHS);
17391 }
17392
17393 // The basic code generated is conceptually straightforward. Pseudo code:
17394 //
17395 // accum = mad_64_32 lhs.lo, rhs.lo, accum
17396 // accum.hi = add (mul lhs.hi, rhs.lo), accum.hi
17397 // accum.hi = add (mul lhs.lo, rhs.hi), accum.hi
17398 //
17399 // The second and third lines are optional, depending on whether the factors
17400 // are {sign,zero}-extended or not.
17401 //
17402 // The actual DAG is noisier than the pseudo code, but only due to
17403 // instructions that disassemble values into low and high parts, and
17404 // assemble the final result.
17405 SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
17406
17407 auto MulLHSLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulLHS);
17408 auto MulRHSLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulRHS);
17409 SDValue Accum =
17410 getMad64_32(DAG, SL, VT: MVT::i64, N0: MulLHSLo, N1: MulRHSLo, N2: AddRHS, Signed: MulSignedLo);
17411
17412 if (!MulSignedLo && (!MulLHSUnsigned32 || !MulRHSUnsigned32)) {
17413 auto [AccumLo, AccumHi] = DAG.SplitScalar(N: Accum, DL: SL, LoVT: MVT::i32, HiVT: MVT::i32);
17414
17415 if (!MulLHSUnsigned32) {
17416 auto MulLHSHi =
17417 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SL, VT: MVT::i32, N1: MulLHS, N2: One);
17418 SDValue MulHi = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: MVT::i32, N1: MulLHSHi, N2: MulRHSLo);
17419 AccumHi = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: MulHi, N2: AccumHi);
17420 }
17421
17422 if (!MulRHSUnsigned32) {
17423 auto MulRHSHi =
17424 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SL, VT: MVT::i32, N1: MulRHS, N2: One);
17425 SDValue MulHi = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: MVT::i32, N1: MulLHSLo, N2: MulRHSHi);
17426 AccumHi = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: MulHi, N2: AccumHi);
17427 }
17428
17429 Accum = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {AccumLo, AccumHi});
17430 Accum = DAG.getBitcast(VT: MVT::i64, V: Accum);
17431 }
17432
17433 if (VT != MVT::i64)
17434 Accum = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Accum);
17435 return Accum;
17436}
17437
17438SDValue
17439SITargetLowering::foldAddSub64WithZeroLowBitsTo32(SDNode *N,
17440 DAGCombinerInfo &DCI) const {
17441 SDValue RHS = N->getOperand(Num: 1);
17442 auto *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
17443 if (!CRHS)
17444 return SDValue();
17445
17446 // TODO: Worth using computeKnownBits? Maybe expensive since it's so
17447 // common.
17448 uint64_t Val = CRHS->getZExtValue();
17449 if (countr_zero(Val) >= 32) {
17450 SelectionDAG &DAG = DCI.DAG;
17451 SDLoc SL(N);
17452 SDValue LHS = N->getOperand(Num: 0);
17453
17454 // Avoid carry machinery if we know the low half of the add does not
17455 // contribute to the final result.
17456 //
17457 // add i64:x, K if computeTrailingZeros(K) >= 32
17458 // => build_pair (add x.hi, K.hi), x.lo
17459
17460 // Breaking the 64-bit add here with this strange constant is unlikely
17461 // to interfere with addressing mode patterns.
17462
17463 SDValue Hi = getHiHalf64(Op: LHS, DAG);
17464 SDValue ConstHi32 = DAG.getConstant(Val: Hi_32(Value: Val), DL: SL, VT: MVT::i32);
17465 unsigned Opcode = N->getOpcode();
17466 if (Opcode == ISD::PTRADD)
17467 Opcode = ISD::ADD;
17468 SDValue AddHi =
17469 DAG.getNode(Opcode, DL: SL, VT: MVT::i32, N1: Hi, N2: ConstHi32, Flags: N->getFlags());
17470
17471 SDValue Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: LHS);
17472 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i64, N1: Lo, N2: AddHi);
17473 }
17474
17475 return SDValue();
17476}
17477
17478// Collect the ultimate src of each of the mul node's operands, and confirm
17479// each operand is 8 bytes.
17480static std::optional<ByteProvider<SDValue>>
17481handleMulOperand(const SDValue &MulOperand) {
17482 auto Byte0 = calculateByteProvider(Op: MulOperand, Index: 0, Depth: 0);
17483 if (!Byte0 || Byte0->isConstantZero()) {
17484 return std::nullopt;
17485 }
17486 auto Byte1 = calculateByteProvider(Op: MulOperand, Index: 1, Depth: 0);
17487 if (Byte1 && !Byte1->isConstantZero()) {
17488 return std::nullopt;
17489 }
17490 return Byte0;
17491}
17492
17493static unsigned addPermMasks(unsigned First, unsigned Second) {
17494 unsigned FirstCs = First & 0x0c0c0c0c;
17495 unsigned SecondCs = Second & 0x0c0c0c0c;
17496 unsigned FirstNoCs = First & ~0x0c0c0c0c;
17497 unsigned SecondNoCs = Second & ~0x0c0c0c0c;
17498
17499 assert((FirstCs & 0xFF) | (SecondCs & 0xFF));
17500 assert((FirstCs & 0xFF00) | (SecondCs & 0xFF00));
17501 assert((FirstCs & 0xFF0000) | (SecondCs & 0xFF0000));
17502 assert((FirstCs & 0xFF000000) | (SecondCs & 0xFF000000));
17503
17504 return (FirstNoCs | SecondNoCs) | (FirstCs & SecondCs);
17505}
17506
17507struct DotSrc {
17508 SDValue SrcOp;
17509 int64_t PermMask;
17510 int64_t DWordOffset;
17511};
17512
17513static void placeSources(ByteProvider<SDValue> &Src0,
17514 ByteProvider<SDValue> &Src1,
17515 SmallVectorImpl<DotSrc> &Src0s,
17516 SmallVectorImpl<DotSrc> &Src1s, int Step) {
17517
17518 assert(Src0.Src.has_value() && Src1.Src.has_value());
17519 // Src0s and Src1s are empty, just place arbitrarily.
17520 if (Step == 0) {
17521 Src0s.push_back(Elt: {.SrcOp: *Src0.Src, .PermMask: ((Src0.SrcOffset % 4) << 24) + 0x0c0c0c,
17522 .DWordOffset: Src0.SrcOffset / 4});
17523 Src1s.push_back(Elt: {.SrcOp: *Src1.Src, .PermMask: ((Src1.SrcOffset % 4) << 24) + 0x0c0c0c,
17524 .DWordOffset: Src1.SrcOffset / 4});
17525 return;
17526 }
17527
17528 for (int BPI = 0; BPI < 2; BPI++) {
17529 std::pair<ByteProvider<SDValue>, ByteProvider<SDValue>> BPP = {Src0, Src1};
17530 if (BPI == 1) {
17531 BPP = {Src1, Src0};
17532 }
17533 unsigned ZeroMask = 0x0c0c0c0c;
17534 unsigned FMask = 0xFF << (8 * (3 - Step));
17535
17536 unsigned FirstMask =
17537 (BPP.first.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
17538 unsigned SecondMask =
17539 (BPP.second.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
17540 // Attempt to find Src vector which contains our SDValue, if so, add our
17541 // perm mask to the existing one. If we are unable to find a match for the
17542 // first SDValue, attempt to find match for the second.
17543 int FirstGroup = -1;
17544 for (int I = 0; I < 2; I++) {
17545 SmallVectorImpl<DotSrc> &Srcs = I == 0 ? Src0s : Src1s;
17546 auto MatchesFirst = [&BPP](DotSrc &IterElt) {
17547 return IterElt.SrcOp == *BPP.first.Src &&
17548 (IterElt.DWordOffset == (BPP.first.SrcOffset / 4));
17549 };
17550
17551 auto *Match = llvm::find_if(Range&: Srcs, P: MatchesFirst);
17552 if (Match != Srcs.end()) {
17553 Match->PermMask = addPermMasks(First: FirstMask, Second: Match->PermMask);
17554 FirstGroup = I;
17555 break;
17556 }
17557 }
17558 if (FirstGroup != -1) {
17559 SmallVectorImpl<DotSrc> &Srcs = FirstGroup == 1 ? Src0s : Src1s;
17560 auto MatchesSecond = [&BPP](DotSrc &IterElt) {
17561 return IterElt.SrcOp == *BPP.second.Src &&
17562 (IterElt.DWordOffset == (BPP.second.SrcOffset / 4));
17563 };
17564 auto *Match = llvm::find_if(Range&: Srcs, P: MatchesSecond);
17565 if (Match != Srcs.end()) {
17566 Match->PermMask = addPermMasks(First: SecondMask, Second: Match->PermMask);
17567 } else
17568 Srcs.push_back(Elt: {.SrcOp: *BPP.second.Src, .PermMask: SecondMask, .DWordOffset: BPP.second.SrcOffset / 4});
17569 return;
17570 }
17571 }
17572
17573 // If we have made it here, then we could not find a match in Src0s or Src1s
17574 // for either Src0 or Src1, so just place them arbitrarily.
17575
17576 unsigned ZeroMask = 0x0c0c0c0c;
17577 unsigned FMask = 0xFF << (8 * (3 - Step));
17578
17579 Src0s.push_back(
17580 Elt: {.SrcOp: *Src0.Src,
17581 .PermMask: ((Src0.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
17582 .DWordOffset: Src0.SrcOffset / 4});
17583 Src1s.push_back(
17584 Elt: {.SrcOp: *Src1.Src,
17585 .PermMask: ((Src1.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
17586 .DWordOffset: Src1.SrcOffset / 4});
17587}
17588
17589static SDValue resolveSources(SelectionDAG &DAG, SDLoc SL,
17590 SmallVectorImpl<DotSrc> &Srcs, bool IsSigned,
17591 bool IsAny) {
17592
17593 // If we just have one source, just permute it accordingly.
17594 if (Srcs.size() == 1) {
17595 auto *Elt = Srcs.begin();
17596 auto EltOp = getDWordFromOffset(DAG, SL, Src: Elt->SrcOp, DWordOffset: Elt->DWordOffset);
17597
17598 // v_perm will produce the original value
17599 if (Elt->PermMask == 0x3020100)
17600 return EltOp;
17601
17602 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: EltOp, N2: EltOp,
17603 N3: DAG.getConstant(Val: Elt->PermMask, DL: SL, VT: MVT::i32));
17604 }
17605
17606 auto *FirstElt = Srcs.begin();
17607 auto *SecondElt = std::next(x: FirstElt);
17608
17609 SmallVector<SDValue, 2> Perms;
17610
17611 // If we have multiple sources in the chain, combine them via perms (using
17612 // calculated perm mask) and Ors.
17613 while (true) {
17614 auto FirstMask = FirstElt->PermMask;
17615 auto SecondMask = SecondElt->PermMask;
17616
17617 unsigned FirstCs = FirstMask & 0x0c0c0c0c;
17618 unsigned FirstPlusFour = FirstMask | 0x04040404;
17619 // 0x0c + 0x04 = 0x10, so anding with 0x0F will produced 0x00 for any
17620 // original 0x0C.
17621 FirstMask = (FirstPlusFour & 0x0F0F0F0F) | FirstCs;
17622
17623 auto PermMask = addPermMasks(First: FirstMask, Second: SecondMask);
17624 auto FirstVal =
17625 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
17626 auto SecondVal =
17627 getDWordFromOffset(DAG, SL, Src: SecondElt->SrcOp, DWordOffset: SecondElt->DWordOffset);
17628
17629 Perms.push_back(Elt: DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: FirstVal,
17630 N2: SecondVal,
17631 N3: DAG.getConstant(Val: PermMask, DL: SL, VT: MVT::i32)));
17632
17633 FirstElt = std::next(x: SecondElt);
17634 if (FirstElt == Srcs.end())
17635 break;
17636
17637 SecondElt = std::next(x: FirstElt);
17638 // If we only have a FirstElt, then just combine that into the cumulative
17639 // source node.
17640 if (SecondElt == Srcs.end()) {
17641 auto EltOp =
17642 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
17643
17644 Perms.push_back(
17645 Elt: DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: EltOp, N2: EltOp,
17646 N3: DAG.getConstant(Val: FirstElt->PermMask, DL: SL, VT: MVT::i32)));
17647 break;
17648 }
17649 }
17650
17651 assert(Perms.size() == 1 || Perms.size() == 2);
17652 return Perms.size() == 2
17653 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Perms[0], N2: Perms[1])
17654 : Perms[0];
17655}
17656
17657static void fixMasks(SmallVectorImpl<DotSrc> &Srcs, unsigned ChainLength) {
17658 for (auto &[EntryVal, EntryMask, EntryOffset] : Srcs) {
17659 EntryMask = EntryMask >> ((4 - ChainLength) * 8);
17660 auto ZeroMask = ChainLength == 2 ? 0x0c0c0000 : 0x0c000000;
17661 EntryMask += ZeroMask;
17662 }
17663}
17664
17665static bool isMul(const SDValue Op) {
17666 auto Opcode = Op.getOpcode();
17667
17668 return (Opcode == ISD::MUL || Opcode == AMDGPUISD::MUL_U24 ||
17669 Opcode == AMDGPUISD::MUL_I24);
17670}
17671
17672static std::optional<bool>
17673checkDot4MulSignedness(const SDValue &N, ByteProvider<SDValue> &Src0,
17674 ByteProvider<SDValue> &Src1, const SDValue &S0Op,
17675 const SDValue &S1Op, const SelectionDAG &DAG) {
17676 // If we both ops are i8s (pre legalize-dag), then the signedness semantics
17677 // of the dot4 is irrelevant.
17678 if (S0Op.getValueSizeInBits() == 8 && S1Op.getValueSizeInBits() == 8)
17679 return false;
17680
17681 auto Known0 = DAG.computeKnownBits(Op: S0Op, Depth: 0);
17682 bool S0IsUnsigned = Known0.countMinLeadingZeros() > 0;
17683 bool S0IsSigned = Known0.countMinLeadingOnes() > 0;
17684 auto Known1 = DAG.computeKnownBits(Op: S1Op, Depth: 0);
17685 bool S1IsUnsigned = Known1.countMinLeadingZeros() > 0;
17686 bool S1IsSigned = Known1.countMinLeadingOnes() > 0;
17687
17688 assert(!(S0IsUnsigned && S0IsSigned));
17689 assert(!(S1IsUnsigned && S1IsSigned));
17690
17691 // There are 9 possible permutations of
17692 // {S0IsUnsigned, S0IsSigned, S1IsUnsigned, S1IsSigned}
17693
17694 // In two permutations, the sign bits are known to be the same for both Ops,
17695 // so simply return Signed / Unsigned corresponding to the MSB
17696
17697 if ((S0IsUnsigned && S1IsUnsigned) || (S0IsSigned && S1IsSigned))
17698 return S0IsSigned;
17699
17700 // In another two permutations, the sign bits are known to be opposite. In
17701 // this case return std::nullopt to indicate a bad match.
17702
17703 if ((S0IsUnsigned && S1IsSigned) || (S0IsSigned && S1IsUnsigned))
17704 return std::nullopt;
17705
17706 // In the remaining five permutations, we don't know the value of the sign
17707 // bit for at least one Op. Since we have a valid ByteProvider, we know that
17708 // the upper bits must be extension bits. Thus, the only ways for the sign
17709 // bit to be unknown is if it was sign extended from unknown value, or if it
17710 // was any extended. In either case, it is correct to use the signed
17711 // version of the signedness semantics of dot4
17712
17713 // In two of such permutations, we known the sign bit is set for
17714 // one op, and the other is unknown. It is okay to used signed version of
17715 // dot4.
17716 if ((S0IsSigned && !(S1IsSigned || S1IsUnsigned)) ||
17717 ((S1IsSigned && !(S0IsSigned || S0IsUnsigned))))
17718 return true;
17719
17720 // In one such permutation, we don't know either of the sign bits. It is okay
17721 // to used the signed version of dot4.
17722 if ((!(S1IsSigned || S1IsUnsigned) && !(S0IsSigned || S0IsUnsigned)))
17723 return true;
17724
17725 // In two of such permutations, we known the sign bit is unset for
17726 // one op, and the other is unknown. Return std::nullopt to indicate a
17727 // bad match.
17728 if ((S0IsUnsigned && !(S1IsSigned || S1IsUnsigned)) ||
17729 ((S1IsUnsigned && !(S0IsSigned || S0IsUnsigned))))
17730 return std::nullopt;
17731
17732 llvm_unreachable("Fully covered condition");
17733}
17734
17735SDValue SITargetLowering::performAddCombine(SDNode *N,
17736 DAGCombinerInfo &DCI) const {
17737 SelectionDAG &DAG = DCI.DAG;
17738 EVT VT = N->getValueType(ResNo: 0);
17739 SDLoc SL(N);
17740 SDValue LHS = N->getOperand(Num: 0);
17741 SDValue RHS = N->getOperand(Num: 1);
17742
17743 if (LHS.getOpcode() == ISD::MUL || RHS.getOpcode() == ISD::MUL) {
17744 if (Subtarget->hasMad64_32()) {
17745 if (SDValue Folded = tryFoldToMad64_32(N, DCI))
17746 return Folded;
17747 }
17748 }
17749
17750 if (SDValue V = reassociateScalarOps(N, DAG)) {
17751 return V;
17752 }
17753
17754 if (VT == MVT::i64) {
17755 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
17756 return Folded;
17757 }
17758
17759 // dot4 produces a 32-bit result, so a wider VT can't be folded.
17760 if (!VT.isVector() && VT.getSizeInBits() <= 32 &&
17761 (isMul(Op: LHS) || isMul(Op: RHS)) && Subtarget->hasDot7Insts() &&
17762 (Subtarget->hasDot1Insts() || Subtarget->hasDot8Insts())) {
17763 SDValue TempNode(N, 0);
17764 std::optional<bool> IsSigned;
17765 SmallVector<DotSrc, 4> Src0s;
17766 SmallVector<DotSrc, 4> Src1s;
17767 SmallVector<SDValue, 4> Src2s;
17768
17769 // Match the v_dot4 tree, while collecting src nodes.
17770 int ChainLength = 0;
17771 for (int I = 0; I < 4; I++) {
17772 auto MulIdx = isMul(Op: LHS) ? 0 : isMul(Op: RHS) ? 1 : -1;
17773 if (MulIdx == -1)
17774 break;
17775 auto Src0 = handleMulOperand(MulOperand: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 0));
17776 if (!Src0)
17777 break;
17778 auto Src1 = handleMulOperand(MulOperand: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 1));
17779 if (!Src1)
17780 break;
17781
17782 auto IterIsSigned = checkDot4MulSignedness(
17783 N: TempNode->getOperand(Num: MulIdx), Src0&: *Src0, Src1&: *Src1,
17784 S0Op: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 0),
17785 S1Op: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 1), DAG);
17786 if (!IterIsSigned)
17787 break;
17788 if (!IsSigned)
17789 IsSigned = *IterIsSigned;
17790 if (*IterIsSigned != *IsSigned)
17791 break;
17792 placeSources(Src0&: *Src0, Src1&: *Src1, Src0s, Src1s, Step: I);
17793 auto AddIdx = 1 - MulIdx;
17794 // Allow the special case where add (add (mul24, 0), mul24) became ->
17795 // add (mul24, mul24).
17796 if (I == 2 && isMul(Op: TempNode->getOperand(Num: AddIdx))) {
17797 Src2s.push_back(Elt: TempNode->getOperand(Num: AddIdx));
17798 auto Src0 =
17799 handleMulOperand(MulOperand: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 0));
17800 if (!Src0)
17801 break;
17802 auto Src1 =
17803 handleMulOperand(MulOperand: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 1));
17804 if (!Src1)
17805 break;
17806 auto IterIsSigned = checkDot4MulSignedness(
17807 N: TempNode->getOperand(Num: AddIdx), Src0&: *Src0, Src1&: *Src1,
17808 S0Op: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 0),
17809 S1Op: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 1), DAG);
17810 if (!IterIsSigned)
17811 break;
17812 assert(IsSigned);
17813 if (*IterIsSigned != *IsSigned)
17814 break;
17815 placeSources(Src0&: *Src0, Src1&: *Src1, Src0s, Src1s, Step: I + 1);
17816 Src2s.push_back(Elt: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
17817 ChainLength = I + 2;
17818 break;
17819 }
17820
17821 TempNode = TempNode->getOperand(Num: AddIdx);
17822 Src2s.push_back(Elt: TempNode);
17823 ChainLength = I + 1;
17824 // The loop body treats TempNode's operands as addends.
17825 if (TempNode.getOpcode() != ISD::ADD)
17826 break;
17827 LHS = TempNode->getOperand(Num: 0);
17828 RHS = TempNode->getOperand(Num: 1);
17829 }
17830
17831 if (ChainLength < 2)
17832 return SDValue();
17833
17834 // Masks were constructed with assumption that we would find a chain of
17835 // length 4. If not, then we need to 0 out the MSB bits (via perm mask of
17836 // 0x0c) so they do not affect dot calculation.
17837 if (ChainLength < 4) {
17838 fixMasks(Srcs&: Src0s, ChainLength);
17839 fixMasks(Srcs&: Src1s, ChainLength);
17840 }
17841
17842 SDValue Src0, Src1;
17843
17844 // If we are just using a single source for both, and have permuted the
17845 // bytes consistently, we can just use the sources without permuting
17846 // (commutation).
17847 bool UseOriginalSrc = false;
17848 if (ChainLength == 4 && Src0s.size() == 1 && Src1s.size() == 1 &&
17849 Src0s.begin()->PermMask == Src1s.begin()->PermMask &&
17850 Src0s.begin()->SrcOp.getValueSizeInBits() >= 32 &&
17851 Src1s.begin()->SrcOp.getValueSizeInBits() >= 32) {
17852 SmallVector<unsigned, 4> SrcBytes;
17853 auto Src0Mask = Src0s.begin()->PermMask;
17854 SrcBytes.push_back(Elt: Src0Mask & 0xFF000000);
17855 bool UniqueEntries = true;
17856 for (auto I = 1; I < 4; I++) {
17857 auto NextByte = Src0Mask & (0xFF << ((3 - I) * 8));
17858
17859 if (is_contained(Range&: SrcBytes, Element: NextByte)) {
17860 UniqueEntries = false;
17861 break;
17862 }
17863 SrcBytes.push_back(Elt: NextByte);
17864 }
17865
17866 if (UniqueEntries) {
17867 UseOriginalSrc = true;
17868
17869 auto *FirstElt = Src0s.begin();
17870 auto FirstEltOp =
17871 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
17872
17873 auto *SecondElt = Src1s.begin();
17874 auto SecondEltOp = getDWordFromOffset(DAG, SL, Src: SecondElt->SrcOp,
17875 DWordOffset: SecondElt->DWordOffset);
17876
17877 Src0 = DAG.getBitcastedAnyExtOrTrunc(Op: FirstEltOp, DL: SL,
17878 VT: MVT::getIntegerVT(BitWidth: 32));
17879 Src1 = DAG.getBitcastedAnyExtOrTrunc(Op: SecondEltOp, DL: SL,
17880 VT: MVT::getIntegerVT(BitWidth: 32));
17881 }
17882 }
17883
17884 if (!UseOriginalSrc) {
17885 Src0 = resolveSources(DAG, SL, Srcs&: Src0s, IsSigned: false, IsAny: true);
17886 Src1 = resolveSources(DAG, SL, Srcs&: Src1s, IsSigned: false, IsAny: true);
17887 }
17888
17889 assert(IsSigned);
17890 SDValue Src2 =
17891 DAG.getExtOrTrunc(IsSigned: *IsSigned, Op: Src2s[ChainLength - 1], DL: SL, VT: MVT::i32);
17892
17893 SDValue IID = DAG.getTargetConstant(Val: *IsSigned ? Intrinsic::amdgcn_sdot4
17894 : Intrinsic::amdgcn_udot4,
17895 DL: SL, VT: MVT::i64);
17896
17897 assert(!VT.isVector());
17898 auto Dot = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32, N1: IID, N2: Src0,
17899 N3: Src1, N4: Src2, N5: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i1));
17900
17901 return DAG.getExtOrTrunc(IsSigned: *IsSigned, Op: Dot, DL: SL, VT);
17902 }
17903
17904 if (VT != MVT::i32 || !DCI.isAfterLegalizeDAG())
17905 return SDValue();
17906
17907 // add x, zext (setcc) => uaddo_carry x, 0, setcc
17908 // add x, sext (setcc) => usubo_carry x, 0, setcc
17909 unsigned Opc = LHS.getOpcode();
17910 if (Opc == ISD::ZERO_EXTEND || Opc == ISD::SIGN_EXTEND ||
17911 Opc == ISD::ANY_EXTEND || Opc == ISD::UADDO_CARRY)
17912 std::swap(a&: RHS, b&: LHS);
17913
17914 Opc = RHS.getOpcode();
17915 switch (Opc) {
17916 default:
17917 break;
17918 case ISD::ZERO_EXTEND:
17919 case ISD::SIGN_EXTEND:
17920 case ISD::ANY_EXTEND: {
17921 auto Cond = RHS.getOperand(i: 0);
17922 // If this won't be a real VOPC output, we would still need to insert an
17923 // extra instruction anyway.
17924 if (!isBoolSGPR(V: Cond))
17925 break;
17926 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1);
17927 SDValue Args[] = {LHS, DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond};
17928 Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::USUBO_CARRY : ISD::UADDO_CARRY;
17929 return DAG.getNode(Opcode: Opc, DL: SL, VTList, Ops: Args);
17930 }
17931 case ISD::UADDO_CARRY: {
17932 // add x, (uaddo_carry y, 0, cc) => uaddo_carry x, y, cc
17933 if (!isNullConstant(V: RHS.getOperand(i: 1)))
17934 break;
17935 SDValue Args[] = {LHS, RHS.getOperand(i: 0), RHS.getOperand(i: 2)};
17936 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: SDLoc(N), VTList: RHS->getVTList(), Ops: Args);
17937 }
17938 }
17939 return SDValue();
17940}
17941
17942SDValue SITargetLowering::performPtrAddCombine(SDNode *N,
17943 DAGCombinerInfo &DCI) const {
17944 SelectionDAG &DAG = DCI.DAG;
17945 SDLoc DL(N);
17946 EVT VT = N->getValueType(ResNo: 0);
17947 SDValue N0 = N->getOperand(Num: 0);
17948 SDValue N1 = N->getOperand(Num: 1);
17949
17950 // The following folds transform PTRADDs into regular arithmetic in cases
17951 // where the PTRADD wouldn't be folded as an immediate offset into memory
17952 // instructions anyway. They are target-specific in that other targets might
17953 // prefer to not lose information about the pointer arithmetic.
17954
17955 // Fold (ptradd x, shl(0 - v, k)) -> sub(x, shl(v, k)).
17956 // Adapted from DAGCombiner::visitADDLikeCommutative.
17957 SDValue V, K;
17958 if (sd_match(N: N1, P: m_Shl(L: m_Neg(V: m_Value(N&: V)), R: m_Value(N&: K)))) {
17959 SDNodeFlags ShlFlags = N1->getFlags();
17960 // If the original shl is NUW and NSW, the first k+1 bits of 0-v are all 0,
17961 // so v is either 0 or the first k+1 bits of v are all 1 -> NSW can be
17962 // preserved.
17963 SDNodeFlags NewShlFlags =
17964 ShlFlags.hasNoUnsignedWrap() && ShlFlags.hasNoSignedWrap()
17965 ? SDNodeFlags::NoSignedWrap
17966 : SDNodeFlags();
17967 SDValue Inner = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: V, N2: K, Flags: NewShlFlags);
17968 DCI.AddToWorklist(N: Inner.getNode());
17969 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: Inner);
17970 }
17971
17972 // Fold into Mad64 if the right-hand side is a MUL. Analogous to a fold in
17973 // performAddCombine.
17974 if (N1.getOpcode() == ISD::MUL) {
17975 if (Subtarget->hasMad64_32()) {
17976 if (SDValue Folded = tryFoldToMad64_32(N, DCI))
17977 return Folded;
17978 }
17979 }
17980
17981 // If the 32 low bits of the constant are all zero, there is nothing to fold
17982 // into an immediate offset, so it's better to eliminate the unnecessary
17983 // addition for the lower 32 bits than to preserve the PTRADD.
17984 // Analogous to a fold in performAddCombine.
17985 if (VT == MVT::i64) {
17986 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
17987 return Folded;
17988 }
17989
17990 if (N1.getOpcode() != ISD::ADD || !N1.hasOneUse())
17991 return SDValue();
17992
17993 SDValue X = N0;
17994 SDValue Y = N1.getOperand(i: 0);
17995 SDValue Z = N1.getOperand(i: 1);
17996 bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(N: Y);
17997 bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(N: Z);
17998
17999 if (!YIsConstant && !ZIsConstant && !X->isDivergent() &&
18000 Y->isDivergent() != Z->isDivergent()) {
18001 // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if x and
18002 // y are uniform and z isn't.
18003 // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if x and
18004 // z are uniform and y isn't.
18005 // The goal is to push uniform operands up in the computation, so that they
18006 // can be handled with scalar operations. We can't use reassociateScalarOps
18007 // for this since it requires two identical commutative operations to
18008 // reassociate.
18009 if (Y->isDivergent())
18010 std::swap(a&: Y, b&: Z);
18011 // If both additions in the original were NUW, reassociation preserves that.
18012 SDNodeFlags ReassocFlags =
18013 (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap;
18014 SDValue UniformInner = DAG.getMemBasePlusOffset(Base: X, Offset: Y, DL, Flags: ReassocFlags);
18015 DCI.AddToWorklist(N: UniformInner.getNode());
18016 return DAG.getMemBasePlusOffset(Base: UniformInner, Offset: Z, DL, Flags: ReassocFlags);
18017 }
18018
18019 return SDValue();
18020}
18021
18022static bool isCtlzOpc(unsigned Opc) {
18023 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_POISON;
18024}
18025
18026SDValue SITargetLowering::performSubCombine(SDNode *N,
18027 DAGCombinerInfo &DCI) const {
18028 SelectionDAG &DAG = DCI.DAG;
18029 EVT VT = N->getValueType(ResNo: 0);
18030
18031 if (VT == MVT::i64) {
18032 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
18033 return Folded;
18034 }
18035
18036 if (VT != MVT::i32)
18037 return SDValue();
18038
18039 SDLoc SL(N);
18040 SDValue LHS = N->getOperand(Num: 0);
18041 SDValue RHS = N->getOperand(Num: 1);
18042
18043 // sub x, zext (setcc) => usubo_carry x, 0, setcc
18044 // sub x, sext (setcc) => uaddo_carry x, 0, setcc
18045 unsigned Opc = RHS.getOpcode();
18046 switch (Opc) {
18047 default:
18048 break;
18049 case ISD::ZERO_EXTEND:
18050 case ISD::SIGN_EXTEND:
18051 case ISD::ANY_EXTEND: {
18052 auto Cond = RHS.getOperand(i: 0);
18053 // If this won't be a real VOPC output, we would still need to insert an
18054 // extra instruction anyway.
18055 if (!isBoolSGPR(V: Cond))
18056 break;
18057 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1);
18058 SDValue Args[] = {LHS, DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond};
18059 Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
18060 return DAG.getNode(Opcode: Opc, DL: SL, VTList, Ops: Args);
18061 }
18062 }
18063
18064 if (LHS.getOpcode() == ISD::USUBO_CARRY) {
18065 // sub (usubo_carry x, 0, cc), y => usubo_carry x, y, cc
18066 if (!isNullConstant(V: LHS.getOperand(i: 1)))
18067 return SDValue();
18068 SDValue Args[] = {LHS.getOperand(i: 0), RHS, LHS.getOperand(i: 2)};
18069 return DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: SDLoc(N), VTList: LHS->getVTList(), Ops: Args);
18070 }
18071
18072 // sub (ctlz (xor x, (sra x, 31))), 1 -> ctls x.
18073 if (isOneConstant(V: RHS) && isCtlzOpc(Opc: LHS.getOpcode())) {
18074 SDValue CtlzSrc = LHS.getOperand(i: 0);
18075 // Check for xor x, (sra x, 31) pattern.
18076 if (CtlzSrc.getOpcode() == ISD::XOR) {
18077 SDValue X = CtlzSrc.getOperand(i: 0);
18078 SDValue SignExt = CtlzSrc.getOperand(i: 1);
18079 // Try both ordering of XOR operands.
18080 if (SignExt.getOpcode() != ISD::SRA)
18081 std::swap(a&: X, b&: SignExt);
18082 if (SignExt.getOpcode() == ISD::SRA && SignExt.getOperand(i: 0) == X) {
18083 ConstantSDNode *ShiftAmt =
18084 dyn_cast<ConstantSDNode>(Val: SignExt.getOperand(i: 1));
18085 unsigned BitWidth = X.getValueType().getScalarSizeInBits();
18086 if (ShiftAmt && ShiftAmt->getZExtValue() == BitWidth - 1)
18087 return DAG.getNode(Opcode: ISD::CTLS, DL: SL, VT, Operand: X);
18088 }
18089 }
18090 }
18091
18092 return SDValue();
18093}
18094
18095SDValue SITargetLowering::performFAddCombine(SDNode *N,
18096 DAGCombinerInfo &DCI) const {
18097 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
18098 return SDValue();
18099
18100 SelectionDAG &DAG = DCI.DAG;
18101 EVT VT = N->getValueType(ResNo: 0);
18102
18103 SDLoc SL(N);
18104 SDValue LHS = N->getOperand(Num: 0);
18105 SDValue RHS = N->getOperand(Num: 1);
18106
18107 // These should really be instruction patterns, but writing patterns with
18108 // source modifiers is a pain.
18109
18110 // fadd (fadd (a, a), b) -> mad 2.0, a, b
18111 if (LHS.getOpcode() == ISD::FADD) {
18112 SDValue A = LHS.getOperand(i: 0);
18113 if (A == LHS.getOperand(i: 1)) {
18114 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: LHS.getNode());
18115 if (FusedOp != 0) {
18116 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
18117 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: RHS);
18118 }
18119 }
18120 }
18121
18122 // fadd (b, fadd (a, a)) -> mad 2.0, a, b
18123 if (RHS.getOpcode() == ISD::FADD) {
18124 SDValue A = RHS.getOperand(i: 0);
18125 if (A == RHS.getOperand(i: 1)) {
18126 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: RHS.getNode());
18127 if (FusedOp != 0) {
18128 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
18129 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: LHS);
18130 }
18131 }
18132 }
18133
18134 return SDValue();
18135}
18136
18137SDValue SITargetLowering::performFSubCombine(SDNode *N,
18138 DAGCombinerInfo &DCI) const {
18139 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
18140 return SDValue();
18141
18142 SelectionDAG &DAG = DCI.DAG;
18143 SDLoc SL(N);
18144 EVT VT = N->getValueType(ResNo: 0);
18145 assert(!VT.isVector());
18146
18147 // Try to get the fneg to fold into the source modifier. This undoes generic
18148 // DAG combines and folds them into the mad.
18149 //
18150 // Only do this if we are not trying to support denormals. v_mad_f32 does
18151 // not support denormals ever.
18152 SDValue LHS = N->getOperand(Num: 0);
18153 SDValue RHS = N->getOperand(Num: 1);
18154 if (LHS.getOpcode() == ISD::FADD) {
18155 // (fsub (fadd a, a), c) -> mad 2.0, a, (fneg c)
18156 SDValue A = LHS.getOperand(i: 0);
18157 if (A == LHS.getOperand(i: 1)) {
18158 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: LHS.getNode());
18159 if (FusedOp != 0) {
18160 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
18161 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
18162
18163 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: NegRHS);
18164 }
18165 }
18166 }
18167
18168 if (RHS.getOpcode() == ISD::FADD) {
18169 // (fsub c, (fadd a, a)) -> mad -2.0, a, c
18170
18171 SDValue A = RHS.getOperand(i: 0);
18172 if (A == RHS.getOperand(i: 1)) {
18173 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: RHS.getNode());
18174 if (FusedOp != 0) {
18175 const SDValue NegTwo = DAG.getConstantFP(Val: -2.0, DL: SL, VT);
18176 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: NegTwo, N3: LHS);
18177 }
18178 }
18179 }
18180
18181 return SDValue();
18182}
18183
18184SDValue SITargetLowering::performFDivCombine(SDNode *N,
18185 DAGCombinerInfo &DCI) const {
18186 SelectionDAG &DAG = DCI.DAG;
18187 SDLoc SL(N);
18188 EVT VT = N->getValueType(ResNo: 0);
18189
18190 if (VT != MVT::f16 && VT != MVT::bf16)
18191 return SDValue();
18192
18193 SDValue LHS = N->getOperand(Num: 0);
18194 SDValue RHS = N->getOperand(Num: 1);
18195
18196 SDNodeFlags Flags = N->getFlags();
18197 SDNodeFlags RHSFlags = RHS->getFlags();
18198 if (!Flags.hasAllowContract() || !RHSFlags.hasAllowContract() ||
18199 !RHS->hasOneUse())
18200 return SDValue();
18201
18202 if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
18203 bool IsNegative = false;
18204 if (CLHS->isOne() || (IsNegative = CLHS->isMinusOne())) {
18205 // fdiv contract 1.0, (sqrt contract x) -> rsq
18206 // fdiv contract -1.0, (sqrt contract x) -> fneg(rsq)
18207 if (RHS.getOpcode() == ISD::FSQRT) {
18208 // TODO: Or in RHS flags, somehow missing from SDNodeFlags
18209 SDValue SqrtOp = RHS.getOperand(i: 0);
18210 SDValue Rsq;
18211 if (isOperationLegal(Op: ISD::FSQRT, VT)) {
18212 // fsqrt legality correlates to rsq availability of the same type.
18213 Rsq = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SL, VT, Operand: SqrtOp, Flags);
18214 } else if (VT == MVT::f16) {
18215 // Targets without 16-bit instructions (gfx6/gfx7) have no f16 rsq,
18216 // but v_rsq_f32 is more than accurate enough for f16. Unlike bf16,
18217 // every f16 value (including denormals) extends to a normal f32, and
18218 // an f16 rsq result is never denormal, so the f32 reciprocal square
18219 // root needs no denormal handling. Compute it in f32 and round back.
18220 SDValue Ext =
18221 DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: SqrtOp, Flags);
18222 SDValue F32Rsq =
18223 DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
18224 Rsq = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: F32Rsq,
18225 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
18226 } else {
18227 // bf16 shares f32's exponent range, so bf16 denormals would extend to
18228 // f32 denormals that v_rsq_f32 does not handle. Leave it expanded.
18229 return SDValue();
18230 }
18231 return IsNegative ? DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Rsq, Flags) : Rsq;
18232 }
18233 }
18234 }
18235
18236 return SDValue();
18237}
18238
18239SDValue SITargetLowering::performFMulCombine(SDNode *N,
18240 DAGCombinerInfo &DCI) const {
18241 SelectionDAG &DAG = DCI.DAG;
18242 EVT VT = N->getValueType(ResNo: 0);
18243 EVT ScalarVT = VT.getScalarType();
18244 EVT IntVT = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
18245
18246 if (!N->isDivergent() && getSubtarget()->hasSALUFloatInsts() &&
18247 (ScalarVT == MVT::f32 || ScalarVT == MVT::f16)) {
18248 // Prefer to use s_mul_f16/f32 instead of v_ldexp_f16/f32.
18249 return SDValue();
18250 }
18251
18252 SDValue LHS = N->getOperand(Num: 0);
18253 SDValue RHS = N->getOperand(Num: 1);
18254
18255 // It is cheaper to realize i32 inline constants as compared against
18256 // materializing f16 or f64 (or even non-inline f32) values,
18257 // possible via ldexp usage, as shown below :
18258 //
18259 // Given : A = 2^a & B = 2^b ; where a and b are integers.
18260 // fmul x, (select y, A, B) -> ldexp( x, (select i32 y, a, b) )
18261 // fmul x, (select y, -A, -B) -> ldexp( (fneg x), (select i32 y, a, b) )
18262 if ((ScalarVT == MVT::f64 || ScalarVT == MVT::f32 || ScalarVT == MVT::f16) &&
18263 (RHS.hasOneUse() && RHS.getOpcode() == ISD::SELECT)) {
18264 const ConstantFPSDNode *TrueNode = isConstOrConstSplatFP(N: RHS.getOperand(i: 1));
18265 if (!TrueNode)
18266 return SDValue();
18267 const ConstantFPSDNode *FalseNode =
18268 isConstOrConstSplatFP(N: RHS.getOperand(i: 2));
18269 if (!FalseNode)
18270 return SDValue();
18271
18272 if (TrueNode->isNegative() != FalseNode->isNegative())
18273 return SDValue();
18274
18275 // For f32, only non-inline constants should be transformed.
18276 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
18277 if (ScalarVT == MVT::f32 &&
18278 TII->isInlineConstant(Imm: TrueNode->getValueAPF()) &&
18279 TII->isInlineConstant(Imm: FalseNode->getValueAPF()))
18280 return SDValue();
18281
18282 int TrueNodeExpVal = TrueNode->getValueAPF().getExactLog2Abs();
18283 if (TrueNodeExpVal == INT_MIN)
18284 return SDValue();
18285 int FalseNodeExpVal = FalseNode->getValueAPF().getExactLog2Abs();
18286 if (FalseNodeExpVal == INT_MIN)
18287 return SDValue();
18288
18289 SDLoc SL(N);
18290 SDValue SelectNode =
18291 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: IntVT, N1: RHS.getOperand(i: 0),
18292 N2: DAG.getSignedConstant(Val: TrueNodeExpVal, DL: SL, VT: IntVT),
18293 N3: DAG.getSignedConstant(Val: FalseNodeExpVal, DL: SL, VT: IntVT));
18294
18295 LHS = TrueNode->isNegative()
18296 ? DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS, Flags: LHS->getFlags())
18297 : LHS;
18298
18299 return DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT, N1: LHS, N2: SelectNode, Flags: N->getFlags());
18300 }
18301
18302 return SDValue();
18303}
18304
18305SDValue SITargetLowering::performFMACombine(SDNode *N,
18306 DAGCombinerInfo &DCI) const {
18307 SelectionDAG &DAG = DCI.DAG;
18308 EVT VT = N->getValueType(ResNo: 0);
18309 SDLoc SL(N);
18310
18311 if (!Subtarget->hasDot10Insts() || VT != MVT::f32)
18312 return SDValue();
18313
18314 // FMA((F32)S0.x, (F32)S1. x, FMA((F32)S0.y, (F32)S1.y, (F32)z)) ->
18315 // FDOT2((V2F16)S0, (V2F16)S1, (F32)z))
18316 SDValue Op1 = N->getOperand(Num: 0);
18317 SDValue Op2 = N->getOperand(Num: 1);
18318 SDValue FMA = N->getOperand(Num: 2);
18319
18320 if (FMA.getOpcode() != ISD::FMA || Op1.getOpcode() != ISD::FP_EXTEND ||
18321 Op2.getOpcode() != ISD::FP_EXTEND)
18322 return SDValue();
18323
18324 // The fdot2 fold (fma_mix -> dot2) is only safe when both instructions agree
18325 // on how f16 subnormal inputs are handled. However, if both FMAs carry afn
18326 // the caller accepts approximate results, so any subnormal flushing
18327 // introduced by dot2 is acceptable regardless of mode.
18328 //
18329 // gfx90a (CDNA2) is the sole exception (dot2UnconditionalFlush): v_dot2c
18330 // unconditionally flushes f16 subnormal inputs to zero regardless of MODE,
18331 // while v_fma_mix_f32 preserves them when ieee=1 (the default compute kernel
18332 // mode). The fold is safe only when f32 denorm = PreserveSign, which implies
18333 // ieee=0 so both flush.
18334 //
18335 // All other GPUs: v_dot2 does NOT flush f16 subnormal inputs. v_fma_mix_f32
18336 // flushes them only when f32 denorm = PreserveSign. The fold is safe only
18337 // when f32 denorm is IEEE (both preserve the subnormal). Dynamic mode is
18338 // also rejected since the runtime value is unknown.
18339 bool AllowInaccuracy = N->getFlags().hasApproximateFuncs() &&
18340 FMA->getFlags().hasApproximateFuncs();
18341 if (!AllowInaccuracy) {
18342 const MachineFunction &MF = DAG.getMachineFunction();
18343 DenormalMode Mode = MF.getDenormalMode(FPType: APFloat::IEEEsingle());
18344 if (Subtarget->dot2UnconditionalFlush()) {
18345 // gfx90a: fold safe only when f32 denorm flushes.
18346 if (Mode != DenormalMode::getPreserveSign())
18347 return SDValue();
18348 } else {
18349 // All other GPUs: fold safe only when f32 denorm is IEEE.
18350 if (Mode != DenormalMode::getIEEE())
18351 return SDValue();
18352 }
18353 }
18354
18355 // fp-contract allows reassociating the fma tree into a dot product.
18356 const TargetOptions &Options = DAG.getTarget().Options;
18357 if (Options.AllowFPOpFusion == FPOpFusion::Fast ||
18358 (N->getFlags().hasAllowContract() &&
18359 FMA->getFlags().hasAllowContract())) {
18360 Op1 = Op1.getOperand(i: 0);
18361 Op2 = Op2.getOperand(i: 0);
18362 if (Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
18363 Op2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
18364 return SDValue();
18365
18366 SDValue Vec1 = Op1.getOperand(i: 0);
18367 SDValue Idx1 = Op1.getOperand(i: 1);
18368 SDValue Vec2 = Op2.getOperand(i: 0);
18369
18370 SDValue FMAOp1 = FMA.getOperand(i: 0);
18371 SDValue FMAOp2 = FMA.getOperand(i: 1);
18372 SDValue FMAAcc = FMA.getOperand(i: 2);
18373
18374 if (FMAOp1.getOpcode() != ISD::FP_EXTEND ||
18375 FMAOp2.getOpcode() != ISD::FP_EXTEND)
18376 return SDValue();
18377
18378 FMAOp1 = FMAOp1.getOperand(i: 0);
18379 FMAOp2 = FMAOp2.getOperand(i: 0);
18380 if (FMAOp1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
18381 FMAOp2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
18382 return SDValue();
18383
18384 SDValue Vec3 = FMAOp1.getOperand(i: 0);
18385 SDValue Vec4 = FMAOp2.getOperand(i: 0);
18386 SDValue Idx2 = FMAOp1.getOperand(i: 1);
18387
18388 if (Idx1 != Op2.getOperand(i: 1) || Idx2 != FMAOp2.getOperand(i: 1) ||
18389 // Idx1 and Idx2 cannot be the same.
18390 Idx1 == Idx2)
18391 return SDValue();
18392
18393 if (Vec1 == Vec2 || Vec3 == Vec4)
18394 return SDValue();
18395
18396 if (Vec1.getValueType() != MVT::v2f16 || Vec2.getValueType() != MVT::v2f16)
18397 return SDValue();
18398
18399 if ((Vec1 == Vec3 && Vec2 == Vec4) || (Vec1 == Vec4 && Vec2 == Vec3)) {
18400 return DAG.getNode(Opcode: AMDGPUISD::FDOT2, DL: SL, VT: MVT::f32, N1: Vec1, N2: Vec2, N3: FMAAcc,
18401 N4: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i1));
18402 }
18403 }
18404 return SDValue();
18405}
18406
18407// Given a double-precision ordered or unordered comparison, return the
18408// condition code for an equivalent integral comparison of the operands' upper
18409// 32 bits, or `SETCC_INVALID` if not possible.
18410// For simplicity, no simplification occurs if the operands are not both known
18411// to have sign bit zero.
18412//
18413// EQ/NE:
18414// If LHS.lo32 == RHS.lo32:
18415// setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, eq/ne
18416// If LHS.lo32 != RHS.lo32:
18417// setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, false/true
18418// The reduction is not possible if operands may be +0 and -0.
18419// For ordered eq / unordered ne, at most one operand may be NaN.
18420// For unordered eq / ordered ne, neither operand can be NaN.
18421//
18422// LT/GE:
18423// If LHS.lo32 >= RHS.lo32 (unsigned):
18424// setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]lt/ge
18425// If LHS.lo32 < RHS.lo32 (unsigned):
18426// setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]le/gt
18427// The reduction is only supported if both operands are nonnegative.
18428// For ordered lt / unordered ge, the RHS cannot be NaN.
18429// For unordered lt / ordered ge, neither operand can be NaN.
18430//
18431// LE/GT:
18432// If LHS.lo32 > RHS.lo32 (unsigned):
18433// setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]lt/ge
18434// If LHS.lo32 <= RHS.lo32 (unsigned):
18435// setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]le/gt
18436// The reduction is only supported if both operands are nonnegative.
18437// For unordered le / ordered gt, the LHS cannot be NaN.
18438// For ordered le / unordered gt, neither operand can be NaN.
18439static ISD::CondCode tryReduceF64CompareToHiHalf(const ISD::CondCode CC,
18440 const SDValue LHS,
18441 const SDValue RHS,
18442 const SelectionDAG &DAG) {
18443 EVT VT = LHS.getValueType();
18444 assert(VT == MVT::f64 && "Incorrect operand type!");
18445
18446 const KnownBits RHSBits = DAG.computeKnownBits(Op: RHS);
18447 // Bail if RHS sign bit is not known to be zero.
18448 if (!RHSBits.Zero.isSignBitSet())
18449 return ISD::SETCC_INVALID;
18450
18451 const KnownBits RHSKnownLo32 = RHSBits.trunc(BitWidth: 32);
18452 const KnownFPClass RHSFPClass =
18453 KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits: RHSBits);
18454 const bool RHSMaybeNaN = !RHSFPClass.isKnownNeverNaN();
18455
18456 const KnownBits LHSBits = DAG.computeKnownBits(Op: LHS);
18457 const KnownBits LHSKnownLo32 = LHSBits.trunc(BitWidth: 32);
18458 const KnownFPClass LHSFPClass =
18459 KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits: LHSBits);
18460 const bool LHSMaybeNaN = !LHSFPClass.isKnownNeverNaN();
18461
18462 // Bail if LHS sign bit is not known to be zero.
18463 if (!LHSBits.Zero.isSignBitSet())
18464 return ISD::SETCC_INVALID;
18465
18466 switch (CC) {
18467 default:
18468 break;
18469 case ISD::SETEQ:
18470 case ISD::SETOEQ:
18471 case ISD::SETUEQ:
18472 case ISD::SETONE:
18473 case ISD::SETUNE: {
18474 // OEQ should be false if either operand is NaN, so it suffices that at
18475 // least one operand is not NaN.
18476 if (CC == ISD::SETOEQ && LHSMaybeNaN && RHSMaybeNaN)
18477 break;
18478 // UEQ should be true if either operand is NaN, but this cannot be checked
18479 // on underlying bits.
18480 if (CC == ISD::SETUEQ && (LHSMaybeNaN || RHSMaybeNaN))
18481 break;
18482 // ONE should be false if either operand is NaN, but this cannot be
18483 // checked on underlying bits.
18484 if (CC == ISD::SETONE && (LHSMaybeNaN || RHSMaybeNaN))
18485 break;
18486 // UNE should be true if either operand is NaN, so it suffices that they
18487 // are not both NaN.
18488 if (CC == ISD::SETUNE && LHSMaybeNaN && RHSMaybeNaN)
18489 break;
18490
18491 const std::optional<bool> KnownEq =
18492 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18493
18494 if (!KnownEq)
18495 break;
18496
18497 if (*KnownEq)
18498 return (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ)
18499 ? ISD::SETEQ
18500 : ISD::SETNE;
18501
18502 return (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ)
18503 ? ISD::SETFALSE
18504 : ISD::SETTRUE;
18505 }
18506 case ISD::SETLT:
18507 case ISD::SETOLT:
18508 case ISD::SETULT:
18509 case ISD::SETGE:
18510 case ISD::SETOGE:
18511 case ISD::SETUGE: {
18512 // OLT should be false if either operand is NaN.
18513 // Since NaNs have maximum exponent and nonzero mantissa, false positives
18514 // are only possible if the RHS is NaN. (No issue with RHS == +inf since
18515 // the inequality is strict)
18516 if (CC == ISD::SETOLT && RHSMaybeNaN)
18517 break;
18518 // ULT should be true if either operand is NaN, but this cannot be ensured
18519 // with a truncated comparison.
18520 if (CC == ISD::SETULT && (LHSMaybeNaN || RHSMaybeNaN))
18521 break;
18522 // OGE should be false if either operand is NaN, but this cannot be
18523 // ensured with a truncated comparison.
18524 if (CC == ISD::SETOGE && (LHSMaybeNaN || RHSMaybeNaN))
18525 break;
18526 // UGE should be true if either operand is NaN.
18527 // False negatives are only possible if the RHS is NaN.
18528 // (No issue with RHS == +inf since the inequality is inclusive)
18529 if (CC == ISD::SETUGE && RHSMaybeNaN)
18530 break;
18531
18532 const std::optional<bool> KnownUge =
18533 KnownBits::uge(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18534
18535 if (!KnownUge)
18536 break;
18537
18538 if (*KnownUge) {
18539 // LHS.lo32 uge RHS.lo32, so LHS >= RHS iff LHS.hi32 >= RHS.hi32
18540 return (CC == ISD::SETLT || CC == ISD::SETOLT || CC == ISD::SETULT)
18541 ? ISD::SETLT
18542 : ISD::SETGE;
18543 }
18544 // LHS.lo32 ult RHS.lo32, so LHS >= RHS iff LHS.hi32 > RHS.hi32
18545 return (CC == ISD::SETLT || CC == ISD::SETOLT || CC == ISD::SETULT)
18546 ? ISD::SETLE
18547 : ISD::SETGT;
18548 }
18549 case ISD::SETLE:
18550 case ISD::SETOLE:
18551 case ISD::SETULE:
18552 case ISD::SETGT:
18553 case ISD::SETOGT:
18554 case ISD::SETUGT: {
18555 // OLE should be false if either operand is NaN, but this cannot be
18556 // ensured with a truncated comparison.
18557 if (CC == ISD::SETOLE && (LHSMaybeNaN || RHSMaybeNaN))
18558 break;
18559 // ULE should be true if either operand is NaN.
18560 // False negatives are only possible if the LHS is NaN.
18561 // (No issue with LHS == +inf since the inequality is inclusive)
18562 if (CC == ISD::SETULE && LHSMaybeNaN)
18563 break;
18564 // OGT should be false if either operand is NaN.
18565 // False positives are only possible if the LHS is NaN.
18566 // (No issue with LHS == +inf since the inequality is strict)
18567 if (CC == ISD::SETOGT && LHSMaybeNaN)
18568 break;
18569 // UGT should be true if either operand is NaN, but this cannot be ensured
18570 // with a truncated comparison.
18571 if (CC == ISD::SETUGT && (LHSMaybeNaN || RHSMaybeNaN))
18572 break;
18573
18574 const std::optional<bool> KnownUle =
18575 KnownBits::ule(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18576
18577 if (!KnownUle)
18578 break;
18579
18580 if (*KnownUle) {
18581 // LHS.lo32 ule RHS.lo32, so LHS <= RHS iff LHS.hi32 <= RHS.hi32
18582 return (CC == ISD::SETLE || CC == ISD::SETOLE || CC == ISD::SETULE)
18583 ? ISD::SETLE
18584 : ISD::SETGT;
18585 }
18586 // LHS.lo32 ugt RHS.lo32, so LHS <= RHS iff LHS.hi32 < RHS.hi32
18587 return (CC == ISD::SETLE || CC == ISD::SETOLE || CC == ISD::SETULE)
18588 ? ISD::SETLT
18589 : ISD::SETGE;
18590 }
18591 }
18592
18593 return ISD::SETCC_INVALID;
18594}
18595
18596SDValue SITargetLowering::performSetCCCombine(SDNode *N,
18597 DAGCombinerInfo &DCI) const {
18598 SelectionDAG &DAG = DCI.DAG;
18599 SDLoc SL(N);
18600
18601 SDValue LHS = N->getOperand(Num: 0);
18602 SDValue RHS = N->getOperand(Num: 1);
18603 EVT VT = LHS.getValueType();
18604 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
18605
18606 auto *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
18607 if (!CRHS) {
18608 CRHS = dyn_cast<ConstantSDNode>(Val&: LHS);
18609 if (CRHS) {
18610 std::swap(a&: LHS, b&: RHS);
18611 CC = getSetCCSwappedOperands(Operation: CC);
18612 }
18613 }
18614
18615 if (CRHS) {
18616 if (VT == MVT::i32 && LHS.getOpcode() == ISD::SIGN_EXTEND &&
18617 isBoolSGPR(V: LHS.getOperand(i: 0))) {
18618 // setcc (sext from i1 cc), -1, ne|sgt|ult) => not cc => xor cc, -1
18619 // setcc (sext from i1 cc), -1, eq|sle|uge) => cc
18620 // setcc (sext from i1 cc), 0, eq|sge|ule) => not cc => xor cc, -1
18621 // setcc (sext from i1 cc), 0, ne|ugt|slt) => cc
18622 if ((CRHS->isAllOnes() &&
18623 (CC == ISD::SETNE || CC == ISD::SETGT || CC == ISD::SETULT)) ||
18624 (CRHS->isZero() &&
18625 (CC == ISD::SETEQ || CC == ISD::SETGE || CC == ISD::SETULE)))
18626 return DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i1, N1: LHS.getOperand(i: 0),
18627 N2: DAG.getAllOnesConstant(DL: SL, VT: MVT::i1));
18628 if ((CRHS->isAllOnes() &&
18629 (CC == ISD::SETEQ || CC == ISD::SETLE || CC == ISD::SETUGE)) ||
18630 (CRHS->isZero() &&
18631 (CC == ISD::SETNE || CC == ISD::SETUGT || CC == ISD::SETLT)))
18632 return LHS.getOperand(i: 0);
18633 }
18634
18635 const APInt &CRHSVal = CRHS->getAPIntValue();
18636 if ((CC == ISD::SETEQ || CC == ISD::SETNE) &&
18637 LHS.getOpcode() == ISD::SELECT &&
18638 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
18639 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2)) &&
18640 isBoolSGPR(V: LHS.getOperand(i: 0))) {
18641 // Given CT != FT:
18642 // setcc (select cc, CT, CF), CF, eq => xor cc, -1
18643 // setcc (select cc, CT, CF), CF, ne => cc
18644 // setcc (select cc, CT, CF), CT, ne => xor cc, -1
18645 // setcc (select cc, CT, CF), CT, eq => cc
18646 const APInt &CT = LHS.getConstantOperandAPInt(i: 1);
18647 const APInt &CF = LHS.getConstantOperandAPInt(i: 2);
18648
18649 if (CT != CF) {
18650 if ((CF == CRHSVal && CC == ISD::SETEQ) ||
18651 (CT == CRHSVal && CC == ISD::SETNE))
18652 return DAG.getNOT(DL: SL, Val: LHS.getOperand(i: 0), VT: MVT::i1);
18653 if ((CF == CRHSVal && CC == ISD::SETNE) ||
18654 (CT == CRHSVal && CC == ISD::SETEQ))
18655 return LHS.getOperand(i: 0);
18656 }
18657 }
18658 }
18659
18660 // Truncate 64-bit setcc to test only upper 32-bits of its operands in the
18661 // following cases where information about the lower 32-bits of its operands
18662 // is known:
18663 //
18664 // If LHS.lo32 == RHS.lo32:
18665 // setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, eq/ne
18666 // If LHS.lo32 != RHS.lo32:
18667 // setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, false/true
18668 // If LHS.lo32 >= RHS.lo32 (unsigned):
18669 // setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]lt/ge
18670 // If LHS.lo32 > RHS.lo32 (unsigned):
18671 // setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]lt/ge
18672 // If LHS.lo32 <= RHS.lo32 (unsigned):
18673 // setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]le/gt
18674 // If LHS.lo32 < RHS.lo32 (unsigned):
18675 // setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]le/gt
18676 if (VT == MVT::i64) {
18677 const KnownBits LHSKnownLo32 = DAG.computeKnownBits(Op: LHS).trunc(BitWidth: 32);
18678 const KnownBits RHSKnownLo32 = DAG.computeKnownBits(Op: RHS).trunc(BitWidth: 32);
18679
18680 // NewCC is valid iff we can truncate the setcc to only test the upper 32
18681 // bits
18682 ISD::CondCode NewCC = ISD::SETCC_INVALID;
18683
18684 switch (CC) {
18685 default:
18686 break;
18687 case ISD::SETEQ: {
18688 const std::optional<bool> KnownEq =
18689 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18690 if (KnownEq)
18691 NewCC = *KnownEq ? ISD::SETEQ : ISD::SETFALSE;
18692
18693 break;
18694 }
18695 case ISD::SETNE: {
18696 const std::optional<bool> KnownEq =
18697 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18698 if (KnownEq)
18699 NewCC = *KnownEq ? ISD::SETNE : ISD::SETTRUE;
18700
18701 break;
18702 }
18703 case ISD::SETULT:
18704 case ISD::SETUGE:
18705 case ISD::SETLT:
18706 case ISD::SETGE: {
18707 const std::optional<bool> KnownUge =
18708 KnownBits::uge(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18709 if (KnownUge) {
18710 if (*KnownUge) {
18711 // LHS.lo32 uge RHS.lo32, so LHS >= RHS iff LHS.hi32 >= RHS.hi32
18712 NewCC = CC;
18713 } else {
18714 // LHS.lo32 ult RHS.lo32, so LHS >= RHS iff LHS.hi32 > RHS.hi32
18715 NewCC = CC == ISD::SETULT ? ISD::SETULE
18716 : CC == ISD::SETUGE ? ISD::SETUGT
18717 : CC == ISD::SETLT ? ISD::SETLE
18718 : ISD::SETGT;
18719 }
18720 }
18721 break;
18722 }
18723 case ISD::SETULE:
18724 case ISD::SETUGT:
18725 case ISD::SETLE:
18726 case ISD::SETGT: {
18727 const std::optional<bool> KnownUle =
18728 KnownBits::ule(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18729 if (KnownUle) {
18730 if (*KnownUle) {
18731 // LHS.lo32 ule RHS.lo32, so LHS <= RHS iff LHS.hi32 <= RHS.hi32
18732 NewCC = CC;
18733 } else {
18734 // LHS.lo32 ugt RHS.lo32, so LHS <= RHS iff LHS.hi32 < RHS.hi32
18735 NewCC = CC == ISD::SETULE ? ISD::SETULT
18736 : CC == ISD::SETUGT ? ISD::SETUGE
18737 : CC == ISD::SETLE ? ISD::SETLT
18738 : ISD::SETGE;
18739 }
18740 }
18741 break;
18742 }
18743 }
18744
18745 if (NewCC != ISD::SETCC_INVALID)
18746 return DAG.getSetCC(DL: SL, VT: N->getValueType(ResNo: 0), LHS: getHiHalf64(Op: LHS, DAG),
18747 RHS: getHiHalf64(Op: RHS, DAG), Cond: NewCC);
18748 }
18749
18750 // Eliminate setcc by using carryout from add/sub instruction
18751
18752 // LHS = ADD i64 RHS, Z LHSlo = UADDO i32 RHSlo, Zlo
18753 // setcc LHS ult RHS -> LHSHi = UADDO_CARRY i32 RHShi, Zhi
18754 // similarly for subtraction
18755
18756 // LHS = ADD i64 Y, 1 LHSlo = UADDO i32 Ylo, 1
18757 // setcc LHS eq 0 -> LHSHi = UADDO_CARRY i32 Yhi, 0
18758
18759 if (VT == MVT::i64 && ((CC == ISD::SETULT &&
18760 sd_match(N: LHS, P: m_Add(L: m_Specific(N: RHS), R: m_Value()))) ||
18761 (CC == ISD::SETUGT &&
18762 sd_match(N: LHS, P: m_Sub(L: m_Specific(N: RHS), R: m_Value()))) ||
18763 (CC == ISD::SETEQ && CRHS && CRHS->isZero() &&
18764 sd_match(N: LHS, P: m_Add(L: m_Value(), R: m_One()))))) {
18765 bool IsAdd = LHS.getOpcode() == ISD::ADD;
18766
18767 SDValue Op0 = LHS.getOperand(i: 0);
18768 SDValue Op1 = LHS.getOperand(i: 1);
18769
18770 SDValue Op0Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Op0);
18771 SDValue Op1Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Op1);
18772
18773 SDValue Op0Hi = getHiHalf64(Op: Op0, DAG);
18774 SDValue Op1Hi = getHiHalf64(Op: Op1, DAG);
18775
18776 SDValue NodeLo =
18777 DAG.getNode(Opcode: IsAdd ? ISD::UADDO : ISD::USUBO, DL: SL,
18778 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1), Ops: {Op0Lo, Op1Lo});
18779
18780 SDValue CarryInHi = NodeLo.getValue(R: 1);
18781 SDValue NodeHi = DAG.getNode(Opcode: IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY,
18782 DL: SL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1),
18783 Ops: {Op0Hi, Op1Hi, CarryInHi});
18784
18785 SDValue ResultLo = NodeLo.getValue(R: 0);
18786 SDValue ResultHi = NodeHi.getValue(R: 0);
18787
18788 SDValue JoinedResult =
18789 DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {ResultLo, ResultHi});
18790
18791 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: JoinedResult);
18792 SDValue Overflow = NodeHi.getValue(R: 1);
18793 DCI.CombineTo(N: LHS.getNode(), Res: Result);
18794 return Overflow;
18795 }
18796
18797 if (VT != MVT::f32 && VT != MVT::f64 &&
18798 (!Subtarget->has16BitInsts() || VT != MVT::f16))
18799 return SDValue();
18800
18801 // Match isinf/isfinite pattern
18802 // (fcmp oeq (fabs x), inf) -> (fp_class x, (p_infinity | n_infinity))
18803 // (fcmp one (fabs x), inf) -> (fp_class x,
18804 // (p_normal | n_normal | p_subnormal | n_subnormal | p_zero | n_zero)
18805 if ((CC == ISD::SETOEQ || CC == ISD::SETONE) &&
18806 LHS.getOpcode() == ISD::FABS) {
18807 const ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
18808 if (!CRHS)
18809 return SDValue();
18810
18811 const APFloat &APF = CRHS->getValueAPF();
18812 if (APF.isInfinity() && !APF.isNegative()) {
18813 const unsigned IsInfMask =
18814 SIInstrFlags::P_INFINITY | SIInstrFlags::N_INFINITY;
18815 const unsigned IsFiniteMask =
18816 SIInstrFlags::N_ZERO | SIInstrFlags::P_ZERO | SIInstrFlags::N_NORMAL |
18817 SIInstrFlags::P_NORMAL | SIInstrFlags::N_SUBNORMAL |
18818 SIInstrFlags::P_SUBNORMAL;
18819 unsigned Mask = CC == ISD::SETOEQ ? IsInfMask : IsFiniteMask;
18820 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL: SL, VT: MVT::i1, N1: LHS.getOperand(i: 0),
18821 N2: DAG.getConstant(Val: Mask, DL: SL, VT: MVT::i32));
18822 }
18823 }
18824
18825 if (VT == MVT::f64) {
18826 ISD::CondCode HiHalfCC = tryReduceF64CompareToHiHalf(CC, LHS, RHS, DAG);
18827 if (HiHalfCC != ISD::SETCC_INVALID)
18828 return DAG.getSetCC(DL: SL, VT: N->getValueType(ResNo: 0), LHS: getHiHalf64(Op: LHS, DAG),
18829 RHS: getHiHalf64(Op: RHS, DAG), Cond: HiHalfCC);
18830 }
18831
18832 return SDValue();
18833}
18834
18835SDValue
18836SITargetLowering::performCvtF32UByteNCombine(SDNode *N,
18837 DAGCombinerInfo &DCI) const {
18838 SelectionDAG &DAG = DCI.DAG;
18839 SDLoc SL(N);
18840 unsigned Offset = N->getOpcode() - AMDGPUISD::CVT_F32_UBYTE0;
18841
18842 SDValue Src = N->getOperand(Num: 0);
18843 SDValue Shift = N->getOperand(Num: 0);
18844
18845 // TODO: Extend type shouldn't matter (assuming legal types).
18846 if (Shift.getOpcode() == ISD::ZERO_EXTEND)
18847 Shift = Shift.getOperand(i: 0);
18848
18849 if (Shift.getOpcode() == ISD::SRL || Shift.getOpcode() == ISD::SHL) {
18850 // cvt_f32_ubyte1 (shl x, 8) -> cvt_f32_ubyte0 x
18851 // cvt_f32_ubyte3 (shl x, 16) -> cvt_f32_ubyte1 x
18852 // cvt_f32_ubyte0 (srl x, 16) -> cvt_f32_ubyte2 x
18853 // cvt_f32_ubyte1 (srl x, 16) -> cvt_f32_ubyte3 x
18854 // cvt_f32_ubyte0 (srl x, 8) -> cvt_f32_ubyte1 x
18855 if (auto *C = dyn_cast<ConstantSDNode>(Val: Shift.getOperand(i: 1))) {
18856 SDValue Shifted = DAG.getZExtOrTrunc(
18857 Op: Shift.getOperand(i: 0), DL: SDLoc(Shift.getOperand(i: 0)), VT: MVT::i32);
18858
18859 unsigned ShiftOffset = 8 * Offset;
18860 if (Shift.getOpcode() == ISD::SHL)
18861 ShiftOffset -= C->getZExtValue();
18862 else
18863 ShiftOffset += C->getZExtValue();
18864
18865 if (ShiftOffset < 32 && (ShiftOffset % 8) == 0) {
18866 return DAG.getNode(Opcode: AMDGPUISD::CVT_F32_UBYTE0 + ShiftOffset / 8, DL: SL,
18867 VT: MVT::f32, Operand: Shifted);
18868 }
18869 }
18870 }
18871
18872 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
18873 APInt DemandedBits = APInt::getBitsSet(numBits: 32, loBit: 8 * Offset, hiBit: 8 * Offset + 8);
18874 if (TLI.SimplifyDemandedBits(Op: Src, DemandedBits, DCI)) {
18875 // We simplified Src. If this node is not dead, visit it again so it is
18876 // folded properly.
18877 if (N->getOpcode() != ISD::DELETED_NODE)
18878 DCI.AddToWorklist(N);
18879 return SDValue(N, 0);
18880 }
18881
18882 // Handle (or x, (srl y, 8)) pattern when known bits are zero.
18883 if (SDValue DemandedSrc =
18884 TLI.SimplifyMultipleUseDemandedBits(Op: Src, DemandedBits, DAG))
18885 return DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: MVT::f32, Operand: DemandedSrc);
18886
18887 return SDValue();
18888}
18889
18890SDValue SITargetLowering::performClampCombine(SDNode *N,
18891 DAGCombinerInfo &DCI) const {
18892 ConstantFPSDNode *CSrc = dyn_cast<ConstantFPSDNode>(Val: N->getOperand(Num: 0));
18893 if (!CSrc)
18894 return SDValue();
18895
18896 const MachineFunction &MF = DCI.DAG.getMachineFunction();
18897 const APFloat &F = CSrc->getValueAPF();
18898 APFloat Zero = APFloat::getZero(Sem: F.getSemantics());
18899 if (F < Zero ||
18900 (F.isNaN() && MF.getInfo<SIMachineFunctionInfo>()->getMode().DX10Clamp)) {
18901 return DCI.DAG.getConstantFP(Val: Zero, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
18902 }
18903
18904 APFloat One = APFloat::getOne(Sem: F.getSemantics());
18905 if (F > One)
18906 return DCI.DAG.getConstantFP(Val: One, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
18907
18908 return getCanonicalConstantFP(DAG&: DCI.DAG, SL: SDLoc(N), VT: N->getValueType(ResNo: 0), C: F);
18909}
18910
18911SDValue SITargetLowering::performSelectCombine(SDNode *N,
18912 DAGCombinerInfo &DCI) const {
18913
18914 // Try to fold CMP + SELECT patterns with shared constants (both FP and
18915 // integer).
18916 // Detect when CMP and SELECT use the same constant and fold them to avoid
18917 // loading the constant twice. Specifically handles patterns like:
18918 // %cmp = icmp eq i32 %val, 4242
18919 // %sel = select i1 %cmp, i32 4242, i32 %other
18920 // It can be optimized to reuse %val instead of 4242 in select.
18921 SDValue Cond = N->getOperand(Num: 0);
18922 SDValue TrueVal = N->getOperand(Num: 1);
18923 SDValue FalseVal = N->getOperand(Num: 2);
18924
18925 // Check if condition is a comparison.
18926 if (Cond.getOpcode() != ISD::SETCC)
18927 return SDValue();
18928
18929 SDValue LHS = Cond.getOperand(i: 0);
18930 SDValue RHS = Cond.getOperand(i: 1);
18931 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
18932
18933 bool isFloatingPoint = LHS.getValueType().isFloatingPoint();
18934 bool isInteger = LHS.getValueType().isInteger();
18935
18936 // Handle simple floating-point and integer types only.
18937 if (!isFloatingPoint && !isInteger)
18938 return SDValue();
18939
18940 // Bare SETEQ/SETNE is the builder's NaN-impossible downgrade.
18941 bool isEquality = CC == ISD::SETEQ || (isFloatingPoint && CC == ISD::SETOEQ);
18942 bool isNonEquality =
18943 CC == ISD::SETNE || (isFloatingPoint && CC == ISD::SETONE);
18944 if (!isEquality && !isNonEquality)
18945 return SDValue();
18946
18947 SDValue ArgVal, ConstVal;
18948 if ((isFloatingPoint && isa<ConstantFPSDNode>(Val: RHS)) ||
18949 (isInteger && isa<ConstantSDNode>(Val: RHS))) {
18950 ConstVal = RHS;
18951 ArgVal = LHS;
18952 } else if ((isFloatingPoint && isa<ConstantFPSDNode>(Val: LHS)) ||
18953 (isInteger && isa<ConstantSDNode>(Val: LHS))) {
18954 ConstVal = LHS;
18955 ArgVal = RHS;
18956 } else {
18957 return SDValue();
18958 }
18959
18960 // Skip optimization for inlinable immediates.
18961 if (isFloatingPoint) {
18962 const APFloat &Val = cast<ConstantFPSDNode>(Val&: ConstVal)->getValueAPF();
18963 if (!Val.isNormal() || Subtarget->getInstrInfo()->isInlineConstant(Imm: Val))
18964 return SDValue();
18965 } else {
18966 const std::optional<int64_t> Val =
18967 cast<ConstantSDNode>(Val&: ConstVal)->getAPIntValue().trySExtValue();
18968 if (Val && AMDGPU::isInlinableIntLiteral(Literal: *Val))
18969 return SDValue();
18970 }
18971
18972 // For equality and non-equality comparisons, patterns:
18973 // select (setcc x, const), const, y -> select (setcc x, const), x, y
18974 // select (setccinv x, const), y, const -> select (setccinv x, const), y, x
18975 if (!(isEquality && TrueVal == ConstVal) &&
18976 !(isNonEquality && FalseVal == ConstVal))
18977 return SDValue();
18978
18979 // SETONE's false arm is also taken for NaN ArgVal, so require NaN excluded.
18980 if (isFloatingPoint && isNonEquality && FalseVal == ConstVal &&
18981 !Cond->getFlags().hasNoNaNs() && !DCI.DAG.isKnownNeverNaN(Op: ArgVal))
18982 return SDValue();
18983
18984 SDValue SelectLHS = (isEquality && TrueVal == ConstVal) ? ArgVal : TrueVal;
18985 SDValue SelectRHS =
18986 (isNonEquality && FalseVal == ConstVal) ? ArgVal : FalseVal;
18987 return DCI.DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Cond,
18988 N2: SelectLHS, N3: SelectRHS);
18989}
18990
18991SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
18992 DAGCombinerInfo &DCI) const {
18993 switch (N->getOpcode()) {
18994 case ISD::ABS:
18995 if (SDValue Res = promoteUniformUnaryOpToI32(Op: SDValue(N, 0), DCI))
18996 return Res;
18997 break;
18998 case ISD::ADD:
18999 case ISD::SUB:
19000 case ISD::SHL:
19001 case ISD::SRL:
19002 case ISD::SRA:
19003 case ISD::AND:
19004 case ISD::OR:
19005 case ISD::XOR:
19006 case ISD::MUL:
19007 case ISD::SETCC:
19008 case ISD::SELECT:
19009 case ISD::SMIN:
19010 case ISD::SMAX:
19011 case ISD::UMIN:
19012 case ISD::UMAX:
19013 case ISD::USUBSAT:
19014 if (auto Res = promoteUniformOpToI32(Op: SDValue(N, 0), DCI))
19015 return Res;
19016 break;
19017 default:
19018 break;
19019 }
19020
19021 if (getTargetMachine().getOptLevel() == CodeGenOptLevel::None)
19022 return SDValue();
19023
19024 switch (N->getOpcode()) {
19025 case ISD::ADD:
19026 return performAddCombine(N, DCI);
19027 case ISD::PTRADD:
19028 return performPtrAddCombine(N, DCI);
19029 case ISD::SUB:
19030 return performSubCombine(N, DCI);
19031 case ISD::FADD:
19032 return performFAddCombine(N, DCI);
19033 case ISD::FSUB:
19034 return performFSubCombine(N, DCI);
19035 case ISD::FDIV:
19036 return performFDivCombine(N, DCI);
19037 case ISD::FMUL:
19038 return performFMulCombine(N, DCI);
19039 case ISD::SETCC:
19040 return performSetCCCombine(N, DCI);
19041 case ISD::SELECT:
19042 if (auto Res = performSelectCombine(N, DCI))
19043 return Res;
19044 break;
19045 case ISD::FMAXNUM:
19046 case ISD::FMINNUM:
19047 case ISD::FMAXNUM_IEEE:
19048 case ISD::FMINNUM_IEEE:
19049 case ISD::FMAXIMUM:
19050 case ISD::FMINIMUM:
19051 case ISD::FMAXIMUMNUM:
19052 case ISD::FMINIMUMNUM:
19053 case ISD::SMAX:
19054 case ISD::SMIN:
19055 case ISD::UMAX:
19056 case ISD::UMIN:
19057 case AMDGPUISD::FMIN_LEGACY:
19058 case AMDGPUISD::FMAX_LEGACY:
19059 return performMinMaxCombine(N, DCI);
19060 case ISD::FMA:
19061 return performFMACombine(N, DCI);
19062 case ISD::AND:
19063 return performAndCombine(N, DCI);
19064 case ISD::OR:
19065 return performOrCombine(N, DCI);
19066 case ISD::FSHR: {
19067 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19068 if (N->getValueType(ResNo: 0) == MVT::i32 && N->isDivergent() &&
19069 TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
19070 return matchPERM(N, DCI);
19071 }
19072 break;
19073 }
19074 case ISD::XOR:
19075 return performXorCombine(N, DCI);
19076 case ISD::ANY_EXTEND:
19077 case ISD::ZERO_EXTEND:
19078 return performZeroOrAnyExtendCombine(N, DCI);
19079 case ISD::SIGN_EXTEND_INREG:
19080 return performSignExtendInRegCombine(N, DCI);
19081 case AMDGPUISD::FP_CLASS:
19082 return performClassCombine(N, DCI);
19083 case ISD::FCANONICALIZE:
19084 return performFCanonicalizeCombine(N, DCI);
19085 case AMDGPUISD::RCP:
19086 return performRcpCombine(N, DCI);
19087 case ISD::FLDEXP:
19088 case AMDGPUISD::FRACT:
19089 case AMDGPUISD::RSQ:
19090 case AMDGPUISD::RCP_LEGACY:
19091 case AMDGPUISD::RCP_IFLAG:
19092 case AMDGPUISD::RSQ_CLAMP: {
19093 // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
19094 SDValue Src = N->getOperand(Num: 0);
19095 if (Src.isUndef())
19096 return Src;
19097 break;
19098 }
19099 case ISD::SINT_TO_FP:
19100 case ISD::UINT_TO_FP:
19101 return performUCharToFloatCombine(N, DCI);
19102 case ISD::FCOPYSIGN:
19103 return performFCopySignCombine(N, DCI);
19104 case AMDGPUISD::CVT_F32_UBYTE0:
19105 case AMDGPUISD::CVT_F32_UBYTE1:
19106 case AMDGPUISD::CVT_F32_UBYTE2:
19107 case AMDGPUISD::CVT_F32_UBYTE3:
19108 return performCvtF32UByteNCombine(N, DCI);
19109 case AMDGPUISD::FMED3:
19110 return performFMed3Combine(N, DCI);
19111 case AMDGPUISD::CVT_PKRTZ_F16_F32:
19112 return performCvtPkRTZCombine(N, DCI);
19113 case AMDGPUISD::CLAMP:
19114 return performClampCombine(N, DCI);
19115 case ISD::SCALAR_TO_VECTOR: {
19116 SelectionDAG &DAG = DCI.DAG;
19117 EVT VT = N->getValueType(ResNo: 0);
19118
19119 // v2i16 (scalar_to_vector i16:x) -> v2i16 (bitcast (any_extend i16:x))
19120 if (VT == MVT::v2i16 || VT == MVT::v2f16 || VT == MVT::v2bf16) {
19121 SDLoc SL(N);
19122 SDValue Src = N->getOperand(Num: 0);
19123 EVT EltVT = Src.getValueType();
19124 if (EltVT != MVT::i16)
19125 Src = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Src);
19126
19127 SDValue Ext = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Src);
19128 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Ext);
19129 }
19130
19131 break;
19132 }
19133 case ISD::EXTRACT_VECTOR_ELT:
19134 return performExtractVectorEltCombine(N, DCI);
19135 case ISD::INSERT_VECTOR_ELT:
19136 return performInsertVectorEltCombine(N, DCI);
19137 case ISD::FP_ROUND:
19138 return performFPRoundCombine(N, DCI);
19139 case ISD::LOAD: {
19140 if (SDValue Widened = widenLoad(Ld: cast<LoadSDNode>(Val: N), DCI))
19141 return Widened;
19142 [[fallthrough]];
19143 }
19144 default: {
19145 if (!DCI.isBeforeLegalize()) {
19146 if (MemSDNode *MemNode = dyn_cast<MemSDNode>(Val: N))
19147 return performMemSDNodeCombine(N: MemNode, DCI);
19148 }
19149
19150 break;
19151 }
19152 }
19153
19154 return AMDGPUTargetLowering::PerformDAGCombine(N, DCI);
19155}
19156
19157/// Helper function for adjustWritemask
19158static unsigned SubIdx2Lane(unsigned Idx) {
19159 switch (Idx) {
19160 default:
19161 return ~0u;
19162 case AMDGPU::sub0:
19163 return 0;
19164 case AMDGPU::sub1:
19165 return 1;
19166 case AMDGPU::sub2:
19167 return 2;
19168 case AMDGPU::sub3:
19169 return 3;
19170 case AMDGPU::sub4:
19171 return 4; // Possible with TFE/LWE
19172 }
19173}
19174
19175/// Adjust the writemask of MIMG, VIMAGE or VSAMPLE instructions
19176SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
19177 SelectionDAG &DAG) const {
19178 unsigned Opcode = Node->getMachineOpcode();
19179
19180 // Subtract 1 because the vdata output is not a MachineSDNode operand.
19181 int D16Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::d16) - 1;
19182 if (D16Idx >= 0 && Node->getConstantOperandVal(Num: D16Idx))
19183 return Node; // not implemented for D16
19184
19185 SDNode *Users[5] = {nullptr};
19186 unsigned Lane = 0;
19187 unsigned DmaskIdx =
19188 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::dmask) - 1;
19189 unsigned OldDmask = Node->getConstantOperandVal(Num: DmaskIdx);
19190 unsigned NewDmask = 0;
19191 unsigned TFEIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::tfe) - 1;
19192 unsigned LWEIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::lwe) - 1;
19193 bool UsesTFC = (int(TFEIdx) >= 0 && Node->getConstantOperandVal(Num: TFEIdx)) ||
19194 (int(LWEIdx) >= 0 && Node->getConstantOperandVal(Num: LWEIdx));
19195 unsigned TFCLane = 0;
19196 bool HasChain = Node->getNumValues() > 1;
19197
19198 if (OldDmask == 0) {
19199 // These are folded out, but on the chance it happens don't assert.
19200 return Node;
19201 }
19202
19203 unsigned OldBitsSet = llvm::popcount(Value: OldDmask);
19204 // Work out which is the TFE/LWE lane if that is enabled.
19205 if (UsesTFC) {
19206 TFCLane = OldBitsSet;
19207 }
19208
19209 // Try to figure out the used register components
19210 for (SDUse &Use : Node->uses()) {
19211
19212 // Don't look at users of the chain.
19213 if (Use.getResNo() != 0)
19214 continue;
19215
19216 SDNode *User = Use.getUser();
19217
19218 // Abort if we can't understand the usage
19219 if (!User->isMachineOpcode() ||
19220 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
19221 return Node;
19222
19223 // Lane means which subreg of %vgpra_vgprb_vgprc_vgprd is used.
19224 // Note that subregs are packed, i.e. Lane==0 is the first bit set
19225 // in OldDmask, so it can be any of X,Y,Z,W; Lane==1 is the second bit
19226 // set, etc.
19227 Lane = SubIdx2Lane(Idx: User->getConstantOperandVal(Num: 1));
19228 if (Lane == ~0u)
19229 return Node;
19230
19231 // Check if the use is for the TFE/LWE generated result at VGPRn+1.
19232 if (UsesTFC && Lane == TFCLane) {
19233 Users[Lane] = User;
19234 } else {
19235 // Set which texture component corresponds to the lane.
19236 unsigned Comp;
19237 for (unsigned i = 0, Dmask = OldDmask; (i <= Lane) && (Dmask != 0); i++) {
19238 Comp = llvm::countr_zero(Val: Dmask);
19239 Dmask &= ~(1 << Comp);
19240 }
19241
19242 // Abort if we have more than one user per component.
19243 if (Users[Lane])
19244 return Node;
19245
19246 Users[Lane] = User;
19247 NewDmask |= 1 << Comp;
19248 }
19249 }
19250
19251 // Don't allow 0 dmask, as hardware assumes one channel enabled.
19252 bool NoChannels = !NewDmask;
19253 if (NoChannels) {
19254 if (!UsesTFC) {
19255 // No uses of the result and not using TFC. Then do nothing.
19256 return Node;
19257 }
19258 // If the original dmask has one channel - then nothing to do
19259 if (OldBitsSet == 1)
19260 return Node;
19261 // Use an arbitrary dmask - required for the instruction to work
19262 NewDmask = 1;
19263 }
19264 // Abort if there's no change
19265 if (NewDmask == OldDmask)
19266 return Node;
19267
19268 unsigned BitsSet = llvm::popcount(Value: NewDmask);
19269
19270 // Check for TFE or LWE - increase the number of channels by one to account
19271 // for the extra return value
19272 // This will need adjustment for D16 if this is also included in
19273 // adjustWriteMask (this function) but at present D16 are excluded.
19274 unsigned NewChannels = BitsSet + UsesTFC;
19275
19276 int NewOpcode =
19277 AMDGPU::getMaskedMIMGOp(Opc: Node->getMachineOpcode(), NewChannels);
19278 assert(NewOpcode != -1 &&
19279 NewOpcode != static_cast<int>(Node->getMachineOpcode()) &&
19280 "failed to find equivalent MIMG op");
19281
19282 // Adjust the writemask in the node
19283 SmallVector<SDValue, 12> Ops;
19284 llvm::append_range(C&: Ops, R: Node->ops().take_front(N: DmaskIdx));
19285 Ops.push_back(Elt: DAG.getTargetConstant(Val: NewDmask, DL: SDLoc(Node), VT: MVT::i32));
19286 llvm::append_range(C&: Ops, R: Node->ops().drop_front(N: DmaskIdx + 1));
19287
19288 MVT SVT = Node->getValueType(ResNo: 0).getVectorElementType().getSimpleVT();
19289
19290 MVT ResultVT = NewChannels == 1
19291 ? SVT
19292 : MVT::getVectorVT(VT: SVT, NumElements: NewChannels == 3 ? 4
19293 : NewChannels == 5 ? 8
19294 : NewChannels);
19295 SDVTList NewVTList =
19296 HasChain ? DAG.getVTList(VT1: ResultVT, VT2: MVT::Other) : DAG.getVTList(VT: ResultVT);
19297
19298 MachineSDNode *NewNode =
19299 DAG.getMachineNode(Opcode: NewOpcode, dl: SDLoc(Node), VTs: NewVTList, Ops);
19300
19301 if (HasChain) {
19302 // Update chain.
19303 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: Node->memoperands());
19304 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: SDValue(NewNode, 1));
19305 }
19306
19307 if (NewChannels == 1) {
19308 assert(Node->hasNUsesOfValue(1, 0));
19309 SDNode *Copy =
19310 DAG.getMachineNode(Opcode: TargetOpcode::COPY, dl: SDLoc(Node),
19311 VT: Users[Lane]->getValueType(ResNo: 0), Op1: SDValue(NewNode, 0));
19312 DAG.ReplaceAllUsesWith(From: Users[Lane], To: Copy);
19313 return nullptr;
19314 }
19315
19316 // Update the users of the node with the new indices
19317 for (unsigned i = 0, Idx = AMDGPU::sub0; i < 5; ++i) {
19318 SDNode *User = Users[i];
19319 if (!User) {
19320 // Handle the special case of NoChannels. We set NewDmask to 1 above, but
19321 // Users[0] is still nullptr because channel 0 doesn't really have a use.
19322 if (i || !NoChannels)
19323 continue;
19324 } else {
19325 SDValue Op = DAG.getTargetConstant(Val: Idx, DL: SDLoc(User), VT: MVT::i32);
19326 SDNode *NewUser = DAG.UpdateNodeOperands(N: User, Op1: SDValue(NewNode, 0), Op2: Op);
19327 if (NewUser != User) {
19328 DAG.ReplaceAllUsesWith(From: SDValue(User, 0), To: SDValue(NewUser, 0));
19329 DAG.RemoveDeadNode(N: User);
19330 }
19331 }
19332
19333 switch (Idx) {
19334 default:
19335 break;
19336 case AMDGPU::sub0:
19337 Idx = AMDGPU::sub1;
19338 break;
19339 case AMDGPU::sub1:
19340 Idx = AMDGPU::sub2;
19341 break;
19342 case AMDGPU::sub2:
19343 Idx = AMDGPU::sub3;
19344 break;
19345 case AMDGPU::sub3:
19346 Idx = AMDGPU::sub4;
19347 break;
19348 }
19349 }
19350
19351 DAG.RemoveDeadNode(N: Node);
19352 return nullptr;
19353}
19354
19355static bool isFrameIndexOp(SDValue Op) {
19356 if (Op.getOpcode() == ISD::AssertZext)
19357 Op = Op.getOperand(i: 0);
19358
19359 return isa<FrameIndexSDNode>(Val: Op);
19360}
19361
19362/// Legalize target independent instructions (e.g. INSERT_SUBREG)
19363/// with frame index operands.
19364/// LLVM assumes that inputs are to these instructions are registers.
19365SDNode *
19366SITargetLowering::legalizeTargetIndependentNode(SDNode *Node,
19367 SelectionDAG &DAG) const {
19368 if (Node->getOpcode() == ISD::CopyToReg) {
19369 RegisterSDNode *DestReg = cast<RegisterSDNode>(Val: Node->getOperand(Num: 1));
19370 SDValue SrcVal = Node->getOperand(Num: 2);
19371
19372 // Insert a copy to a VReg_1 virtual register so LowerI1Copies doesn't have
19373 // to try understanding copies to physical registers.
19374 if (SrcVal.getValueType() == MVT::i1 && DestReg->getReg().isPhysical()) {
19375 SDLoc SL(Node);
19376 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
19377 SDValue VReg = DAG.getRegister(
19378 Reg: MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_1RegClass), VT: MVT::i1);
19379
19380 SDNode *Glued = Node->getGluedNode();
19381 SDValue ToVReg = DAG.getCopyToReg(
19382 Chain: Node->getOperand(Num: 0), dl: SL, Reg: VReg, N: SrcVal,
19383 Glue: SDValue(Glued, Glued ? Glued->getNumValues() - 1 : 0));
19384 SDValue ToResultReg = DAG.getCopyToReg(Chain: ToVReg, dl: SL, Reg: SDValue(DestReg, 0),
19385 N: VReg, Glue: ToVReg.getValue(R: 1));
19386 DAG.ReplaceAllUsesWith(From: Node, To: ToResultReg.getNode());
19387 DAG.RemoveDeadNode(N: Node);
19388 return ToResultReg.getNode();
19389 }
19390 }
19391
19392 SmallVector<SDValue, 8> Ops;
19393 for (unsigned i = 0; i < Node->getNumOperands(); ++i) {
19394 if (!isFrameIndexOp(Op: Node->getOperand(Num: i))) {
19395 Ops.push_back(Elt: Node->getOperand(Num: i));
19396 continue;
19397 }
19398
19399 SDLoc DL(Node);
19400 Ops.push_back(Elt: SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL,
19401 VT: Node->getOperand(Num: i).getValueType(),
19402 Op1: Node->getOperand(Num: i)),
19403 0));
19404 }
19405
19406 return DAG.UpdateNodeOperands(N: Node, Ops);
19407}
19408
19409/// Fold the instructions after selecting them.
19410/// Returns null if users were already updated.
19411SDNode *SITargetLowering::PostISelFolding(MachineSDNode *Node,
19412 SelectionDAG &DAG) const {
19413 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19414 unsigned Opcode = Node->getMachineOpcode();
19415
19416 if (TII->isImage(Opcode) && !TII->get(Opcode).mayStore() &&
19417 !TII->isGather4(Opcode) &&
19418 AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::dmask)) {
19419 return adjustWritemask(Node, DAG);
19420 }
19421
19422 if (Opcode == AMDGPU::INSERT_SUBREG || Opcode == AMDGPU::REG_SEQUENCE) {
19423 legalizeTargetIndependentNode(Node, DAG);
19424 return Node;
19425 }
19426
19427 switch (Opcode) {
19428 case AMDGPU::V_DIV_SCALE_F32_e64:
19429 case AMDGPU::V_DIV_SCALE_F64_e64: {
19430 // Satisfy the operand register constraint when one of the inputs is
19431 // undefined. Ordinarily each undef value will have its own implicit_def of
19432 // a vreg, so force these to use a single register.
19433 SDValue Src0 = Node->getOperand(Num: 1);
19434 SDValue Src1 = Node->getOperand(Num: 3);
19435 SDValue Src2 = Node->getOperand(Num: 5);
19436
19437 if ((Src0.isMachineOpcode() &&
19438 Src0.getMachineOpcode() != AMDGPU::IMPLICIT_DEF) &&
19439 (Src0 == Src1 || Src0 == Src2))
19440 break;
19441
19442 MVT VT = Src0.getValueType().getSimpleVT();
19443 const TargetRegisterClass *RC =
19444 getRegClassFor(VT, isDivergent: Src0.getNode()->isDivergent());
19445
19446 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
19447 SDValue UndefReg = DAG.getRegister(Reg: MRI.createVirtualRegister(RegClass: RC), VT);
19448
19449 SDValue ImpDef = DAG.getCopyToReg(Chain: DAG.getEntryNode(), dl: SDLoc(Node), Reg: UndefReg,
19450 N: Src0, Glue: SDValue());
19451
19452 // src0 must be the same register as src1 or src2, even if the value is
19453 // undefined, so make sure we don't violate this constraint.
19454 if (Src0.isMachineOpcode() &&
19455 Src0.getMachineOpcode() == AMDGPU::IMPLICIT_DEF) {
19456 if (Src1.isMachineOpcode() &&
19457 Src1.getMachineOpcode() != AMDGPU::IMPLICIT_DEF)
19458 Src0 = Src1;
19459 else if (Src2.isMachineOpcode() &&
19460 Src2.getMachineOpcode() != AMDGPU::IMPLICIT_DEF)
19461 Src0 = Src2;
19462 else {
19463 assert(Src1.getMachineOpcode() == AMDGPU::IMPLICIT_DEF);
19464 Src0 = UndefReg;
19465 Src1 = UndefReg;
19466 }
19467 } else
19468 break;
19469
19470 SmallVector<SDValue, 9> Ops(Node->ops());
19471 Ops[1] = Src0;
19472 Ops[3] = Src1;
19473 Ops[5] = Src2;
19474 Ops.push_back(Elt: ImpDef.getValue(R: 1));
19475 return DAG.getMachineNode(Opcode, dl: SDLoc(Node), VTs: Node->getVTList(), Ops);
19476 }
19477 default:
19478 break;
19479 }
19480
19481 return Node;
19482}
19483
19484// Any MIMG instructions that use tfe or lwe require an initialization of the
19485// result register that will be written in the case of a memory access failure.
19486// The required code is also added to tie this init code to the result of the
19487// img instruction.
19488void SITargetLowering::AddMemOpInit(MachineInstr &MI) const {
19489 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19490 const SIRegisterInfo &TRI = TII->getRegisterInfo();
19491 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
19492 MachineBasicBlock &MBB = *MI.getParent();
19493
19494 int DstIdx =
19495 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdata);
19496 unsigned InitIdx = 0;
19497
19498 if (TII->isImage(MI)) {
19499 MachineOperand *TFE = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe);
19500 MachineOperand *LWE = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::lwe);
19501 MachineOperand *D16 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::d16);
19502
19503 if (!TFE && !LWE) // intersect_ray
19504 return;
19505
19506 unsigned TFEVal = TFE ? TFE->getImm() : 0;
19507 unsigned LWEVal = LWE ? LWE->getImm() : 0;
19508 unsigned D16Val = D16 ? D16->getImm() : 0;
19509
19510 if (!TFEVal && !LWEVal)
19511 return;
19512
19513 // At least one of TFE or LWE are non-zero
19514 // We have to insert a suitable initialization of the result value and
19515 // tie this to the dest of the image instruction.
19516
19517 // Calculate which dword we have to initialize to 0.
19518 MachineOperand *MO_Dmask = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dmask);
19519
19520 // check that dmask operand is found.
19521 assert(MO_Dmask && "Expected dmask operand in instruction");
19522
19523 unsigned dmask = MO_Dmask->getImm();
19524 // Determine the number of active lanes taking into account the
19525 // Gather4 special case
19526 unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(Value: dmask);
19527
19528 bool Packed = !Subtarget->hasUnpackedD16VMem();
19529
19530 InitIdx = D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1;
19531
19532 // Abandon attempt if the dst size isn't large enough
19533 // - this is in fact an error but this is picked up elsewhere and
19534 // reported correctly.
19535 const TargetRegisterClass *DstRC = TII->getRegClass(MCID: MI.getDesc(), OpNum: DstIdx);
19536
19537 uint32_t DstSize = TRI.getRegSizeInBits(RC: *DstRC) / 32;
19538 if (DstSize < InitIdx)
19539 return;
19540 } else if (TII->isMUBUF(MI) && AMDGPU::getMUBUFTfe(Opc: MI.getOpcode())) {
19541 const TargetRegisterClass *DstRC = TII->getRegClass(MCID: MI.getDesc(), OpNum: DstIdx);
19542 InitIdx = TRI.getRegSizeInBits(RC: *DstRC) / 32;
19543 } else {
19544 return;
19545 }
19546
19547 const DebugLoc &DL = MI.getDebugLoc();
19548
19549 // Create a register for the initialization value.
19550 Register PrevDst = MRI.cloneVirtualRegister(VReg: MI.getOperand(i: DstIdx).getReg());
19551 unsigned NewDst = 0; // Final initialized value will be in here
19552
19553 // If PRTStrictNull feature is enabled (the default) then initialize
19554 // all the result registers to 0, otherwise just the error indication
19555 // register (VGPRn+1)
19556 unsigned SizeLeft = Subtarget->usePRTStrictNull() ? InitIdx : 1;
19557 unsigned CurrIdx = Subtarget->usePRTStrictNull() ? 0 : (InitIdx - 1);
19558
19559 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: PrevDst);
19560 for (; SizeLeft; SizeLeft--, CurrIdx++) {
19561 NewDst = MRI.createVirtualRegister(RegClass: TII->getOpRegClass(MI, OpNo: DstIdx));
19562 // Initialize dword
19563 Register SubReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
19564 // clang-format off
19565 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: SubReg)
19566 .addImm(Val: 0);
19567 // clang-format on
19568 // Insert into the super-reg
19569 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: NewDst)
19570 .addReg(RegNo: PrevDst)
19571 .addReg(RegNo: SubReg)
19572 .addImm(Val: SIRegisterInfo::getSubRegFromChannel(Channel: CurrIdx));
19573
19574 PrevDst = NewDst;
19575 }
19576
19577 // Add as an implicit operand
19578 MI.addOperand(Op: MachineOperand::CreateReg(Reg: NewDst, isDef: false, isImp: true));
19579
19580 // Tie the just added implicit operand to the dst
19581 MI.tieOperands(DefIdx: DstIdx, UseIdx: MI.getNumOperands() - 1);
19582}
19583
19584/// Assign the register class depending on the number of
19585/// bits set in the writemask
19586void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
19587 SDNode *Node) const {
19588 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19589
19590 MachineFunction *MF = MI.getMF();
19591 MachineRegisterInfo &MRI = MF->getRegInfo();
19592
19593 if (TII->isVOP3(Opcode: MI.getOpcode())) {
19594 // Make sure constant bus requirements are respected.
19595 TII->legalizeOperandsVOP3(MRI, MI);
19596
19597 if (TII->isMAI(MI)) {
19598 // The ordinary src0, src1, src2 were legalized above.
19599 //
19600 // We have to also legalize the appended v_mfma_ld_scale_b32 operands,
19601 // as a separate instruction.
19602 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
19603 Name: AMDGPU::OpName::scale_src0);
19604 if (Src0Idx != -1) {
19605 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
19606 Name: AMDGPU::OpName::scale_src1);
19607 if (TII->usesConstantBus(MRI, MI, OpIdx: Src0Idx) &&
19608 TII->usesConstantBus(MRI, MI, OpIdx: Src1Idx))
19609 TII->legalizeOpWithMove(MI, OpIdx: Src1Idx);
19610 }
19611 }
19612
19613 return;
19614 }
19615
19616 if (TII->isImage(MI))
19617 TII->enforceOperandRCAlignment(MI, OpName: AMDGPU::OpName::vaddr);
19618}
19619
19620static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL,
19621 uint64_t Val) {
19622 SDValue K = DAG.getTargetConstant(Val, DL, VT: MVT::i32);
19623 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: K), 0);
19624}
19625
19626MachineSDNode *SITargetLowering::wrapAddr64Rsrc(SelectionDAG &DAG,
19627 const SDLoc &DL,
19628 SDValue Ptr) const {
19629 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19630
19631 // Build the half of the subregister with the constants before building the
19632 // full 128-bit register. If we are building multiple resource descriptors,
19633 // this will allow CSEing of the 2-component register.
19634 const SDValue Ops0[] = {
19635 DAG.getTargetConstant(Val: AMDGPU::SGPR_64RegClassID, DL, VT: MVT::i32),
19636 buildSMovImm32(DAG, DL, Val: 0),
19637 DAG.getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32),
19638 buildSMovImm32(DAG, DL, Val: TII->getDefaultRsrcDataFormat() >> 32),
19639 DAG.getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32)};
19640
19641 SDValue SubRegHi = SDValue(
19642 DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v2i32, Ops: Ops0), 0);
19643
19644 // Combine the constants and the pointer.
19645 const SDValue Ops1[] = {
19646 DAG.getTargetConstant(Val: AMDGPU::SGPR_128RegClassID, DL, VT: MVT::i32), Ptr,
19647 DAG.getTargetConstant(Val: AMDGPU::sub0_sub1, DL, VT: MVT::i32), SubRegHi,
19648 DAG.getTargetConstant(Val: AMDGPU::sub2_sub3, DL, VT: MVT::i32)};
19649
19650 return DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v4i32, Ops: Ops1);
19651}
19652
19653/// Return a resource descriptor with the 'Add TID' bit enabled
19654/// The TID (Thread ID) is multiplied by the stride value (bits [61:48]
19655/// of the resource descriptor) to create an offset, which is added to
19656/// the resource pointer.
19657MachineSDNode *SITargetLowering::buildRSRC(SelectionDAG &DAG, const SDLoc &DL,
19658 SDValue Ptr, uint32_t RsrcDword1,
19659 uint64_t RsrcDword2And3) const {
19660 SDValue PtrLo = DAG.getTargetExtractSubreg(SRIdx: AMDGPU::sub0, DL, VT: MVT::i32, Operand: Ptr);
19661 SDValue PtrHi = DAG.getTargetExtractSubreg(SRIdx: AMDGPU::sub1, DL, VT: MVT::i32, Operand: Ptr);
19662 if (RsrcDword1) {
19663 PtrHi = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: PtrHi,
19664 N2: DAG.getConstant(Val: RsrcDword1, DL, VT: MVT::i32));
19665 }
19666
19667 SDValue DataLo =
19668 buildSMovImm32(DAG, DL, Val: RsrcDword2And3 & UINT64_C(0xFFFFFFFF));
19669 SDValue DataHi = buildSMovImm32(DAG, DL, Val: RsrcDword2And3 >> 32);
19670
19671 const SDValue Ops[] = {
19672 DAG.getTargetConstant(Val: AMDGPU::SGPR_128RegClassID, DL, VT: MVT::i32),
19673 PtrLo,
19674 DAG.getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32),
19675 PtrHi,
19676 DAG.getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32),
19677 DataLo,
19678 DAG.getTargetConstant(Val: AMDGPU::sub2, DL, VT: MVT::i32),
19679 DataHi,
19680 DAG.getTargetConstant(Val: AMDGPU::sub3, DL, VT: MVT::i32)};
19681
19682 return DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v4i32, Ops);
19683}
19684
19685//===----------------------------------------------------------------------===//
19686// SI Inline Assembly Support
19687//===----------------------------------------------------------------------===//
19688
19689std::pair<unsigned, const TargetRegisterClass *>
19690SITargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI_,
19691 StringRef Constraint,
19692 MVT VT) const {
19693 const SIRegisterInfo *TRI = static_cast<const SIRegisterInfo *>(TRI_);
19694
19695 const TargetRegisterClass *RC = nullptr;
19696 if (Constraint.size() == 1) {
19697 // Check if we cannot determine the bit size of the given value type. This
19698 // can happen, for example, in this situation where we have an empty struct
19699 // (size 0): `call void asm "", "v"({} poison)`-
19700 if (VT == MVT::Other)
19701 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
19702 const unsigned BitWidth = VT.getSizeInBits();
19703 switch (Constraint[0]) {
19704 default:
19705 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
19706 case 's':
19707 case 'r':
19708 switch (BitWidth) {
19709 case 16:
19710 RC = &AMDGPU::SReg_32RegClass;
19711 break;
19712 case 64:
19713 RC = &AMDGPU::SGPR_64RegClass;
19714 break;
19715 default:
19716 RC = SIRegisterInfo::getSGPRClassForBitWidth(BitWidth);
19717 if (!RC)
19718 return std::pair(0U, nullptr);
19719 break;
19720 }
19721 break;
19722 case 'v':
19723 switch (BitWidth) {
19724 case 1:
19725 return std::pair(0U, nullptr);
19726 case 16:
19727 RC = Subtarget->useRealTrue16Insts() ? &AMDGPU::VGPR_16RegClass
19728 : &AMDGPU::VGPR_32_Lo256RegClass;
19729 break;
19730 default:
19731 RC = Subtarget->has1024AddressableVGPRs()
19732 ? TRI->getAlignedLo256VGPRClassForBitWidth(BitWidth)
19733 : TRI->getVGPRClassForBitWidth(BitWidth);
19734 if (!RC)
19735 return std::pair(0U, nullptr);
19736 break;
19737 }
19738 break;
19739 case 'a':
19740 if (!Subtarget->hasMAIInsts())
19741 break;
19742 switch (BitWidth) {
19743 case 1:
19744 return std::pair(0U, nullptr);
19745 case 16:
19746 RC = &AMDGPU::AGPR_32RegClass;
19747 break;
19748 default:
19749 RC = TRI->getAGPRClassForBitWidth(BitWidth);
19750 if (!RC)
19751 return std::pair(0U, nullptr);
19752 break;
19753 }
19754 break;
19755 }
19756 } else if (Constraint == "VA" && Subtarget->hasGFX90AInsts()) {
19757 const unsigned BitWidth = VT.getSizeInBits();
19758 switch (BitWidth) {
19759 case 16:
19760 RC = &AMDGPU::AV_32RegClass;
19761 break;
19762 default:
19763 RC = TRI->getVectorSuperClassForBitWidth(BitWidth);
19764 if (!RC)
19765 return std::pair(0U, nullptr);
19766 break;
19767 }
19768 }
19769
19770 // We actually support i128, i16 and f16 as inline parameters
19771 // even if they are not reported as legal
19772 if (RC && (isTypeLegal(VT) || VT.SimpleTy == MVT::i128 ||
19773 VT.SimpleTy == MVT::i16 || VT.SimpleTy == MVT::f16))
19774 return std::pair(0U, RC);
19775
19776 auto [Kind, Idx, NumRegs] = AMDGPU::parseAsmConstraintPhysReg(Constraint);
19777 if (Kind != '\0') {
19778 if (Kind == 'v') {
19779 RC = &AMDGPU::VGPR_32_Lo256RegClass;
19780 } else if (Kind == 's') {
19781 RC = &AMDGPU::SGPR_32RegClass;
19782 } else if (Kind == 'a') {
19783 RC = &AMDGPU::AGPR_32RegClass;
19784 }
19785
19786 if (RC) {
19787 if (NumRegs > 1) {
19788 if (Idx >= RC->getNumRegs() || Idx + NumRegs - 1 >= RC->getNumRegs())
19789 return std::pair(0U, nullptr);
19790
19791 uint32_t Width = NumRegs * 32;
19792 // Prohibit constraints for register ranges with a width that does not
19793 // match the required type.
19794 if (VT.SimpleTy != MVT::Other && Width != VT.getSizeInBits())
19795 return std::pair(0U, nullptr);
19796
19797 MCRegister Reg = RC->getRegister(i: Idx);
19798 if (SIRegisterInfo::isVGPRClass(RC))
19799 RC = TRI->getVGPRClassForBitWidth(BitWidth: Width);
19800 else if (SIRegisterInfo::isSGPRClass(RC))
19801 RC = TRI->getSGPRClassForBitWidth(BitWidth: Width);
19802 else if (SIRegisterInfo::isAGPRClass(RC))
19803 RC = TRI->getAGPRClassForBitWidth(BitWidth: Width);
19804 if (RC) {
19805 Reg = TRI->getMatchingSuperReg(Reg, SubIdx: AMDGPU::sub0, RC);
19806 if (!Reg) {
19807 // The register class does not contain the requested register,
19808 // e.g., because it is an SGPR pair that would violate alignment
19809 // requirements.
19810 return std::pair(0U, nullptr);
19811 }
19812 return std::pair(Reg, RC);
19813 }
19814 }
19815
19816 // Reject types that do not fit a single 32-bit register: any scalar wider
19817 // than 32 bits, or a vector that is not exactly 32 bits.
19818 if (VT.SimpleTy != MVT::Other &&
19819 (VT.getSizeInBits() > 32 ||
19820 (VT.isVector() && VT.getSizeInBits() != 32)))
19821 return std::pair(0U, nullptr);
19822 if (RC && Idx < RC->getNumRegs())
19823 return std::pair(RC->getRegister(i: Idx), RC);
19824 return std::pair(0U, nullptr);
19825 }
19826 }
19827
19828 auto Ret = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
19829 if (Ret.first)
19830 Ret.second = TRI->getPhysRegBaseClass(Reg: Ret.first);
19831
19832 return Ret;
19833}
19834
19835static bool isImmConstraint(StringRef Constraint) {
19836 if (Constraint.size() == 1) {
19837 switch (Constraint[0]) {
19838 default:
19839 break;
19840 case 'I':
19841 case 'J':
19842 case 'A':
19843 case 'B':
19844 case 'C':
19845 return true;
19846 }
19847 } else if (Constraint == "DA" || Constraint == "DB") {
19848 return true;
19849 }
19850 return false;
19851}
19852
19853SITargetLowering::ConstraintType
19854SITargetLowering::getConstraintType(StringRef Constraint) const {
19855 if (Constraint.size() == 1) {
19856 switch (Constraint[0]) {
19857 default:
19858 break;
19859 case 's':
19860 case 'v':
19861 case 'a':
19862 return C_RegisterClass;
19863 }
19864 } else if (Constraint.size() == 2) {
19865 if (Constraint == "VA")
19866 return C_RegisterClass;
19867 }
19868 if (isImmConstraint(Constraint)) {
19869 return C_Other;
19870 }
19871 return TargetLowering::getConstraintType(Constraint);
19872}
19873
19874static uint64_t clearUnusedBits(uint64_t Val, unsigned Size) {
19875 if (!AMDGPU::isInlinableIntLiteral(Literal: Val)) {
19876 Val = Val & maskTrailingOnes<uint64_t>(N: Size);
19877 }
19878 return Val;
19879}
19880
19881void SITargetLowering::LowerAsmOperandForConstraint(SDValue Op,
19882 StringRef Constraint,
19883 std::vector<SDValue> &Ops,
19884 SelectionDAG &DAG) const {
19885 if (isImmConstraint(Constraint)) {
19886 uint64_t Val;
19887 if (getAsmOperandConstVal(Op, Val) &&
19888 checkAsmConstraintVal(Op, Constraint, Val)) {
19889 Val = clearUnusedBits(Val, Size: Op.getScalarValueSizeInBits());
19890 Ops.push_back(x: DAG.getTargetConstant(Val, DL: SDLoc(Op), VT: MVT::i64));
19891 }
19892 } else {
19893 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
19894 }
19895}
19896
19897bool SITargetLowering::getAsmOperandConstVal(SDValue Op, uint64_t &Val) const {
19898 unsigned Size = Op.getScalarValueSizeInBits();
19899 if (Size > 64)
19900 return false;
19901
19902 if (Size == 16 && !Subtarget->has16BitInsts())
19903 return false;
19904
19905 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
19906 Val = C->getSExtValue();
19907 return true;
19908 }
19909 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
19910 Val = C->getValueAPF().bitcastToAPInt().getSExtValue();
19911 return true;
19912 }
19913 if (BuildVectorSDNode *V = dyn_cast<BuildVectorSDNode>(Val&: Op)) {
19914 if (Size != 16 || Op.getNumOperands() != 2)
19915 return false;
19916 if (Op.getOperand(i: 0).isUndef() || Op.getOperand(i: 1).isUndef())
19917 return false;
19918 if (ConstantSDNode *C = V->getConstantSplatNode()) {
19919 Val = C->getSExtValue();
19920 return true;
19921 }
19922 if (ConstantFPSDNode *C = V->getConstantFPSplatNode()) {
19923 Val = C->getValueAPF().bitcastToAPInt().getSExtValue();
19924 return true;
19925 }
19926 }
19927
19928 return false;
19929}
19930
19931bool SITargetLowering::checkAsmConstraintVal(SDValue Op, StringRef Constraint,
19932 uint64_t Val) const {
19933 if (Constraint.size() == 1) {
19934 switch (Constraint[0]) {
19935 case 'I':
19936 return AMDGPU::isInlinableIntLiteral(Literal: Val);
19937 case 'J':
19938 return isInt<16>(x: Val);
19939 case 'A':
19940 return checkAsmConstraintValA(Op, Val);
19941 case 'B':
19942 return isInt<32>(x: Val);
19943 case 'C':
19944 return isUInt<32>(x: clearUnusedBits(Val, Size: Op.getScalarValueSizeInBits())) ||
19945 AMDGPU::isInlinableIntLiteral(Literal: Val);
19946 default:
19947 break;
19948 }
19949 } else if (Constraint.size() == 2) {
19950 if (Constraint == "DA") {
19951 int64_t HiBits = static_cast<int32_t>(Val >> 32);
19952 int64_t LoBits = static_cast<int32_t>(Val);
19953 return checkAsmConstraintValA(Op, Val: HiBits, MaxSize: 32) &&
19954 checkAsmConstraintValA(Op, Val: LoBits, MaxSize: 32);
19955 }
19956 if (Constraint == "DB") {
19957 return true;
19958 }
19959 }
19960 llvm_unreachable("Invalid asm constraint");
19961}
19962
19963bool SITargetLowering::checkAsmConstraintValA(SDValue Op, uint64_t Val,
19964 unsigned MaxSize) const {
19965 unsigned Size = std::min<unsigned>(a: Op.getScalarValueSizeInBits(), b: MaxSize);
19966 bool HasInv2Pi = Subtarget->hasInv2PiInlineImm();
19967 if (Size == 16) {
19968 MVT VT = Op.getSimpleValueType();
19969 switch (VT.SimpleTy) {
19970 default:
19971 return false;
19972 case MVT::i16:
19973 return AMDGPU::isInlinableLiteralI16(Literal: Val, HasInv2Pi);
19974 case MVT::f16:
19975 return AMDGPU::isInlinableLiteralFP16(Literal: Val, HasInv2Pi);
19976 case MVT::bf16:
19977 return AMDGPU::isInlinableLiteralBF16(Literal: Val, HasInv2Pi);
19978 case MVT::v2i16:
19979 return AMDGPU::getInlineEncodingV2I16(Literal: Val).has_value();
19980 case MVT::v2f16:
19981 return AMDGPU::getInlineEncodingV2F16(Literal: Val).has_value();
19982 case MVT::v2bf16:
19983 return AMDGPU::getInlineEncodingV2BF16(Literal: Val).has_value();
19984 }
19985 }
19986 if ((Size == 32 && AMDGPU::isInlinableLiteral32(Literal: Val, HasInv2Pi)) ||
19987 (Size == 64 && AMDGPU::isInlinableLiteral64(Literal: Val, HasInv2Pi)))
19988 return true;
19989 return false;
19990}
19991
19992static int getAlignedAGPRClassID(unsigned UnalignedClassID) {
19993 switch (UnalignedClassID) {
19994 case AMDGPU::VReg_64RegClassID:
19995 return AMDGPU::VReg_64_Align2RegClassID;
19996 case AMDGPU::VReg_96RegClassID:
19997 return AMDGPU::VReg_96_Align2RegClassID;
19998 case AMDGPU::VReg_128RegClassID:
19999 return AMDGPU::VReg_128_Align2RegClassID;
20000 case AMDGPU::VReg_160RegClassID:
20001 return AMDGPU::VReg_160_Align2RegClassID;
20002 case AMDGPU::VReg_192RegClassID:
20003 return AMDGPU::VReg_192_Align2RegClassID;
20004 case AMDGPU::VReg_224RegClassID:
20005 return AMDGPU::VReg_224_Align2RegClassID;
20006 case AMDGPU::VReg_256RegClassID:
20007 return AMDGPU::VReg_256_Align2RegClassID;
20008 case AMDGPU::VReg_288RegClassID:
20009 return AMDGPU::VReg_288_Align2RegClassID;
20010 case AMDGPU::VReg_320RegClassID:
20011 return AMDGPU::VReg_320_Align2RegClassID;
20012 case AMDGPU::VReg_352RegClassID:
20013 return AMDGPU::VReg_352_Align2RegClassID;
20014 case AMDGPU::VReg_384RegClassID:
20015 return AMDGPU::VReg_384_Align2RegClassID;
20016 case AMDGPU::VReg_512RegClassID:
20017 return AMDGPU::VReg_512_Align2RegClassID;
20018 case AMDGPU::VReg_1024RegClassID:
20019 return AMDGPU::VReg_1024_Align2RegClassID;
20020 case AMDGPU::AReg_64RegClassID:
20021 return AMDGPU::AReg_64_Align2RegClassID;
20022 case AMDGPU::AReg_96RegClassID:
20023 return AMDGPU::AReg_96_Align2RegClassID;
20024 case AMDGPU::AReg_128RegClassID:
20025 return AMDGPU::AReg_128_Align2RegClassID;
20026 case AMDGPU::AReg_160RegClassID:
20027 return AMDGPU::AReg_160_Align2RegClassID;
20028 case AMDGPU::AReg_192RegClassID:
20029 return AMDGPU::AReg_192_Align2RegClassID;
20030 case AMDGPU::AReg_256RegClassID:
20031 return AMDGPU::AReg_256_Align2RegClassID;
20032 case AMDGPU::AReg_512RegClassID:
20033 return AMDGPU::AReg_512_Align2RegClassID;
20034 case AMDGPU::AReg_1024RegClassID:
20035 return AMDGPU::AReg_1024_Align2RegClassID;
20036 default:
20037 return -1;
20038 }
20039}
20040
20041// Figure out which registers should be reserved for stack access. Only after
20042// the function is legalized do we know all of the non-spill stack objects or if
20043// calls are present.
20044void SITargetLowering::finalizeLowering(MachineFunction &MF) const {
20045 MachineRegisterInfo &MRI = MF.getRegInfo();
20046 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
20047 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
20048 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
20049 const SIInstrInfo *TII = ST.getInstrInfo();
20050
20051 if (Info->isEntryFunction()) {
20052 // Callable functions have fixed registers used for stack access.
20053 reservePrivateMemoryRegs(TM: getTargetMachine(), MF, TRI: *TRI, Info&: *Info);
20054 }
20055
20056 // TODO: Move this logic to getReservedRegs()
20057 // Reserve the SGPR(s) to save/restore EXEC for WWM spill/copy handling.
20058 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF);
20059 Register SReg = ST.isWave32()
20060 ? AMDGPU::SGPR_32RegClass.getRegister(i: MaxNumSGPRs - 1)
20061 : TRI->getAlignedHighSGPRForRC(MF, /*Align=*/2,
20062 RC: &AMDGPU::SGPR_64RegClass);
20063 Info->setSGPRForEXECCopy(SReg);
20064
20065 assert(!TRI->isSubRegister(Info->getScratchRSrcReg(),
20066 Info->getStackPtrOffsetReg()));
20067 if (Info->getStackPtrOffsetReg() != AMDGPU::SP_REG)
20068 MRI.replaceRegWith(FromReg: AMDGPU::SP_REG, ToReg: Info->getStackPtrOffsetReg());
20069
20070 // We need to worry about replacing the default register with itself in case
20071 // of MIR testcases missing the MFI.
20072 if (Info->getScratchRSrcReg() != AMDGPU::PRIVATE_RSRC_REG)
20073 MRI.replaceRegWith(FromReg: AMDGPU::PRIVATE_RSRC_REG, ToReg: Info->getScratchRSrcReg());
20074
20075 if (Info->getFrameOffsetReg() != AMDGPU::FP_REG)
20076 MRI.replaceRegWith(FromReg: AMDGPU::FP_REG, ToReg: Info->getFrameOffsetReg());
20077
20078 Info->limitOccupancy(MF);
20079
20080 if (ST.isWave32() && !MF.empty()) {
20081 for (auto &MBB : MF) {
20082 for (auto &MI : MBB) {
20083 TII->fixImplicitOperands(MI);
20084 }
20085 }
20086 }
20087
20088 // FIXME: This is a hack to fixup AGPR classes to use the properly aligned
20089 // classes if required. Ideally the register class constraints would differ
20090 // per-subtarget, but there's no easy way to achieve that right now. This is
20091 // not a problem for VGPRs because the correctly aligned VGPR class is implied
20092 // from using them as the register class for legal types.
20093 if (ST.needsAlignedVGPRs()) {
20094 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
20095 const Register Reg = Register::index2VirtReg(Index: I);
20096 const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg);
20097 if (!RC)
20098 continue;
20099 int NewClassID = getAlignedAGPRClassID(UnalignedClassID: RC->getID());
20100 if (NewClassID != -1)
20101 MRI.setRegClass(Reg, RC: TRI->getRegClass(i: NewClassID));
20102 }
20103 }
20104
20105 TargetLoweringBase::finalizeLowering(MF);
20106}
20107
20108void SITargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
20109 KnownBits &Known,
20110 const APInt &DemandedElts,
20111 const SelectionDAG &DAG,
20112 unsigned Depth) const {
20113 Known.resetAll();
20114 unsigned Opc = Op.getOpcode();
20115 switch (Opc) {
20116 case ISD::INTRINSIC_WO_CHAIN: {
20117 unsigned IID = Op.getConstantOperandVal(i: 0);
20118 switch (IID) {
20119 case Intrinsic::amdgcn_mbcnt_lo:
20120 case Intrinsic::amdgcn_mbcnt_hi: {
20121 const GCNSubtarget &ST =
20122 DAG.getMachineFunction().getSubtarget<GCNSubtarget>();
20123 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
20124 // most 31 + src1.
20125 Known.Zero.setBitsFrom(
20126 IID == Intrinsic::amdgcn_mbcnt_lo ? ST.getWavefrontSizeLog2() : 5);
20127 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
20128 Known = KnownBits::add(LHS: Known, RHS: Known2);
20129 return;
20130 }
20131 }
20132 break;
20133 }
20134 }
20135 return AMDGPUTargetLowering::computeKnownBitsForTargetNode(
20136 Op, Known, DemandedElts, DAG, Depth);
20137}
20138
20139void SITargetLowering::computeKnownBitsForStackObjectPointer(
20140 KnownBits &Known, const MachineFunction &MF, Align Alignment) const {
20141 TargetLowering::computeKnownBitsForStackObjectPointer(Known, MF, Alignment);
20142
20143 // Set the high bits to zero based on the maximum allowed scratch size per
20144 // wave. We can't use vaddr in MUBUF instructions if we don't know the address
20145 // calculation won't overflow, so assume the sign bit is never set.
20146 Known.Zero.setHighBits(getSubtarget()->getKnownHighZeroBitsForFrameIndex());
20147}
20148
20149static void knownBitsForWorkitemID(const GCNSubtarget &ST,
20150 GISelValueTracking &VT, KnownBits &Known,
20151 unsigned Dim) {
20152 unsigned MaxValue =
20153 ST.getMaxWorkitemID(Kernel: VT.getMachineFunction().getFunction(), Dimension: Dim);
20154 Known.Zero.setHighBits(llvm::countl_zero(Val: MaxValue));
20155}
20156
20157static void knownBitsForSBFE(const MachineInstr &MI, GISelValueTracking &VT,
20158 KnownBits &Known, const APInt &DemandedElts,
20159 unsigned BFEWidth, bool SExt, unsigned Depth) {
20160 const MachineRegisterInfo &MRI = VT.getMachineFunction().getRegInfo();
20161 const MachineOperand &Src1 = MI.getOperand(i: 2);
20162
20163 unsigned Src1Cst = 0;
20164 if (Src1.isImm()) {
20165 Src1Cst = Src1.getImm();
20166 } else if (Src1.isReg()) {
20167 auto Cst = getIConstantVRegValWithLookThrough(VReg: Src1.getReg(), MRI);
20168 if (!Cst)
20169 return;
20170 Src1Cst = Cst->Value.getZExtValue();
20171 } else {
20172 return;
20173 }
20174
20175 // Offset is at bits [4:0] for 32 bit, [5:0] for 64 bit.
20176 // Width is always [22:16].
20177 const unsigned Offset =
20178 Src1Cst & maskTrailingOnes<unsigned>(N: (BFEWidth == 32) ? 5 : 6);
20179 const unsigned Width = (Src1Cst >> 16) & maskTrailingOnes<unsigned>(N: 6);
20180
20181 if (Width >= BFEWidth) // Ill-formed.
20182 return;
20183
20184 VT.computeKnownBitsImpl(R: MI.getOperand(i: 1).getReg(), Known, DemandedElts,
20185 Depth: Depth + 1);
20186
20187 Known = Known.extractBits(NumBits: Width, BitPosition: Offset);
20188
20189 if (SExt)
20190 Known = Known.sext(BitWidth: BFEWidth);
20191 else
20192 Known = Known.zext(BitWidth: BFEWidth);
20193}
20194
20195void SITargetLowering::computeKnownBitsForTargetInstr(
20196 GISelValueTracking &VT, Register R, KnownBits &Known,
20197 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
20198 unsigned Depth) const {
20199 Known.resetAll();
20200 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
20201 switch (MI->getOpcode()) {
20202 case AMDGPU::S_BFE_I32:
20203 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 32,
20204 /*SExt=*/true, Depth);
20205 case AMDGPU::S_BFE_U32:
20206 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 32,
20207 /*SExt=*/false, Depth);
20208 case AMDGPU::S_BFE_I64:
20209 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 64,
20210 /*SExt=*/true, Depth);
20211 case AMDGPU::S_BFE_U64:
20212 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 64,
20213 /*SExt=*/false, Depth);
20214 case AMDGPU::G_INTRINSIC:
20215 case AMDGPU::G_INTRINSIC_CONVERGENT: {
20216 Intrinsic::ID IID = cast<GIntrinsic>(Val: MI)->getIntrinsicID();
20217 switch (IID) {
20218 case Intrinsic::amdgcn_workitem_id_x:
20219 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 0);
20220 break;
20221 case Intrinsic::amdgcn_workitem_id_y:
20222 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 1);
20223 break;
20224 case Intrinsic::amdgcn_workitem_id_z:
20225 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 2);
20226 break;
20227 case Intrinsic::amdgcn_mbcnt_lo:
20228 case Intrinsic::amdgcn_mbcnt_hi: {
20229 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
20230 // most 31 + src1.
20231 Known.Zero.setBitsFrom(IID == Intrinsic::amdgcn_mbcnt_lo
20232 ? getSubtarget()->getWavefrontSizeLog2()
20233 : 5);
20234 KnownBits Known2;
20235 VT.computeKnownBitsImpl(R: MI->getOperand(i: 3).getReg(), Known&: Known2, DemandedElts,
20236 Depth: Depth + 1);
20237 Known = KnownBits::add(LHS: Known, RHS: Known2);
20238 break;
20239 }
20240 case Intrinsic::amdgcn_groupstaticsize: {
20241 // We can report everything over the maximum size as 0. We can't report
20242 // based on the actual size because we don't know if it's accurate or not
20243 // at any given point.
20244 Known.Zero.setHighBits(
20245 llvm::countl_zero(Val: getSubtarget()->getAddressableLocalMemorySize()));
20246 break;
20247 }
20248 }
20249 break;
20250 }
20251 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
20252 Known.Zero.setHighBits(24);
20253 break;
20254 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
20255 Known.Zero.setHighBits(16);
20256 break;
20257 case AMDGPU::G_AMDGPU_COPY_SCC_VCC:
20258 // G_AMDGPU_COPY_SCC_VCC converts a uniform boolean in VCC to SGPR s32,
20259 // producing exactly 0 or 1.
20260 Known.Zero.setHighBits(Known.getBitWidth() - 1);
20261 break;
20262 case AMDGPU::G_AMDGPU_SMED3:
20263 case AMDGPU::G_AMDGPU_UMED3: {
20264 auto [Dst, Src0, Src1, Src2] = MI->getFirst4Regs();
20265
20266 KnownBits Known2;
20267 VT.computeKnownBitsImpl(R: Src2, Known&: Known2, DemandedElts, Depth: Depth + 1);
20268 if (Known2.isUnknown())
20269 break;
20270
20271 KnownBits Known1;
20272 VT.computeKnownBitsImpl(R: Src1, Known&: Known1, DemandedElts, Depth: Depth + 1);
20273 if (Known1.isUnknown())
20274 break;
20275
20276 KnownBits Known0;
20277 VT.computeKnownBitsImpl(R: Src0, Known&: Known0, DemandedElts, Depth: Depth + 1);
20278 if (Known0.isUnknown())
20279 break;
20280
20281 // TODO: Handle LeadZero/LeadOne from UMIN/UMAX handling.
20282 Known.Zero = Known0.Zero & Known1.Zero & Known2.Zero;
20283 Known.One = Known0.One & Known1.One & Known2.One;
20284 break;
20285 }
20286 }
20287}
20288
20289Align SITargetLowering::computeKnownAlignForTargetInstr(
20290 GISelValueTracking &VT, Register R, const MachineRegisterInfo &MRI,
20291 unsigned Depth) const {
20292 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
20293 if (auto *GI = dyn_cast<GIntrinsic>(Val: MI)) {
20294 // FIXME: Can this move to generic code? What about the case where the call
20295 // site specifies a lower alignment?
20296 Intrinsic::ID IID = GI->getIntrinsicID();
20297 LLVMContext &Ctx = VT.getMachineFunction().getFunction().getContext();
20298 AttributeList Attrs =
20299 Intrinsic::getAttributes(C&: Ctx, id: IID, FT: Intrinsic::getType(Context&: Ctx, id: IID));
20300 if (MaybeAlign RetAlign = Attrs.getRetAlignment())
20301 return *RetAlign;
20302 }
20303 return Align(1);
20304}
20305
20306Align SITargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
20307 const Align PrefAlign = TargetLowering::getPrefLoopAlignment(ML);
20308 const Align CacheLineAlign = Align(64);
20309
20310 // GFX950: Prevent an 8-byte instruction at loop header from being split by
20311 // the 32-byte instruction fetch window boundary. This avoids a significant
20312 // fetch delay after backward branch. We use 32-byte alignment with max
20313 // padding of 4 bytes (one s_nop), see getMaxPermittedBytesForAlignment().
20314 if (ML && !DisableLoopAlignment &&
20315 getSubtarget()->hasLoopHeadInstSplitSensitivity()) {
20316 const MachineBasicBlock *Header = ML->getHeader();
20317 // Respect user-specified or previously set alignment.
20318 if (Header->getAlignment() != PrefAlign)
20319 return Header->getAlignment();
20320 if (needsFetchWindowAlignment(MBB: *Header))
20321 return Align(32);
20322 }
20323
20324 // Pre-GFX10 target did not benefit from loop alignment
20325 if (!ML || DisableLoopAlignment || !getSubtarget()->hasInstPrefetch() ||
20326 getSubtarget()->hasInstFwdPrefetchBug())
20327 return PrefAlign;
20328
20329 // On GFX10 I$ is 4 x 64 bytes cache lines.
20330 // By default prefetcher keeps one cache line behind and reads two ahead.
20331 // We can modify it with S_INST_PREFETCH for larger loops to have two lines
20332 // behind and one ahead.
20333 // Therefor we can benefit from aligning loop headers if loop fits 192 bytes.
20334 // If loop fits 64 bytes it always spans no more than two cache lines and
20335 // does not need an alignment.
20336 // Else if loop is less or equal 128 bytes we do not need to modify prefetch,
20337 // Else if loop is less or equal 192 bytes we need two lines behind.
20338
20339 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
20340 const MachineBasicBlock *Header = ML->getHeader();
20341 if (Header->getAlignment() != PrefAlign)
20342 return Header->getAlignment(); // Already processed.
20343
20344 unsigned LoopSize = 0;
20345 for (const MachineBasicBlock *MBB : ML->blocks()) {
20346 // If inner loop block is aligned assume in average half of the alignment
20347 // size to be added as nops.
20348 if (MBB != Header)
20349 LoopSize += MBB->getAlignment().value() / 2;
20350
20351 for (const MachineInstr &MI : *MBB) {
20352 LoopSize += TII->getInstSizeInBytes(MI);
20353 if (LoopSize > 192)
20354 return PrefAlign;
20355 }
20356 }
20357
20358 if (LoopSize <= 64)
20359 return PrefAlign;
20360
20361 if (LoopSize <= 128)
20362 return CacheLineAlign;
20363
20364 // If any of parent loops is surrounded by prefetch instructions do not
20365 // insert new for inner loop, which would reset parent's settings.
20366 for (MachineLoop *P = ML->getParentLoop(); P; P = P->getParentLoop()) {
20367 if (MachineBasicBlock *Exit = P->getExitBlock()) {
20368 auto I = Exit->getFirstNonDebugInstr();
20369 if (I != Exit->end() && I->getOpcode() == AMDGPU::S_INST_PREFETCH)
20370 return CacheLineAlign;
20371 }
20372 }
20373
20374 MachineBasicBlock *Pre = ML->getLoopPreheader();
20375 MachineBasicBlock *Exit = ML->getExitBlock();
20376
20377 if (Pre && Exit) {
20378 auto PreTerm = Pre->getFirstTerminator();
20379 if (PreTerm == Pre->begin() ||
20380 std::prev(x: PreTerm)->getOpcode() != AMDGPU::S_INST_PREFETCH)
20381 BuildMI(BB&: *Pre, I: PreTerm, MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_INST_PREFETCH))
20382 .addImm(Val: 1); // prefetch 2 lines behind PC
20383
20384 auto ExitHead = Exit->getFirstNonDebugInstr();
20385 if (ExitHead == Exit->end() ||
20386 ExitHead->getOpcode() != AMDGPU::S_INST_PREFETCH)
20387 BuildMI(BB&: *Exit, I: ExitHead, MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_INST_PREFETCH))
20388 .addImm(Val: 2); // prefetch 1 line behind PC
20389 }
20390
20391 return CacheLineAlign;
20392}
20393
20394unsigned SITargetLowering::getMaxPermittedBytesForAlignment(
20395 MachineBasicBlock *MBB) const {
20396 // GFX950: Limit padding to 4 bytes (one s_nop) for blocks where an 8-byte
20397 // instruction could be split by the 32-byte fetch window boundary.
20398 // See getPrefLoopAlignment() for context.
20399 if (needsFetchWindowAlignment(MBB: *MBB))
20400 return 4;
20401 return TargetLowering::getMaxPermittedBytesForAlignment(MBB);
20402}
20403
20404bool SITargetLowering::needsFetchWindowAlignment(
20405 const MachineBasicBlock &MBB) const {
20406 if (!getSubtarget()->hasLoopHeadInstSplitSensitivity())
20407 return false;
20408 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
20409 for (const MachineInstr &MI : MBB) {
20410 if (MI.isMetaInstruction())
20411 continue;
20412 // Instructions larger than 4 bytes can be split by a 32-byte boundary.
20413 return TII->getInstSizeInBytes(MI) > 4;
20414 }
20415 return false;
20416}
20417
20418[[maybe_unused]]
20419static bool isCopyFromRegOfInlineAsm(const SDNode *N) {
20420 assert(N->getOpcode() == ISD::CopyFromReg);
20421 do {
20422 // Follow the chain until we find an INLINEASM node.
20423 N = N->getOperand(Num: 0).getNode();
20424 if (N->getOpcode() == ISD::INLINEASM || N->getOpcode() == ISD::INLINEASM_BR)
20425 return true;
20426 } while (N->getOpcode() == ISD::CopyFromReg);
20427 return false;
20428}
20429
20430bool SITargetLowering::isSDNodeSourceOfDivergence(const SDNode *N,
20431 FunctionLoweringInfo *FLI,
20432 UniformityInfo *UA) const {
20433 switch (N->getOpcode()) {
20434 case ISD::CopyFromReg: {
20435 const RegisterSDNode *R = cast<RegisterSDNode>(Val: N->getOperand(Num: 1));
20436 const MachineRegisterInfo &MRI = FLI->MF->getRegInfo();
20437 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
20438 Register Reg = R->getReg();
20439
20440 // FIXME: Why does this need to consider isLiveIn?
20441 if (Reg.isPhysical() || MRI.isLiveIn(Reg))
20442 return !TRI->isSGPRReg(MRI, Reg);
20443
20444 if (const Value *V = FLI->getValueFromVirtualReg(Vreg: R->getReg()))
20445 return UA->isDivergentAtDef(V);
20446
20447 assert(Reg == FLI->DemoteRegister || isCopyFromRegOfInlineAsm(N));
20448 return !TRI->isSGPRReg(MRI, Reg);
20449 }
20450 case ISD::LOAD: {
20451 const LoadSDNode *L = cast<LoadSDNode>(Val: N);
20452 unsigned AS = L->getAddressSpace();
20453 // A flat load may access private memory.
20454 return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS;
20455 }
20456 case ISD::CALLSEQ_END:
20457 return true;
20458 case ISD::INTRINSIC_WO_CHAIN:
20459 return AMDGPU::isIntrinsicSourceOfDivergence(IntrID: N->getConstantOperandVal(Num: 0));
20460 case ISD::INTRINSIC_W_CHAIN:
20461 return AMDGPU::isIntrinsicSourceOfDivergence(IntrID: N->getConstantOperandVal(Num: 1));
20462 case AMDGPUISD::ATOMIC_CMP_SWAP:
20463 case AMDGPUISD::BUFFER_ATOMIC_SWAP:
20464 case AMDGPUISD::BUFFER_ATOMIC_ADD:
20465 case AMDGPUISD::BUFFER_ATOMIC_SUB:
20466 case AMDGPUISD::BUFFER_ATOMIC_SMIN:
20467 case AMDGPUISD::BUFFER_ATOMIC_UMIN:
20468 case AMDGPUISD::BUFFER_ATOMIC_SMAX:
20469 case AMDGPUISD::BUFFER_ATOMIC_UMAX:
20470 case AMDGPUISD::BUFFER_ATOMIC_AND:
20471 case AMDGPUISD::BUFFER_ATOMIC_OR:
20472 case AMDGPUISD::BUFFER_ATOMIC_XOR:
20473 case AMDGPUISD::BUFFER_ATOMIC_INC:
20474 case AMDGPUISD::BUFFER_ATOMIC_DEC:
20475 case AMDGPUISD::BUFFER_ATOMIC_CMPSWAP:
20476 case AMDGPUISD::BUFFER_ATOMIC_FADD:
20477 case AMDGPUISD::BUFFER_ATOMIC_FMIN:
20478 case AMDGPUISD::BUFFER_ATOMIC_FMAX:
20479 // Target-specific read-modify-write atomics are sources of divergence.
20480 return true;
20481 default:
20482 if (auto *A = dyn_cast<AtomicSDNode>(Val: N)) {
20483 // Generic read-modify-write atomics are sources of divergence.
20484 return A->readMem() && A->writeMem();
20485 }
20486 return false;
20487 }
20488}
20489
20490bool SITargetLowering::denormalsEnabledForType(const SelectionDAG &DAG,
20491 EVT VT) const {
20492 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
20493 case MVT::f32:
20494 return !denormalModeIsFlushAllF32(MF: DAG.getMachineFunction());
20495 case MVT::f64:
20496 case MVT::f16:
20497 return !denormalModeIsFlushAllF64F16(MF: DAG.getMachineFunction());
20498 default:
20499 return false;
20500 }
20501}
20502
20503bool SITargetLowering::denormalsEnabledForType(
20504 LLT Ty, const MachineFunction &MF) const {
20505 switch (Ty.getScalarSizeInBits()) {
20506 case 32:
20507 return !denormalModeIsFlushAllF32(MF);
20508 case 64:
20509 case 16:
20510 return !denormalModeIsFlushAllF64F16(MF);
20511 default:
20512 return false;
20513 }
20514}
20515
20516bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
20517 const APInt &DemandedElts,
20518 const SelectionDAG &DAG,
20519 bool SNaN,
20520 unsigned Depth) const {
20521 if (Op.getOpcode() == AMDGPUISD::CLAMP) {
20522 const MachineFunction &MF = DAG.getMachineFunction();
20523 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
20524
20525 if (Info->getMode().DX10Clamp)
20526 return true; // Clamped to 0.
20527 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
20528 }
20529
20530 return AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(Op, DemandedElts,
20531 DAG, SNaN, Depth);
20532}
20533
20534// On older subtargets, global FP atomic instructions have a hardcoded FP mode
20535// and do not support FP32 denormals, and only support v2f16/f64 denormals.
20536static bool atomicIgnoresDenormalModeOrFPModeIsFTZ(const AtomicRMWInst *RMW) {
20537 if (RMW->hasMetadata(Kind: "amdgpu.ignore.denormal.mode"))
20538 return true;
20539
20540 const fltSemantics &Flt = RMW->getType()->getScalarType()->getFltSemantics();
20541 auto DenormMode = RMW->getFunction()->getDenormalMode(FPType: Flt);
20542 if (DenormMode == DenormalMode::getPreserveSign())
20543 return true;
20544
20545 // TODO: Remove this.
20546 return RMW->getFunction()
20547 ->getFnAttribute(Kind: "amdgpu-unsafe-fp-atomics")
20548 .getValueAsBool();
20549}
20550
20551static OptimizationRemark emitAtomicRMWLegalRemark(const AtomicRMWInst *RMW) {
20552 LLVMContext &Ctx = RMW->getContext();
20553 StringRef MemScope =
20554 Ctx.getSyncScopeName(Id: RMW->getSyncScopeID()).value_or(u: "system");
20555
20556 return OptimizationRemark(DEBUG_TYPE, "Passed", RMW)
20557 << "Hardware instruction generated for atomic "
20558 << RMW->getOperationName(Op: RMW->getOperation())
20559 << " operation at memory scope " << MemScope;
20560}
20561
20562static bool isV2F16OrV2BF16(Type *Ty) {
20563 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
20564 Type *EltTy = VT->getElementType();
20565 return VT->getNumElements() == 2 &&
20566 (EltTy->isHalfTy() || EltTy->isBFloatTy());
20567 }
20568
20569 return false;
20570}
20571
20572static bool isV2F16(Type *Ty) {
20573 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
20574 return VT && VT->getNumElements() == 2 && VT->getElementType()->isHalfTy();
20575}
20576
20577static bool isV2BF16(Type *Ty) {
20578 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
20579 return VT && VT->getNumElements() == 2 && VT->getElementType()->isBFloatTy();
20580}
20581
20582/// \return true if atomicrmw integer ops work for the type.
20583static bool isAtomicRMWLegalIntTy(Type *Ty) {
20584 if (auto *IT = dyn_cast<IntegerType>(Val: Ty)) {
20585 unsigned BW = IT->getBitWidth();
20586 return BW == 32 || BW == 64;
20587 }
20588
20589 return false;
20590}
20591
20592/// \return true if this atomicrmw xchg type can be selected.
20593static bool isAtomicRMWLegalXChgTy(const AtomicRMWInst *RMW) {
20594 Type *Ty = RMW->getType();
20595 if (isAtomicRMWLegalIntTy(Ty))
20596 return true;
20597
20598 if (PointerType *PT = dyn_cast<PointerType>(Val: Ty)) {
20599 const DataLayout &DL = RMW->getFunction()->getParent()->getDataLayout();
20600 unsigned BW = DL.getPointerSizeInBits(AS: PT->getAddressSpace());
20601 return BW == 32 || BW == 64;
20602 }
20603
20604 if (Ty->isFloatTy() || Ty->isDoubleTy())
20605 return true;
20606
20607 if (FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
20608 return VT->getNumElements() == 2 &&
20609 VT->getElementType()->getPrimitiveSizeInBits() == 16;
20610 }
20611
20612 return false;
20613}
20614
20615/// \returns true if it's valid to emit a native instruction for \p RMW, based
20616/// on the properties of the target memory.
20617static bool globalMemoryFPAtomicIsLegal(const GCNSubtarget &Subtarget,
20618 const AtomicRMWInst *RMW,
20619 bool HasSystemScope) {
20620 // The remote/fine-grained access logic is different from the integer
20621 // atomics. Without AgentScopeFineGrainedRemoteMemoryAtomics support,
20622 // fine-grained access does not work, even for a device local allocation.
20623 //
20624 // With AgentScopeFineGrainedRemoteMemoryAtomics, system scoped device local
20625 // allocations work.
20626 if (HasSystemScope) {
20627 if (Subtarget.hasAgentScopeFineGrainedRemoteMemoryAtomics() &&
20628 RMW->hasMetadata(Kind: "amdgpu.no.remote.memory"))
20629 return true;
20630 if (Subtarget.hasEmulatedSystemScopeAtomics())
20631 return true;
20632 } else if (Subtarget.hasAgentScopeFineGrainedRemoteMemoryAtomics())
20633 return true;
20634
20635 return RMW->hasMetadata(Kind: "amdgpu.no.fine.grained.memory");
20636}
20637
20638/// \return Action to perform on AtomicRMWInsts for integer operations.
20639static TargetLowering::AtomicExpansionKind
20640atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW) {
20641 return isAtomicRMWLegalIntTy(Ty: RMW->getType())
20642 ? TargetLowering::AtomicExpansionKind::None
20643 : TargetLowering::AtomicExpansionKind::CmpXChg;
20644}
20645
20646/// Return if a flat address space atomicrmw can access private memory.
20647static bool flatInstrMayAccessPrivate(const Instruction *I) {
20648 const MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_noalias_addrspace);
20649 return !MD ||
20650 !AMDGPU::hasValueInRangeLikeMetadata(MD: *MD, Val: AMDGPUAS::PRIVATE_ADDRESS);
20651}
20652
20653static TargetLowering::AtomicExpansionKind
20654getPrivateAtomicExpansionKind(const GCNSubtarget &STI) {
20655 // For GAS, lower to flat atomic.
20656 return STI.hasGloballyAddressableScratch()
20657 ? TargetLowering::AtomicExpansionKind::CustomExpand
20658 : TargetLowering::AtomicExpansionKind::NotAtomic;
20659}
20660
20661TargetLowering::AtomicExpansionKind
20662SITargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
20663 unsigned AS = RMW->getPointerAddressSpace();
20664 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
20665 return getPrivateAtomicExpansionKind(STI: *getSubtarget());
20666
20667 // 64-bit flat atomics that dynamically reside in private memory will silently
20668 // be dropped.
20669 //
20670 // Note that we will emit a new copy of the original atomic in the expansion,
20671 // which will be incrementally relegalized.
20672 const DataLayout &DL = RMW->getFunction()->getDataLayout();
20673 if (AS == AMDGPUAS::FLAT_ADDRESS &&
20674 DL.getTypeSizeInBits(Ty: RMW->getType()) == 64 &&
20675 flatInstrMayAccessPrivate(I: RMW))
20676 return AtomicExpansionKind::CustomExpand;
20677
20678 auto ReportUnsafeHWInst = [=](TargetLowering::AtomicExpansionKind Kind) {
20679 OptimizationRemarkEmitter ORE(RMW->getFunction());
20680 ORE.emit(RemarkBuilder: [=]() {
20681 return emitAtomicRMWLegalRemark(RMW) << " due to an unsafe request.";
20682 });
20683 return Kind;
20684 };
20685
20686 auto SSID = RMW->getSyncScopeID();
20687 bool HasSystemScope =
20688 SSID == SyncScope::System ||
20689 SSID == RMW->getContext().getOrInsertSyncScopeID(SSN: *getAtomicScopeIRString(
20690 T: getTargetMachine().getTargetTriple(), S: AtomicScope::System,
20691 /*OneAddressSpace=*/IsSingleAddressSpace: true));
20692
20693 auto Op = RMW->getOperation();
20694 switch (Op) {
20695 case AtomicRMWInst::Xchg:
20696 // PCIe supports add and xchg for system atomics.
20697 return isAtomicRMWLegalXChgTy(RMW)
20698 ? TargetLowering::AtomicExpansionKind::None
20699 : TargetLowering::AtomicExpansionKind::CmpXChg;
20700 case AtomicRMWInst::Add:
20701 // PCIe supports add and xchg for system atomics.
20702 return atomicSupportedIfLegalIntType(RMW);
20703 case AtomicRMWInst::Sub:
20704 case AtomicRMWInst::And:
20705 case AtomicRMWInst::Or:
20706 case AtomicRMWInst::Xor:
20707 case AtomicRMWInst::Max:
20708 case AtomicRMWInst::Min:
20709 case AtomicRMWInst::UMax:
20710 case AtomicRMWInst::UMin:
20711 case AtomicRMWInst::UIncWrap:
20712 case AtomicRMWInst::UDecWrap:
20713 case AtomicRMWInst::USubCond:
20714 case AtomicRMWInst::USubSat: {
20715 if (Op == AtomicRMWInst::USubCond && !Subtarget->hasCondSubInsts())
20716 return AtomicExpansionKind::CmpXChg;
20717 if (Op == AtomicRMWInst::USubSat && !Subtarget->hasSubClampInsts())
20718 return AtomicExpansionKind::CmpXChg;
20719 if (Op == AtomicRMWInst::USubCond || Op == AtomicRMWInst::USubSat) {
20720 auto *IT = dyn_cast<IntegerType>(Val: RMW->getType());
20721 if (!IT || IT->getBitWidth() != 32)
20722 return AtomicExpansionKind::CmpXChg;
20723 }
20724
20725 if (AMDGPU::isFlatGlobalAddrSpace(AS) ||
20726 AS == AMDGPUAS::BUFFER_FAT_POINTER) {
20727 if (Subtarget->hasEmulatedSystemScopeAtomics())
20728 return atomicSupportedIfLegalIntType(RMW);
20729
20730 // On most subtargets, for atomicrmw operations other than add/xchg,
20731 // whether or not the instructions will behave correctly depends on where
20732 // the address physically resides and what interconnect is used in the
20733 // system configuration. On some some targets the instruction will nop,
20734 // and in others synchronization will only occur at degraded device scope.
20735 //
20736 // If the allocation is known local to the device, the instructions should
20737 // work correctly.
20738 if (RMW->hasMetadata(Kind: "amdgpu.no.remote.memory"))
20739 return atomicSupportedIfLegalIntType(RMW);
20740
20741 // If fine-grained remote memory works at device scope, we don't need to
20742 // do anything.
20743 if (!HasSystemScope &&
20744 Subtarget->hasAgentScopeFineGrainedRemoteMemoryAtomics())
20745 return atomicSupportedIfLegalIntType(RMW);
20746
20747 // If we are targeting a remote allocated address, it depends what kind of
20748 // allocation the address belongs to.
20749 //
20750 // If the allocation is fine-grained (in host memory, or in PCIe peer
20751 // device memory), the operation will fail depending on the target.
20752 //
20753 // Note fine-grained host memory access does work on APUs or if XGMI is
20754 // used, but we do not know if we are targeting an APU or the system
20755 // configuration from the ISA version/target-cpu.
20756 if (RMW->hasMetadata(Kind: "amdgpu.no.fine.grained.memory"))
20757 return atomicSupportedIfLegalIntType(RMW);
20758
20759 if (Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Or ||
20760 Op == AtomicRMWInst::Xor) {
20761 // Atomic sub/or/xor do not work over PCI express, but atomic add
20762 // does. InstCombine transforms these with 0 to or, so undo that.
20763 if (const Constant *ConstVal = dyn_cast<Constant>(Val: RMW->getValOperand());
20764 ConstVal && ConstVal->isNullValue())
20765 return AtomicExpansionKind::CustomExpand;
20766 }
20767
20768 // If the allocation could be in remote, fine-grained memory, the rmw
20769 // instructions may fail. cmpxchg should work, so emit that. On some
20770 // system configurations, PCIe atomics aren't supported so cmpxchg won't
20771 // even work, so you're out of luck anyway.
20772
20773 // In summary:
20774 //
20775 // Cases that may fail:
20776 // - fine-grained pinned host memory
20777 // - fine-grained migratable host memory
20778 // - fine-grained PCIe peer device
20779 //
20780 // Cases that should work, but may be treated overly conservatively.
20781 // - fine-grained host memory on an APU
20782 // - fine-grained XGMI peer device
20783 return AtomicExpansionKind::CmpXChg;
20784 }
20785
20786 return atomicSupportedIfLegalIntType(RMW);
20787 }
20788 case AtomicRMWInst::FAdd: {
20789 Type *Ty = RMW->getType();
20790
20791 // TODO: Handle REGION_ADDRESS
20792 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
20793 // DS F32 FP atomics do respect the denormal mode, but the rounding mode
20794 // is fixed to round-to-nearest-even.
20795 //
20796 // F64 / PK_F16 / PK_BF16 never flush and are also fixed to
20797 // round-to-nearest-even.
20798 //
20799 // We ignore the rounding mode problem, even in strictfp. The C++ standard
20800 // suggests it is OK if the floating-point mode may not match the calling
20801 // thread.
20802 if (Ty->isFloatTy()) {
20803 return Subtarget->hasLDSFPAtomicAddF32() ? AtomicExpansionKind::None
20804 : AtomicExpansionKind::CmpXChg;
20805 }
20806
20807 if (Ty->isDoubleTy()) {
20808 // Ignores denormal mode, but we don't consider flushing mandatory.
20809 return Subtarget->hasLDSFPAtomicAddF64() ? AtomicExpansionKind::None
20810 : AtomicExpansionKind::CmpXChg;
20811 }
20812
20813 if (Subtarget->hasAtomicDsPkAdd16Insts() && isV2F16OrV2BF16(Ty))
20814 return AtomicExpansionKind::None;
20815
20816 return AtomicExpansionKind::CmpXChg;
20817 }
20818
20819 // LDS atomics respect the denormal mode from the mode register.
20820 //
20821 // Traditionally f32 global/buffer memory atomics would unconditionally
20822 // flush denormals, but newer targets do not flush. f64/f16/bf16 cases never
20823 // flush.
20824 //
20825 // On targets with flat atomic fadd, denormals would flush depending on
20826 // whether the target address resides in LDS or global memory. We consider
20827 // this flat-maybe-flush as will-flush.
20828 if (Ty->isFloatTy() &&
20829 !Subtarget->hasMemoryAtomicFaddF32DenormalSupport() &&
20830 !atomicIgnoresDenormalModeOrFPModeIsFTZ(RMW))
20831 return AtomicExpansionKind::CmpXChg;
20832
20833 // FIXME: These ReportUnsafeHWInsts are imprecise. Some of these cases are
20834 // safe. The message phrasing also should be better.
20835 if (globalMemoryFPAtomicIsLegal(Subtarget: *Subtarget, RMW, HasSystemScope)) {
20836 if (AS == AMDGPUAS::FLAT_ADDRESS) {
20837 // gfx942, gfx12
20838 if (Subtarget->hasAtomicFlatPkAdd16Insts() && isV2F16OrV2BF16(Ty))
20839 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20840 } else if (AMDGPU::isExtendedGlobalAddrSpace(AS)) {
20841 // gfx90a, gfx942, gfx12
20842 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
20843 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20844
20845 // gfx942, gfx12
20846 if (Subtarget->hasAtomicGlobalPkAddBF16Inst() && isV2BF16(Ty))
20847 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20848 } else if (AS == AMDGPUAS::BUFFER_FAT_POINTER) {
20849 // gfx90a, gfx942, gfx12
20850 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
20851 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20852
20853 // While gfx90a/gfx942 supports v2bf16 for global/flat, it does not for
20854 // buffer. gfx12 does have the buffer version.
20855 if (Subtarget->hasAtomicBufferPkAddBF16Inst() && isV2BF16(Ty))
20856 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20857 }
20858
20859 // global and flat atomic fadd f64: gfx90a, gfx942.
20860 if (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() && Ty->isDoubleTy())
20861 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20862
20863 if (AS != AMDGPUAS::FLAT_ADDRESS) {
20864 if (Ty->isFloatTy()) {
20865 // global/buffer atomic fadd f32 no-rtn: gfx908, gfx90a, gfx942,
20866 // gfx11+.
20867 if (RMW->use_empty() && Subtarget->hasAtomicFaddNoRtnInsts())
20868 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20869 // global/buffer atomic fadd f32 rtn: gfx90a, gfx942, gfx11+.
20870 if (!RMW->use_empty() && Subtarget->hasAtomicFaddRtnInsts())
20871 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20872 } else {
20873 // gfx908
20874 if (RMW->use_empty() &&
20875 Subtarget->hasAtomicBufferGlobalPkAddF16NoRtnInsts() &&
20876 isV2F16(Ty))
20877 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20878 }
20879 }
20880
20881 // flat atomic fadd f32: gfx942, gfx11+.
20882 if (AS == AMDGPUAS::FLAT_ADDRESS && Ty->isFloatTy()) {
20883 if (Subtarget->hasFlatAtomicFaddF32Inst())
20884 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20885
20886 // If it is in flat address space, and the type is float, we will try to
20887 // expand it, if the target supports global and lds atomic fadd. The
20888 // reason we need that is, in the expansion, we emit the check of
20889 // address space. If it is in global address space, we emit the global
20890 // atomic fadd; if it is in shared address space, we emit the LDS atomic
20891 // fadd.
20892 if (Subtarget->hasLDSFPAtomicAddF32()) {
20893 if (RMW->use_empty() && Subtarget->hasAtomicFaddNoRtnInsts())
20894 return AtomicExpansionKind::CustomExpand;
20895 if (!RMW->use_empty() && Subtarget->hasAtomicFaddRtnInsts())
20896 return AtomicExpansionKind::CustomExpand;
20897 }
20898 }
20899 }
20900
20901 return AtomicExpansionKind::CmpXChg;
20902 }
20903 case AtomicRMWInst::FMin:
20904 case AtomicRMWInst::FMax: {
20905 Type *Ty = RMW->getType();
20906
20907 // LDS float and double fmin/fmax were always supported.
20908 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
20909 return Ty->isFloatTy() || Ty->isDoubleTy() ? AtomicExpansionKind::None
20910 : AtomicExpansionKind::CmpXChg;
20911 }
20912
20913 if (globalMemoryFPAtomicIsLegal(Subtarget: *Subtarget, RMW, HasSystemScope)) {
20914 // For flat and global cases:
20915 // float, double in gfx7. Manual claims denormal support.
20916 // Removed in gfx8.
20917 // float, double restored in gfx10.
20918 // double removed again in gfx11, so only f32 for gfx11/gfx12.
20919 //
20920 // For gfx9, gfx90a and gfx942 support f64 for global (same as fadd), but
20921 // no f32.
20922 if (AS == AMDGPUAS::FLAT_ADDRESS) {
20923 if (Subtarget->hasAtomicFMinFMaxF32FlatInsts() && Ty->isFloatTy())
20924 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20925 if (Subtarget->hasAtomicFMinFMaxF64FlatInsts() && Ty->isDoubleTy())
20926 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20927 } else if (AMDGPU::isExtendedGlobalAddrSpace(AS) ||
20928 AS == AMDGPUAS::BUFFER_FAT_POINTER) {
20929 if (Subtarget->hasAtomicFMinFMaxF32GlobalInsts() && Ty->isFloatTy())
20930 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20931 if (Subtarget->hasAtomicFMinFMaxF64GlobalInsts() && Ty->isDoubleTy())
20932 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20933 }
20934 }
20935
20936 return AtomicExpansionKind::CmpXChg;
20937 }
20938 case AtomicRMWInst::Nand:
20939 case AtomicRMWInst::FSub:
20940 default:
20941 return AtomicExpansionKind::CmpXChg;
20942 }
20943
20944 llvm_unreachable("covered atomicrmw op switch");
20945}
20946
20947TargetLowering::AtomicExpansionKind
20948SITargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
20949 return LI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS
20950 ? getPrivateAtomicExpansionKind(STI: *getSubtarget())
20951 : AtomicExpansionKind::None;
20952}
20953
20954TargetLowering::AtomicExpansionKind
20955SITargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
20956 return SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS
20957 ? getPrivateAtomicExpansionKind(STI: *getSubtarget())
20958 : AtomicExpansionKind::None;
20959}
20960
20961TargetLowering::AtomicExpansionKind
20962SITargetLowering::shouldExpandAtomicCmpXchgInIR(
20963 const AtomicCmpXchgInst *CmpX) const {
20964 unsigned AddrSpace = CmpX->getPointerAddressSpace();
20965 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)
20966 return getPrivateAtomicExpansionKind(STI: *getSubtarget());
20967
20968 if (AddrSpace != AMDGPUAS::FLAT_ADDRESS || !flatInstrMayAccessPrivate(I: CmpX))
20969 return AtomicExpansionKind::None;
20970
20971 const DataLayout &DL = CmpX->getDataLayout();
20972
20973 Type *ValTy = CmpX->getNewValOperand()->getType();
20974
20975 // If a 64-bit flat atomic may alias private, we need to avoid using the
20976 // atomic in the private case.
20977 return DL.getTypeSizeInBits(Ty: ValTy) == 64 ? AtomicExpansionKind::CustomExpand
20978 : AtomicExpansionKind::None;
20979}
20980
20981const TargetRegisterClass *
20982SITargetLowering::getRegClassFor(MVT VT, bool isDivergent) const {
20983 const TargetRegisterClass *RC = TargetLoweringBase::getRegClassFor(VT, isDivergent: false);
20984 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
20985 if (RC == &AMDGPU::VReg_1RegClass && !isDivergent)
20986 return Subtarget->isWave64() ? &AMDGPU::SReg_64RegClass
20987 : &AMDGPU::SReg_32RegClass;
20988 if (!TRI->isSGPRClass(RC) && !isDivergent)
20989 return TRI->getEquivalentSGPRClass(VRC: RC);
20990 if (TRI->isSGPRClass(RC) && isDivergent) {
20991 if (Subtarget->hasGFX90AInsts())
20992 return TRI->getEquivalentAVClass(SRC: RC);
20993 return TRI->getEquivalentVGPRClass(SRC: RC);
20994 }
20995
20996 return RC;
20997}
20998
20999// FIXME: This is a workaround for DivergenceAnalysis not understanding always
21000// uniform values (as produced by the mask results of control flow intrinsics)
21001// used outside of divergent blocks. The phi users need to also be treated as
21002// always uniform.
21003//
21004// FIXME: DA is no longer in-use. Does this still apply to UniformityAnalysis?
21005static bool hasCFUser(const Value *V, SmallPtrSet<const Value *, 16> &Visited,
21006 unsigned WaveSize) {
21007 // FIXME: We assume we never cast the mask results of a control flow
21008 // intrinsic.
21009 // Early exit if the type won't be consistent as a compile time hack.
21010 IntegerType *IT = dyn_cast<IntegerType>(Val: V->getType());
21011 if (!IT || IT->getBitWidth() != WaveSize)
21012 return false;
21013
21014 if (!isa<Instruction>(Val: V))
21015 return false;
21016 if (!Visited.insert(Ptr: V).second)
21017 return false;
21018 bool Result = false;
21019 for (const auto *U : V->users()) {
21020 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: U)) {
21021 if (V == U->getOperand(i: 1)) {
21022 switch (Intrinsic->getIntrinsicID()) {
21023 default:
21024 Result = false;
21025 break;
21026 case Intrinsic::amdgcn_if_break:
21027 case Intrinsic::amdgcn_if:
21028 case Intrinsic::amdgcn_else:
21029 Result = true;
21030 break;
21031 }
21032 }
21033 if (V == U->getOperand(i: 0)) {
21034 switch (Intrinsic->getIntrinsicID()) {
21035 default:
21036 Result = false;
21037 break;
21038 case Intrinsic::amdgcn_end_cf:
21039 case Intrinsic::amdgcn_loop:
21040 Result = true;
21041 break;
21042 }
21043 }
21044 } else {
21045 Result = hasCFUser(V: U, Visited, WaveSize);
21046 }
21047 if (Result)
21048 break;
21049 }
21050 return Result;
21051}
21052
21053bool SITargetLowering::requiresUniformRegister(MachineFunction &MF,
21054 const Value *V) const {
21055 if (const CallInst *CI = dyn_cast<CallInst>(Val: V)) {
21056 if (CI->isInlineAsm()) {
21057 // FIXME: This cannot give a correct answer. This should only trigger in
21058 // the case where inline asm returns mixed SGPR and VGPR results, used
21059 // outside the defining block. We don't have a specific result to
21060 // consider, so this assumes if any value is SGPR, the overall register
21061 // also needs to be SGPR.
21062 const SIRegisterInfo *SIRI = Subtarget->getRegisterInfo();
21063 TargetLowering::AsmOperandInfoVector TargetConstraints = ParseConstraints(
21064 DL: MF.getDataLayout(), TRI: Subtarget->getRegisterInfo(), Call: *CI);
21065 for (auto &TC : TargetConstraints) {
21066 if (TC.Type == InlineAsm::isOutput) {
21067 ComputeConstraintToUse(OpInfo&: TC, Op: SDValue());
21068 const TargetRegisterClass *RC =
21069 getRegForInlineAsmConstraint(TRI_: SIRI, Constraint: TC.ConstraintCode,
21070 VT: TC.ConstraintVT)
21071 .second;
21072 if (RC && SIRI->isSGPRClass(RC))
21073 return true;
21074 }
21075 }
21076 }
21077 }
21078 SmallPtrSet<const Value *, 16> Visited;
21079 return hasCFUser(V, Visited, WaveSize: Subtarget->getWavefrontSize());
21080}
21081
21082bool SITargetLowering::hasMemSDNodeUser(SDNode *N) const {
21083 for (SDUse &Use : N->uses()) {
21084 if (MemSDNode *M = dyn_cast<MemSDNode>(Val: Use.getUser())) {
21085 if (getBasePtrIndex(N: M) == Use.getOperandNo())
21086 return true;
21087 }
21088 }
21089 return false;
21090}
21091
21092bool SITargetLowering::isReassocProfitable(SelectionDAG &DAG, SDValue N0,
21093 SDValue N1) const {
21094 if (!N0.hasOneUse())
21095 return false;
21096 // Take care of the opportunity to keep N0 uniform
21097 if (N0->isDivergent() || !N1->isDivergent())
21098 return true;
21099 // Check if we have a good chance to form the memory access pattern with the
21100 // base and offset
21101 return (DAG.isBaseWithConstantOffset(Op: N0) &&
21102 hasMemSDNodeUser(N: *N0->user_begin()));
21103}
21104
21105bool SITargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
21106 Register N0, Register N1) const {
21107 return MRI.hasOneNonDBGUse(RegNo: N0); // FIXME: handle regbanks
21108}
21109
21110MachineMemOperand::Flags
21111SITargetLowering::getTargetMMOFlags(const Instruction &I) const {
21112 // Propagate metadata set by AMDGPUAnnotateUniformValues to the MMO of a load.
21113 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
21114 if (I.getMetadata(Kind: "amdgpu.noclobber"))
21115 Flags |= MONoClobber;
21116 if (I.getMetadata(Kind: "amdgpu.last.use"))
21117 Flags |= MOLastUse;
21118 return Flags;
21119}
21120
21121void SITargetLowering::emitExpandAtomicAddrSpacePredicate(
21122 Instruction *AI) const {
21123 // Given: atomicrmw fadd ptr %addr, float %val ordering
21124 //
21125 // With this expansion we produce the following code:
21126 // [...]
21127 // %is.shared = call i1 @llvm.amdgcn.is.shared(ptr %addr)
21128 // br i1 %is.shared, label %atomicrmw.shared, label %atomicrmw.check.private
21129 //
21130 // atomicrmw.shared:
21131 // %cast.shared = addrspacecast ptr %addr to ptr addrspace(3)
21132 // %loaded.shared = atomicrmw fadd ptr addrspace(3) %cast.shared,
21133 // float %val ordering
21134 // br label %atomicrmw.phi
21135 //
21136 // atomicrmw.check.private:
21137 // %is.private = call i1 @llvm.amdgcn.is.private(ptr %int8ptr)
21138 // br i1 %is.private, label %atomicrmw.private, label %atomicrmw.global
21139 //
21140 // atomicrmw.private:
21141 // %cast.private = addrspacecast ptr %addr to ptr addrspace(5)
21142 // %loaded.private = load float, ptr addrspace(5) %cast.private
21143 // %val.new = fadd float %loaded.private, %val
21144 // store float %val.new, ptr addrspace(5) %cast.private
21145 // br label %atomicrmw.phi
21146 //
21147 // atomicrmw.global:
21148 // %cast.global = addrspacecast ptr %addr to ptr addrspace(1)
21149 // %loaded.global = atomicrmw fadd ptr addrspace(1) %cast.global,
21150 // float %val ordering
21151 // br label %atomicrmw.phi
21152 //
21153 // atomicrmw.phi:
21154 // %loaded.phi = phi float [ %loaded.shared, %atomicrmw.shared ],
21155 // [ %loaded.private, %atomicrmw.private ],
21156 // [ %loaded.global, %atomicrmw.global ]
21157 // br label %atomicrmw.end
21158 //
21159 // atomicrmw.end:
21160 // [...]
21161 //
21162 //
21163 // For 64-bit atomics which may reside in private memory, we perform a simpler
21164 // version that only inserts the private check, and uses the flat operation.
21165
21166 IRBuilder<> Builder(AI);
21167 LLVMContext &Ctx = Builder.getContext();
21168
21169 auto *RMW = dyn_cast<AtomicRMWInst>(Val: AI);
21170 const unsigned PtrOpIdx = RMW ? AtomicRMWInst::getPointerOperandIndex()
21171 : AtomicCmpXchgInst::getPointerOperandIndex();
21172 Value *Addr = AI->getOperand(i: PtrOpIdx);
21173
21174 /// TODO: Only need to check private, then emit flat-known-not private (no
21175 /// need for shared block, or cast to global).
21176 AtomicCmpXchgInst *CX = dyn_cast<AtomicCmpXchgInst>(Val: AI);
21177
21178 Align Alignment;
21179 if (RMW)
21180 Alignment = RMW->getAlign();
21181 else if (CX)
21182 Alignment = CX->getAlign();
21183 else
21184 llvm_unreachable("unhandled atomic operation");
21185
21186 // FullFlatEmulation is true if we need to issue the private, shared, and
21187 // global cases.
21188 //
21189 // If this is false, we are only dealing with the flat-targeting-private case,
21190 // where we only insert a check for private and still use the flat instruction
21191 // for global and shared.
21192
21193 bool FullFlatEmulation =
21194 RMW && RMW->getOperation() == AtomicRMWInst::FAdd &&
21195 ((Subtarget->hasAtomicFaddInsts() && RMW->getType()->isFloatTy()) ||
21196 (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() &&
21197 RMW->getType()->isDoubleTy()));
21198
21199 // If the return value isn't used, do not introduce a false use in the phi.
21200 bool ReturnValueIsUsed = !AI->use_empty();
21201
21202 BasicBlock *BB = Builder.GetInsertBlock();
21203 Function *F = BB->getParent();
21204 BasicBlock *ExitBB =
21205 BB->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "atomicrmw.end");
21206 BasicBlock *SharedBB = nullptr;
21207
21208 BasicBlock *CheckPrivateBB = BB;
21209 if (FullFlatEmulation) {
21210 SharedBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.shared", Parent: F, InsertBefore: ExitBB);
21211 CheckPrivateBB =
21212 BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.check.private", Parent: F, InsertBefore: ExitBB);
21213 }
21214
21215 BasicBlock *PrivateBB =
21216 BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.private", Parent: F, InsertBefore: ExitBB);
21217 BasicBlock *GlobalBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.global", Parent: F, InsertBefore: ExitBB);
21218 BasicBlock *PhiBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.phi", Parent: F, InsertBefore: ExitBB);
21219
21220 std::prev(x: BB->end())->eraseFromParent();
21221 Builder.SetInsertPoint(BB);
21222
21223 Value *LoadedShared = nullptr;
21224 if (FullFlatEmulation) {
21225 Value *IsShared = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_is_shared,
21226 Args: {Addr}, FMFSource: nullptr, Name: "is.shared");
21227 Builder.CreateCondBr(Cond: IsShared, True: SharedBB, False: CheckPrivateBB);
21228 Builder.SetInsertPoint(SharedBB);
21229 Value *CastToLocal = Builder.CreateAddrSpaceCast(
21230 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::LOCAL_ADDRESS));
21231
21232 Instruction *Clone = AI->clone();
21233 Clone->insertInto(ParentBB: SharedBB, It: SharedBB->end());
21234 Clone->getOperandUse(i: PtrOpIdx).set(CastToLocal);
21235 LoadedShared = Clone;
21236
21237 Builder.CreateBr(Dest: PhiBB);
21238 Builder.SetInsertPoint(CheckPrivateBB);
21239 }
21240
21241 Value *IsPrivate = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_is_private,
21242 Args: {Addr}, FMFSource: nullptr, Name: "is.private");
21243 Builder.CreateCondBr(Cond: IsPrivate, True: PrivateBB, False: GlobalBB);
21244
21245 Builder.SetInsertPoint(PrivateBB);
21246
21247 Value *CastToPrivate = Builder.CreateAddrSpaceCast(
21248 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::PRIVATE_ADDRESS));
21249
21250 Value *LoadedPrivate;
21251 if (RMW) {
21252 LoadedPrivate = Builder.CreateAlignedLoad(
21253 Ty: RMW->getType(), Ptr: CastToPrivate, Align: RMW->getAlign(), Name: "loaded.private");
21254
21255 Value *NewVal = buildAtomicRMWValue(Op: RMW->getOperation(), Builder,
21256 Loaded: LoadedPrivate, Val: RMW->getValOperand());
21257
21258 Builder.CreateAlignedStore(Val: NewVal, Ptr: CastToPrivate, Align: RMW->getAlign());
21259 } else {
21260 auto [ResultLoad, Equal] =
21261 buildCmpXchgValue(Builder, Ptr: CastToPrivate, Cmp: CX->getCompareOperand(),
21262 Val: CX->getNewValOperand(), Alignment: CX->getAlign());
21263
21264 Value *Insert = Builder.CreateInsertValue(Agg: PoisonValue::get(T: CX->getType()),
21265 Val: ResultLoad, Idxs: 0);
21266 LoadedPrivate = Builder.CreateInsertValue(Agg: Insert, Val: Equal, Idxs: 1);
21267 }
21268
21269 Builder.CreateBr(Dest: PhiBB);
21270
21271 Builder.SetInsertPoint(GlobalBB);
21272
21273 // Continue using a flat instruction if we only emitted the check for private.
21274 Instruction *LoadedGlobal = AI;
21275 if (FullFlatEmulation) {
21276 Value *CastToGlobal = Builder.CreateAddrSpaceCast(
21277 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::GLOBAL_ADDRESS));
21278 AI->getOperandUse(i: PtrOpIdx).set(CastToGlobal);
21279 }
21280
21281 AI->removeFromParent();
21282 AI->insertInto(ParentBB: GlobalBB, It: GlobalBB->end());
21283
21284 // The new atomicrmw may go through another round of legalization later.
21285 if (!FullFlatEmulation) {
21286 // We inserted the runtime check already, make sure we do not try to
21287 // re-expand this.
21288 // TODO: Should union with any existing metadata.
21289 MDBuilder MDB(F->getContext());
21290 MDNode *RangeNotPrivate =
21291 MDB.createRange(Lo: APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
21292 Hi: APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
21293 LoadedGlobal->setMetadata(KindID: LLVMContext::MD_noalias_addrspace,
21294 Node: RangeNotPrivate);
21295 }
21296
21297 Builder.CreateBr(Dest: PhiBB);
21298
21299 Builder.SetInsertPoint(PhiBB);
21300
21301 if (ReturnValueIsUsed) {
21302 PHINode *Loaded = Builder.CreatePHI(Ty: AI->getType(), NumReservedValues: 3);
21303 AI->replaceAllUsesWith(V: Loaded);
21304 if (FullFlatEmulation)
21305 Loaded->addIncoming(V: LoadedShared, BB: SharedBB);
21306 Loaded->addIncoming(V: LoadedPrivate, BB: PrivateBB);
21307 Loaded->addIncoming(V: LoadedGlobal, BB: GlobalBB);
21308 Loaded->takeName(V: AI);
21309 }
21310
21311 Builder.CreateBr(Dest: ExitBB);
21312}
21313
21314static void convertScratchAtomicToFlatAtomic(Instruction *I,
21315 unsigned PtrOpIdx) {
21316 Value *PtrOp = I->getOperand(i: PtrOpIdx);
21317 assert(PtrOp->getType()->getPointerAddressSpace() ==
21318 AMDGPUAS::PRIVATE_ADDRESS);
21319
21320 Type *FlatPtr = PointerType::get(C&: I->getContext(), AddressSpace: AMDGPUAS::FLAT_ADDRESS);
21321 Value *ASCast = CastInst::CreatePointerCast(S: PtrOp, Ty: FlatPtr, Name: "scratch.ascast",
21322 InsertBefore: I->getIterator());
21323 I->setOperand(i: PtrOpIdx, Val: ASCast);
21324}
21325
21326void SITargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
21327 AtomicRMWInst::BinOp Op = AI->getOperation();
21328
21329 if (AI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21330 return convertScratchAtomicToFlatAtomic(I: AI, PtrOpIdx: AI->getPointerOperandIndex());
21331
21332 if (Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Or ||
21333 Op == AtomicRMWInst::Xor) {
21334 if (const auto *ConstVal = dyn_cast<Constant>(Val: AI->getValOperand());
21335 ConstVal && ConstVal->isNullValue()) {
21336 // atomicrmw or %ptr, 0 -> atomicrmw add %ptr, 0
21337 AI->setOperation(AtomicRMWInst::Add);
21338
21339 // We may still need the private-alias-flat handling below.
21340
21341 // TODO: Skip this for cases where we cannot access remote memory.
21342 }
21343 }
21344
21345 // The non-flat expansions should only perform the de-canonicalization of
21346 // identity values.
21347 if (AI->getPointerAddressSpace() != AMDGPUAS::FLAT_ADDRESS)
21348 return;
21349
21350 emitExpandAtomicAddrSpacePredicate(AI);
21351}
21352
21353void SITargetLowering::emitExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) const {
21354 if (CI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21355 return convertScratchAtomicToFlatAtomic(I: CI, PtrOpIdx: CI->getPointerOperandIndex());
21356
21357 emitExpandAtomicAddrSpacePredicate(AI: CI);
21358}
21359
21360void SITargetLowering::emitExpandAtomicLoad(LoadInst *LI) const {
21361 if (LI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21362 return convertScratchAtomicToFlatAtomic(I: LI, PtrOpIdx: LI->getPointerOperandIndex());
21363
21364 llvm_unreachable(
21365 "Expand Atomic Load only handles SCRATCH -> FLAT conversion");
21366}
21367
21368void SITargetLowering::emitExpandAtomicStore(StoreInst *SI) const {
21369 if (SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21370 return convertScratchAtomicToFlatAtomic(I: SI, PtrOpIdx: SI->getPointerOperandIndex());
21371
21372 llvm_unreachable(
21373 "Expand Atomic Store only handles SCRATCH -> FLAT conversion");
21374}
21375
21376LoadInst *
21377SITargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
21378 IRBuilder<> Builder(AI);
21379 auto Order = AI->getOrdering();
21380
21381 // The optimization removes store aspect of the atomicrmw. Therefore, cache
21382 // must be flushed if the atomic ordering had a release semantics. This is
21383 // not necessary a fence, a release fence just coincides to do that flush.
21384 // Avoid replacing of an atomicrmw with a release semantics.
21385 if (isReleaseOrStronger(AO: Order))
21386 return nullptr;
21387
21388 LoadInst *LI = Builder.CreateAlignedLoad(
21389 Ty: AI->getType(), Ptr: AI->getPointerOperand(), Align: AI->getAlign());
21390 LI->setAtomic(Ordering: Order, SSID: AI->getSyncScopeID());
21391 LI->copyMetadata(SrcInst: *AI);
21392 LI->takeName(V: AI);
21393 AI->replaceAllUsesWith(V: LI);
21394 AI->eraseFromParent();
21395 return LI;
21396}
21397