1//===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Custom DAG lowering for SI
11//
12//===----------------------------------------------------------------------===//
13
14#include "SIISelLowering.h"
15#include "AMDGPU.h"
16#include "AMDGPUIGroupLP.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPULaneMaskUtils.h"
19#include "AMDGPUMemoryUtils.h"
20#include "AMDGPUSelectionDAGInfo.h"
21#include "AMDGPUTargetMachine.h"
22#include "GCNSubtarget.h"
23#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
24#include "SIMachineFunctionInfo.h"
25#include "SIRegisterInfo.h"
26#include "llvm/ADT/APFloat.h"
27#include "llvm/ADT/APInt.h"
28#include "llvm/ADT/FloatingPointMode.h"
29#include "llvm/ADT/Statistic.h"
30#include "llvm/Analysis/OptimizationRemarkEmitter.h"
31#include "llvm/Analysis/UniformityAnalysis.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/ByteProvider.h"
34#include "llvm/CodeGen/FunctionLoweringInfo.h"
35#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
36#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
37#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
38#include "llvm/CodeGen/MachineFrameInfo.h"
39#include "llvm/CodeGen/MachineFunction.h"
40#include "llvm/CodeGen/MachineLoopInfo.h"
41#include "llvm/CodeGen/PseudoSourceValueManager.h"
42#include "llvm/CodeGen/SDPatternMatch.h"
43#include "llvm/IR/DiagnosticInfo.h"
44#include "llvm/IR/IRBuilder.h"
45#include "llvm/IR/IntrinsicInst.h"
46#include "llvm/IR/IntrinsicsAMDGPU.h"
47#include "llvm/IR/IntrinsicsR600.h"
48#include "llvm/IR/MDBuilder.h"
49#include "llvm/Support/AMDGPUAddrSpace.h"
50#include "llvm/Support/CommandLine.h"
51#include "llvm/Support/KnownBits.h"
52#include "llvm/Support/ModRef.h"
53#include "llvm/TargetParser/AtomicScope.h"
54#include "llvm/Transforms/Utils/LowerAtomic.h"
55#include <optional>
56
57using namespace llvm;
58using namespace llvm::SDPatternMatch;
59
60#define DEBUG_TYPE "si-lower"
61
62STATISTIC(NumTailCalls, "Number of tail calls");
63
64static cl::opt<bool>
65 DisableLoopAlignment("amdgpu-disable-loop-alignment",
66 cl::desc("Do not align and prefetch loops"),
67 cl::init(Val: false));
68
69static cl::opt<bool> UseDivergentRegisterIndexing(
70 "amdgpu-use-divergent-register-indexing", cl::Hidden,
71 cl::desc("Use indirect register addressing for divergent indexes"),
72 cl::init(Val: false));
73
74static DenormalFPEnv getDenormalFPEnv(const MachineFunction &MF) {
75 return MF.getInfo<SIMachineFunctionInfo>()->getMode().getDenormalFPEnv();
76}
77
78static bool denormalModeIsFlushAllF32(const MachineFunction &MF) {
79 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
80 return Info->getMode().FP32Denormals == DenormalMode::getPreserveSign();
81}
82
83static bool denormalModeIsFlushAllF64F16(const MachineFunction &MF) {
84 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
85 return Info->getMode().FP64FP16Denormals == DenormalMode::getPreserveSign();
86}
87
88static unsigned findFirstFreeSGPR(CCState &CCInfo) {
89 unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
90 for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) {
91 if (!CCInfo.isAllocated(Reg: AMDGPU::SGPR0 + Reg)) {
92 return AMDGPU::SGPR0 + Reg;
93 }
94 }
95 llvm_unreachable("Cannot allocate sgpr");
96}
97
98SITargetLowering::SITargetLowering(const TargetMachine &TM,
99 const GCNSubtarget &STI)
100 : AMDGPUTargetLowering(TM, STI, STI), Subtarget(&STI) {
101 addRegisterClass(VT: MVT::i1, RC: &AMDGPU::VReg_1RegClass);
102 addRegisterClass(VT: MVT::i64, RC: &AMDGPU::SReg_64RegClass);
103
104 addRegisterClass(VT: MVT::i32, RC: &AMDGPU::SReg_32RegClass);
105
106 const SIRegisterInfo *TRI = STI.getRegisterInfo();
107 const TargetRegisterClass *V32RegClass =
108 TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 32);
109 addRegisterClass(VT: MVT::f32, RC: V32RegClass);
110
111 addRegisterClass(VT: MVT::v2i32, RC: &AMDGPU::SReg_64RegClass);
112
113 const TargetRegisterClass *V64RegClass =
114 TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 64);
115
116 addRegisterClass(VT: MVT::f64, RC: V64RegClass);
117 addRegisterClass(VT: MVT::v2f32, RC: V64RegClass);
118 addRegisterClass(VT: MVT::Untyped, RC: V64RegClass);
119
120 addRegisterClass(VT: MVT::v3i32, RC: &AMDGPU::SGPR_96RegClass);
121 addRegisterClass(VT: MVT::v3f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 96));
122
123 addRegisterClass(VT: MVT::v2i64, RC: &AMDGPU::SGPR_128RegClass);
124 addRegisterClass(VT: MVT::v2f64, RC: &AMDGPU::SGPR_128RegClass);
125
126 addRegisterClass(VT: MVT::v4i32, RC: &AMDGPU::SGPR_128RegClass);
127 addRegisterClass(VT: MVT::v4f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 128));
128
129 addRegisterClass(VT: MVT::v5i32, RC: &AMDGPU::SGPR_160RegClass);
130 addRegisterClass(VT: MVT::v5f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 160));
131
132 addRegisterClass(VT: MVT::v6i32, RC: &AMDGPU::SGPR_192RegClass);
133 addRegisterClass(VT: MVT::v6f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 192));
134
135 addRegisterClass(VT: MVT::v3i64, RC: &AMDGPU::SGPR_192RegClass);
136 addRegisterClass(VT: MVT::v3f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 192));
137
138 addRegisterClass(VT: MVT::v7i32, RC: &AMDGPU::SGPR_224RegClass);
139 addRegisterClass(VT: MVT::v7f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 224));
140
141 addRegisterClass(VT: MVT::v8i32, RC: &AMDGPU::SGPR_256RegClass);
142 addRegisterClass(VT: MVT::v8f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 256));
143
144 addRegisterClass(VT: MVT::v4i64, RC: &AMDGPU::SGPR_256RegClass);
145 addRegisterClass(VT: MVT::v4f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 256));
146
147 addRegisterClass(VT: MVT::v9i32, RC: &AMDGPU::SGPR_288RegClass);
148 addRegisterClass(VT: MVT::v9f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 288));
149
150 addRegisterClass(VT: MVT::v10i32, RC: &AMDGPU::SGPR_320RegClass);
151 addRegisterClass(VT: MVT::v10f32,
152 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 320));
153
154 addRegisterClass(VT: MVT::v11i32, RC: &AMDGPU::SGPR_352RegClass);
155 addRegisterClass(VT: MVT::v11f32,
156 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 352));
157
158 addRegisterClass(VT: MVT::v12i32, RC: &AMDGPU::SGPR_384RegClass);
159 addRegisterClass(VT: MVT::v12f32,
160 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 384));
161
162 addRegisterClass(VT: MVT::v16i32, RC: &AMDGPU::SGPR_512RegClass);
163 addRegisterClass(VT: MVT::v16f32,
164 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 512));
165
166 addRegisterClass(VT: MVT::v8i64, RC: &AMDGPU::SGPR_512RegClass);
167 addRegisterClass(VT: MVT::v8f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 512));
168
169 addRegisterClass(VT: MVT::v16i64, RC: &AMDGPU::SGPR_1024RegClass);
170 addRegisterClass(VT: MVT::v16f64,
171 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 1024));
172
173 if (Subtarget->has16BitInsts()) {
174 if (Subtarget->useRealTrue16Insts()) {
175 addRegisterClass(VT: MVT::i16, RC: &AMDGPU::VGPR_16RegClass);
176 addRegisterClass(VT: MVT::f16, RC: &AMDGPU::VGPR_16RegClass);
177 addRegisterClass(VT: MVT::bf16, RC: &AMDGPU::VGPR_16RegClass);
178 } else {
179 addRegisterClass(VT: MVT::i16, RC: &AMDGPU::SReg_32RegClass);
180 addRegisterClass(VT: MVT::f16, RC: &AMDGPU::SReg_32RegClass);
181 addRegisterClass(VT: MVT::bf16, RC: &AMDGPU::SReg_32RegClass);
182 }
183
184 // Unless there are also VOP3P operations, not operations are really legal.
185 addRegisterClass(VT: MVT::v2i16, RC: &AMDGPU::SReg_32RegClass);
186 addRegisterClass(VT: MVT::v2f16, RC: &AMDGPU::SReg_32RegClass);
187 addRegisterClass(VT: MVT::v2bf16, RC: &AMDGPU::SReg_32RegClass);
188 addRegisterClass(VT: MVT::v4i16, RC: &AMDGPU::SReg_64RegClass);
189 addRegisterClass(VT: MVT::v4f16, RC: &AMDGPU::SReg_64RegClass);
190 addRegisterClass(VT: MVT::v4bf16, RC: &AMDGPU::SReg_64RegClass);
191 addRegisterClass(VT: MVT::v8i16, RC: &AMDGPU::SGPR_128RegClass);
192 addRegisterClass(VT: MVT::v8f16, RC: &AMDGPU::SGPR_128RegClass);
193 addRegisterClass(VT: MVT::v8bf16, RC: &AMDGPU::SGPR_128RegClass);
194 addRegisterClass(VT: MVT::v16i16, RC: &AMDGPU::SGPR_256RegClass);
195 addRegisterClass(VT: MVT::v16f16, RC: &AMDGPU::SGPR_256RegClass);
196 addRegisterClass(VT: MVT::v16bf16, RC: &AMDGPU::SGPR_256RegClass);
197 addRegisterClass(VT: MVT::v32i16, RC: &AMDGPU::SGPR_512RegClass);
198 addRegisterClass(VT: MVT::v32f16, RC: &AMDGPU::SGPR_512RegClass);
199 addRegisterClass(VT: MVT::v32bf16, RC: &AMDGPU::SGPR_512RegClass);
200 }
201
202 addRegisterClass(VT: MVT::v32i32, RC: &AMDGPU::VReg_1024RegClass);
203 addRegisterClass(VT: MVT::v32f32,
204 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 1024));
205
206 computeRegisterProperties(TRI: Subtarget->getRegisterInfo());
207
208 setMinFunctionAlignment(Align(4));
209 setPrefFunctionAlignment(Align(STI.getInstCacheLineSize()));
210
211 // The boolean content concept here is too inflexible. Compares only ever
212 // really produce a 1-bit result. Any copy/extend from these will turn into a
213 // select, and zext/1 or sext/-1 are equally cheap. Arbitrarily choose 0/1, as
214 // it's what most targets use.
215 setBooleanContents(ZeroOrOneBooleanContent);
216 setBooleanVectorContents(ZeroOrOneBooleanContent);
217
218 // We need to custom lower vector stores from local memory
219 setOperationAction(Ops: ISD::LOAD,
220 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
221 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
222 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
223 MVT::i1, MVT::v32i32},
224 Action: Custom);
225
226 setOperationAction(Ops: ISD::STORE,
227 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
228 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
229 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
230 MVT::i1, MVT::v32i32},
231 Action: Custom);
232
233 if (isTypeLegal(VT: MVT::bf16)) {
234 for (unsigned Opc :
235 {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
236 ISD::FREM, ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
237 ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FSQRT, ISD::FCBRT,
238 ISD::FSIN, ISD::FCOS, ISD::FPOW, ISD::FPOWI,
239 ISD::FLDEXP, ISD::FFREXP, ISD::FLOG, ISD::FLOG2,
240 ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FEXP10,
241 ISD::FCEIL, ISD::FTRUNC, ISD::FRINT, ISD::FNEARBYINT,
242 ISD::FROUND, ISD::FROUNDEVEN, ISD::FFLOOR, ISD::FCANONICALIZE,
243 ISD::SETCC}) {
244 setOperationAction(Op: Opc, VT: MVT::bf16, Action: Promote);
245 }
246
247 // Only targets with packed bf16 instructions, e.g. gfx13.
248 if (Subtarget->hasBF16PackedInsts()) {
249 // Don't use Expand for fsub - the DAG combiner will undo fadd+fneg back
250 // to fsub, causing a libcall (which doesn't exist for bf16). Instead,
251 // directly expand to widened v2bf16 operations.
252 setOperationAction(Op: ISD::FSUB, VT: MVT::bf16, Action: Custom);
253 // Promote scalar operations to a v2bf16 operation with an unused high
254 // lane.
255 for (unsigned Opc : {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FMAXNUM,
256 ISD::FMINNUM, ISD::FCANONICALIZE})
257 AddPromotedToType(Opc, OrigVT: MVT::bf16, DestVT: MVT::v2bf16);
258 }
259
260 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::bf16, Action: Expand);
261
262 setOperationAction(Op: ISD::SELECT, VT: MVT::bf16, Action: Promote);
263 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::bf16, DestVT: MVT::i16);
264
265 setOperationAction(Op: ISD::FABS, VT: MVT::bf16, Action: Legal);
266 setOperationAction(Op: ISD::FNEG, VT: MVT::bf16, Action: Legal);
267 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::bf16, Action: Legal);
268
269 // We only need to custom lower because we can't specify an action for bf16
270 // sources.
271 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
272 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
273 }
274
275 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i16, Action: Expand);
276 setTruncStoreAction(ValVT: MVT::v3i32, MemVT: MVT::v3i16, Action: Expand);
277 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i16, Action: Expand);
278 setTruncStoreAction(ValVT: MVT::v8i32, MemVT: MVT::v8i16, Action: Expand);
279 setTruncStoreAction(ValVT: MVT::v16i32, MemVT: MVT::v16i16, Action: Expand);
280 setTruncStoreAction(ValVT: MVT::v32i32, MemVT: MVT::v32i16, Action: Expand);
281 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i8, Action: Expand);
282 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i8, Action: Expand);
283 setTruncStoreAction(ValVT: MVT::v8i32, MemVT: MVT::v8i8, Action: Expand);
284 setTruncStoreAction(ValVT: MVT::v16i32, MemVT: MVT::v16i8, Action: Expand);
285 setTruncStoreAction(ValVT: MVT::v32i32, MemVT: MVT::v32i8, Action: Expand);
286 setTruncStoreAction(ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Expand);
287 setTruncStoreAction(ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Expand);
288 setTruncStoreAction(ValVT: MVT::v8i16, MemVT: MVT::v8i8, Action: Expand);
289 setTruncStoreAction(ValVT: MVT::v16i16, MemVT: MVT::v16i8, Action: Expand);
290 setTruncStoreAction(ValVT: MVT::v32i16, MemVT: MVT::v32i8, Action: Expand);
291
292 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i16, Action: Expand);
293 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i32, Action: Expand);
294 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i8, Action: Expand);
295 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i8, Action: Expand);
296 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i16, Action: Expand);
297 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i32, Action: Expand);
298 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i32, Action: Expand);
299
300 setOperationAction(Ops: ISD::GlobalAddress, VTs: {MVT::i32, MVT::i64}, Action: Custom);
301 setOperationAction(Ops: ISD::BlockAddress, VTs: {MVT::i32, MVT::i64}, Action: Custom);
302 setOperationAction(Ops: ISD::ExternalSymbol, VTs: {MVT::i32, MVT::i64}, Action: Custom);
303
304 setOperationAction(Op: ISD::SELECT, VT: MVT::i1, Action: Promote);
305 setOperationAction(Op: ISD::SELECT, VT: MVT::i64, Action: Custom);
306 setOperationAction(Op: ISD::SELECT, VT: MVT::f64, Action: Promote);
307 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::f64, DestVT: MVT::i64);
308
309 setOperationAction(Ops: ISD::FSQRT, VTs: {MVT::f32, MVT::f64}, Action: Custom);
310
311 setOperationAction(Ops: ISD::SELECT_CC,
312 VTs: {MVT::f32, MVT::i32, MVT::i64, MVT::f64, MVT::i1}, Action: Expand);
313
314 setOperationAction(Op: ISD::SETCC, VT: MVT::i1, Action: Promote);
315 setOperationAction(Ops: ISD::SETCC, VTs: {MVT::v2i1, MVT::v4i1}, Action: Expand);
316 AddPromotedToType(Opc: ISD::SETCC, OrigVT: MVT::i1, DestVT: MVT::i32);
317
318 setOperationAction(Ops: ISD::TRUNCATE,
319 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
320 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
321 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32},
322 Action: Expand);
323 setOperationAction(Ops: ISD::FP_ROUND,
324 VTs: {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
325 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v9f32,
326 MVT::v10f32, MVT::v11f32, MVT::v12f32, MVT::v16f32},
327 Action: Expand);
328
329 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG,
330 VTs: {MVT::v2i1, MVT::v4i1, MVT::v2i8, MVT::v4i8, MVT::v2i16,
331 MVT::v3i16, MVT::v4i16, MVT::Other},
332 Action: Custom);
333
334 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
335 setOperationAction(Ops: ISD::BR_CC,
336 VTs: {MVT::i1, MVT::i32, MVT::i64, MVT::f32, MVT::f64}, Action: Expand);
337
338 setOperationAction(Ops: {ISD::ABS, ISD::UADDO, ISD::USUBO}, VT: MVT::i32, Action: Legal);
339 setOperationAction(Ops: {ISD::UADDO, ISD::USUBO}, VT: MVT::i64, Action: Legal);
340
341 setOperationAction(Ops: {ISD::UADDO_CARRY, ISD::USUBO_CARRY}, VT: MVT::i32, Action: Legal);
342 setOperationAction(Ops: {ISD::UADDO_CARRY, ISD::USUBO_CARRY}, VT: MVT::i64, Action: Legal);
343
344 setOperationAction(Ops: {ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS}, VT: MVT::i64,
345 Action: Expand);
346
347 setOperationAction(Op: ISD::INLINEASM, VT: MVT::Other, Action: Custom);
348
349 // We only support LOAD/STORE and vector manipulation ops for vectors
350 // with > 4 elements.
351 for (MVT VT :
352 {MVT::v8i32, MVT::v8f32, MVT::v9i32, MVT::v9f32, MVT::v10i32,
353 MVT::v10f32, MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32,
354 MVT::v16i32, MVT::v16f32, MVT::v2i64, MVT::v2f64, MVT::v4i16,
355 MVT::v4f16, MVT::v4bf16, MVT::v3i64, MVT::v3f64, MVT::v6i32,
356 MVT::v6f32, MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64,
357 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
358 MVT::v16bf16, MVT::v16i64, MVT::v16f64, MVT::v32i32, MVT::v32f32,
359 MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
360 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
361 switch (Op) {
362 case ISD::LOAD:
363 case ISD::STORE:
364 case ISD::BUILD_VECTOR:
365 case ISD::BITCAST:
366 case ISD::UNDEF:
367 case ISD::POISON:
368 case ISD::EXTRACT_VECTOR_ELT:
369 case ISD::INSERT_VECTOR_ELT:
370 case ISD::SCALAR_TO_VECTOR:
371 case ISD::IS_FPCLASS:
372 break;
373 case ISD::EXTRACT_SUBVECTOR:
374 case ISD::INSERT_SUBVECTOR:
375 case ISD::CONCAT_VECTORS:
376 setOperationAction(Op, VT, Action: Custom);
377 break;
378 default:
379 setOperationAction(Op, VT, Action: Expand);
380 break;
381 }
382 }
383 }
384
385 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v4f32, Action: Expand);
386
387 // TODO: For dynamic 64-bit vector inserts/extracts, should emit a pseudo that
388 // is expanded to avoid having two separate loops in case the index is a VGPR.
389
390 // Most operations are naturally 32-bit vector operations. We only support
391 // load and store of i64 vectors, so promote v2i64 vector operations to v4i32.
392 for (MVT Vec64 : {MVT::v2i64, MVT::v2f64}) {
393 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
394 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v4i32);
395
396 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
397 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v4i32);
398
399 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
400 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v4i32);
401
402 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
403 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v4i32);
404 }
405
406 for (MVT Vec64 : {MVT::v3i64, MVT::v3f64}) {
407 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
408 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v6i32);
409
410 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
411 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v6i32);
412
413 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
414 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v6i32);
415
416 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
417 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v6i32);
418 }
419
420 for (MVT Vec64 : {MVT::v4i64, MVT::v4f64}) {
421 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
422 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v8i32);
423
424 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
425 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v8i32);
426
427 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
428 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v8i32);
429
430 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
431 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v8i32);
432 }
433
434 for (MVT Vec64 : {MVT::v8i64, MVT::v8f64}) {
435 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
436 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v16i32);
437
438 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
439 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v16i32);
440
441 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
442 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v16i32);
443
444 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
445 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v16i32);
446 }
447
448 for (MVT Vec64 : {MVT::v16i64, MVT::v16f64}) {
449 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
450 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v32i32);
451
452 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
453 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v32i32);
454
455 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
456 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v32i32);
457
458 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
459 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v32i32);
460 }
461
462 setOperationAction(Ops: ISD::VECTOR_SHUFFLE,
463 VTs: {MVT::v4i32, MVT::v4f32, MVT::v8i32, MVT::v8f32,
464 MVT::v16i32, MVT::v16f32, MVT::v32i32, MVT::v32f32},
465 Action: Custom);
466
467 if (Subtarget->hasPkMovB32()) {
468 // TODO: 16-bit element vectors should be legal with even aligned elements.
469 // TODO: Can be legal with wider source types than the result with
470 // subregister extracts.
471 setOperationAction(Ops: ISD::VECTOR_SHUFFLE, VTs: {MVT::v2i32, MVT::v2f32}, Action: Legal);
472 }
473
474 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VT: MVT::v2i32, Action: Legal);
475 // Prevent SELECT v2i32 from being implemented with the above bitwise ops and
476 // instead lower to cndmask in SITargetLowering::LowerSELECT().
477 setOperationAction(Op: ISD::SELECT, VT: MVT::v2i32, Action: Custom);
478 // Enable MatchRotate to produce ISD::ROTR, which is later transformed to
479 // alignbit.
480 setOperationAction(Op: ISD::ROTR, VT: MVT::v2i32, Action: Custom);
481
482 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs: {MVT::v4f16, MVT::v4i16, MVT::v4bf16},
483 Action: Custom);
484
485 // Avoid stack access for these.
486 // TODO: Generalize to more vector types.
487 setOperationAction(Ops: {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
488 VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v2i8, MVT::v4i8,
489 MVT::v8i8, MVT::v4i16, MVT::v4f16, MVT::v4bf16},
490 Action: Custom);
491
492 // Deal with vec3 vector operations when widened to vec4.
493 setOperationAction(Ops: ISD::INSERT_SUBVECTOR,
494 VTs: {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32}, Action: Custom);
495
496 // Deal with vec5/6/7 vector operations when widened to vec8.
497 setOperationAction(Ops: ISD::INSERT_SUBVECTOR,
498 VTs: {MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
499 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
500 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
501 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
502 Action: Custom);
503
504 // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling,
505 // and output demarshalling
506 setOperationAction(Ops: ISD::ATOMIC_CMP_SWAP, VTs: {MVT::i32, MVT::i64}, Action: Custom);
507
508 // We can't return success/failure, only the old value,
509 // let LLVM add the comparison
510 setOperationAction(Ops: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VTs: {MVT::i32, MVT::i64},
511 Action: Expand);
512
513 setOperationAction(Ops: ISD::ADDRSPACECAST, VTs: {MVT::i32, MVT::i64}, Action: Custom);
514
515 setOperationAction(Ops: ISD::BITREVERSE, VTs: {MVT::i32, MVT::i64}, Action: Legal);
516
517 // FIXME: This should be narrowed to i32, but that only happens if i64 is
518 // illegal.
519 // FIXME: Should lower sub-i32 bswaps to bit-ops without v_perm_b32.
520 setOperationAction(Ops: ISD::BSWAP, VTs: {MVT::i64, MVT::i32}, Action: Legal);
521
522 // On SI this is s_memtime and s_memrealtime on VI.
523 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Legal);
524
525 if (Subtarget->hasSMemRealTime() ||
526 Subtarget->getGeneration() >= AMDGPUSubtarget::GFX11)
527 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64, Action: Legal);
528 setOperationAction(Ops: {ISD::TRAP, ISD::DEBUGTRAP}, VT: MVT::Other, Action: Custom);
529
530 if (Subtarget->has16BitInsts()) {
531 setOperationAction(Ops: {ISD::FPOW, ISD::FPOWI}, VT: MVT::f16, Action: Promote);
532 setOperationAction(Ops: {ISD::FLOG, ISD::FEXP, ISD::FLOG10}, VT: MVT::f16, Action: Custom);
533 setOperationAction(Ops: ISD::IS_FPCLASS, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Legal);
534 setOperationAction(Ops: {ISD::FLOG2, ISD::FEXP2}, VT: MVT::f16, Action: Legal);
535 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f16, Action: Legal);
536 } else {
537 setOperationAction(Op: ISD::FSQRT, VT: MVT::f16, Action: Custom);
538 }
539
540 if (Subtarget->hasMadMacF32Insts())
541 setOperationAction(Op: ISD::FMAD, VT: MVT::f32, Action: Legal);
542
543 setOperationAction(Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT: MVT::i32, Action: Custom);
544 setOperationAction(Ops: {ISD::CTTZ, ISD::CTTZ_ZERO_POISON}, VT: MVT::i32, Action: Custom);
545 setOperationAction(Op: ISD::CTLS, VT: MVT::i32, Action: Custom);
546
547 // We only really have 32-bit BFE instructions (and 16-bit on VI).
548 //
549 // On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any
550 // effort to match them now. We want this to be false for i64 cases when the
551 // extraction isn't restricted to the upper or lower half. Ideally we would
552 // have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that
553 // span the midpoint are probably relatively rare, so don't worry about them
554 // for now.
555 setHasExtractBitsInsn(true);
556
557 // Clamp modifier on add/sub
558 if (Subtarget->hasIntClamp())
559 setOperationAction(Ops: {ISD::UADDSAT, ISD::USUBSAT}, VT: MVT::i32, Action: Legal);
560
561 if (Subtarget->hasAddNoCarryInsts())
562 setOperationAction(Ops: {ISD::SADDSAT, ISD::SSUBSAT}, VTs: {MVT::i16, MVT::i32},
563 Action: Legal);
564
565 // Do not have s_{min|max}_*f64 instruction f64 will only be lowered to
566 // v_{min|max}_*f64
567 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
568 setOperationAction(
569 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
570 VTs: {MVT::f64, MVT::f32}, Action: Legal);
571 } else {
572 setOperationAction(
573 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
574 VTs: {MVT::f64, MVT::f32}, Action: Custom);
575 // These are really only legal for ieee_mode functions. We should be
576 // avoiding them for functions that don't have ieee_mode enabled, so just
577 // say they are legal.
578 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE},
579 VTs: {MVT::f64, MVT::f32}, Action: Legal);
580 }
581
582 if (Subtarget->haveRoundOpsF64())
583 setOperationAction(Ops: {ISD::FTRUNC, ISD::FCEIL, ISD::FROUNDEVEN}, VT: MVT::f64,
584 Action: Legal);
585 else
586 setOperationAction(Ops: {ISD::FCEIL, ISD::FTRUNC, ISD::FROUNDEVEN, ISD::FFLOOR},
587 VT: MVT::f64, Action: Custom);
588
589 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f64, Action: Legal);
590 setOperationAction(Ops: {ISD::FLDEXP, ISD::STRICT_FLDEXP}, VTs: {MVT::f32, MVT::f64},
591 Action: Legal);
592 setOperationAction(Ops: ISD::FFREXP, VTs: {MVT::f32, MVT::f64}, Action: Custom);
593
594 setOperationAction(Ops: {ISD::FSIN, ISD::FCOS, ISD::FDIV}, VT: MVT::f32, Action: Custom);
595 setOperationAction(Op: ISD::FDIV, VT: MVT::f64, Action: Custom);
596
597 setOperationAction(Ops: ISD::BF16_TO_FP, VTs: {MVT::i16, MVT::f32, MVT::f64}, Action: Expand);
598 setOperationAction(Ops: ISD::FP_TO_BF16, VTs: {MVT::i16, MVT::f32, MVT::f64}, Action: Expand);
599
600 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i32,
601 Action: Custom);
602 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i16,
603 Action: Custom);
604 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i1,
605 Action: Custom);
606
607 // Custom lower these because we can't specify a rule based on an illegal
608 // source bf16.
609 setOperationAction(Ops: {ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, VT: MVT::f32, Action: Custom);
610 setOperationAction(Ops: {ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, VT: MVT::f64, Action: Custom);
611
612 if (Subtarget->has16BitInsts()) {
613 setOperationAction(Ops: {ISD::Constant, ISD::SMIN, ISD::SMAX, ISD::UMIN,
614 ISD::UMAX, ISD::UADDSAT, ISD::USUBSAT},
615 VT: MVT::i16, Action: Legal);
616
617 AddPromotedToType(Opc: ISD::SIGN_EXTEND, OrigVT: MVT::i16, DestVT: MVT::i32);
618
619 setOperationAction(Ops: {ISD::ROTR, ISD::ROTL, ISD::SELECT_CC, ISD::BR_CC},
620 VT: MVT::i16, Action: Expand);
621
622 setOperationAction(Ops: {ISD::SIGN_EXTEND, ISD::SDIV, ISD::UDIV, ISD::SREM,
623 ISD::UREM, ISD::BITREVERSE, ISD::CTTZ,
624 ISD::CTTZ_ZERO_POISON, ISD::CTLZ, ISD::CTLZ_ZERO_POISON,
625 ISD::CTPOP},
626 VT: MVT::i16, Action: Promote);
627
628 setOperationAction(Op: ISD::LOAD, VT: MVT::i16, Action: Custom);
629
630 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i16, Action: Expand);
631
632 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::i16, Action: Promote);
633 AddPromotedToType(Opc: ISD::FP16_TO_FP, OrigVT: MVT::i16, DestVT: MVT::i32);
634 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::i16, Action: Promote);
635 AddPromotedToType(Opc: ISD::FP_TO_FP16, OrigVT: MVT::i16, DestVT: MVT::i32);
636
637 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: MVT::i16, Action: Custom);
638 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: MVT::i32, Action: Custom);
639 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i16, Action: Custom);
640 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i1, Action: Custom);
641
642 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i32, Action: Custom);
643
644 // F16 - Constant Actions.
645 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Legal);
646 setOperationAction(Op: ISD::ConstantFP, VT: MVT::bf16, Action: Legal);
647
648 // F16 - Load/Store Actions.
649 setOperationAction(Op: ISD::LOAD, VT: MVT::f16, Action: Promote);
650 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f16, DestVT: MVT::i16);
651 setOperationAction(Op: ISD::STORE, VT: MVT::f16, Action: Promote);
652 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f16, DestVT: MVT::i16);
653
654 // BF16 - Load/Store Actions.
655 setOperationAction(Op: ISD::LOAD, VT: MVT::bf16, Action: Promote);
656 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::bf16, DestVT: MVT::i16);
657 setOperationAction(Op: ISD::STORE, VT: MVT::bf16, Action: Promote);
658 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::bf16, DestVT: MVT::i16);
659
660 // F16 - VOP1 Actions.
661 setOperationAction(Ops: {ISD::FP_ROUND, ISD::STRICT_FP_ROUND, ISD::FCOS,
662 ISD::FSIN, ISD::FROUND},
663 VT: MVT::f16, Action: Custom);
664
665 // BF16 - VOP1 Actions.
666 if (Subtarget->hasBF16TransInsts())
667 setOperationAction(Ops: {ISD::FCOS, ISD::FSIN, ISD::FDIV}, VT: MVT::bf16, Action: Custom);
668
669 // F16 - VOP2 Actions.
670 setOperationAction(Ops: {ISD::BR_CC, ISD::SELECT_CC}, VTs: {MVT::f16, MVT::bf16},
671 Action: Expand);
672 setOperationAction(Ops: {ISD::FLDEXP, ISD::STRICT_FLDEXP}, VT: MVT::f16, Action: Custom);
673 setOperationAction(Op: ISD::FFREXP, VT: MVT::f16, Action: Custom);
674 setOperationAction(Op: ISD::FDIV, VT: MVT::f16, Action: Custom);
675
676 // F16 - VOP3 Actions.
677 setOperationAction(Op: ISD::FMA, VT: MVT::f16, Action: Legal);
678 if (STI.hasMadF16())
679 setOperationAction(Op: ISD::FMAD, VT: MVT::f16, Action: Legal);
680
681 for (MVT VT :
682 {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v4i16, MVT::v4f16,
683 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16,
684 MVT::v16f16, MVT::v16bf16, MVT::v32i16, MVT::v32f16}) {
685 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
686 switch (Op) {
687 case ISD::LOAD:
688 case ISD::STORE:
689 case ISD::BUILD_VECTOR:
690 case ISD::BITCAST:
691 case ISD::UNDEF:
692 case ISD::POISON:
693 case ISD::EXTRACT_VECTOR_ELT:
694 case ISD::INSERT_VECTOR_ELT:
695 case ISD::INSERT_SUBVECTOR:
696 case ISD::SCALAR_TO_VECTOR:
697 case ISD::IS_FPCLASS:
698 break;
699 case ISD::EXTRACT_SUBVECTOR:
700 case ISD::CONCAT_VECTORS:
701 case ISD::FSIN:
702 case ISD::FCOS:
703 setOperationAction(Op, VT, Action: Custom);
704 break;
705 default:
706 setOperationAction(Op, VT, Action: Expand);
707 break;
708 }
709 }
710 }
711
712 // v_perm_b32 can handle either of these.
713 setOperationAction(Ops: ISD::BSWAP, VTs: {MVT::i16, MVT::v2i16}, Action: Legal);
714 setOperationAction(Op: ISD::BSWAP, VT: MVT::v4i16, Action: Custom);
715
716 // Legalize vector types for sat conversions to select v_cvt_pk_[iu]16_f32.
717 if (Subtarget->hasVCvtPkIU16F32())
718 setOperationAction(
719 Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT},
720 VTs: {MVT::v2i16, MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16},
721 Action: Custom);
722
723 // XXX - Do these do anything? Vector constants turn into build_vector.
724 setOperationAction(Ops: ISD::Constant, VTs: {MVT::v2i16, MVT::v2f16}, Action: Legal);
725
726 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON},
727 VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Action: Legal);
728
729 setOperationAction(Op: ISD::STORE, VT: MVT::v2i16, Action: Promote);
730 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2i16, DestVT: MVT::i32);
731 setOperationAction(Op: ISD::STORE, VT: MVT::v2f16, Action: Promote);
732 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f16, DestVT: MVT::i32);
733
734 setOperationAction(Op: ISD::LOAD, VT: MVT::v2i16, Action: Promote);
735 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2i16, DestVT: MVT::i32);
736 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f16, Action: Promote);
737 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f16, DestVT: MVT::i32);
738
739 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::v2i16, Action: Promote);
740 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::v2i16, DestVT: MVT::i32);
741 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::v2f16, Action: Promote);
742 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::v2f16, DestVT: MVT::i32);
743
744 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::v2i16, Action: Promote);
745 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::v2i16, DestVT: MVT::i32);
746 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::v2f16, Action: Promote);
747 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::v2f16, DestVT: MVT::i32);
748
749 setOperationAction(Op: ISD::AND, VT: MVT::v2i16, Action: Promote);
750 AddPromotedToType(Opc: ISD::AND, OrigVT: MVT::v2i16, DestVT: MVT::i32);
751 setOperationAction(Op: ISD::OR, VT: MVT::v2i16, Action: Promote);
752 AddPromotedToType(Opc: ISD::OR, OrigVT: MVT::v2i16, DestVT: MVT::i32);
753 setOperationAction(Op: ISD::XOR, VT: MVT::v2i16, Action: Promote);
754 AddPromotedToType(Opc: ISD::XOR, OrigVT: MVT::v2i16, DestVT: MVT::i32);
755
756 setOperationAction(Op: ISD::LOAD, VT: MVT::v4i16, Action: Promote);
757 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
758 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f16, Action: Promote);
759 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
760 setOperationAction(Op: ISD::LOAD, VT: MVT::v4bf16, Action: Promote);
761 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4bf16, DestVT: MVT::v2i32);
762
763 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::v4i16, Action: Promote);
764 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::v4i16, DestVT: MVT::i64);
765 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::v4f16, Action: Promote);
766 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::v4f16, DestVT: MVT::i64);
767
768 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::v4i16, Action: Promote);
769 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::v4i16, DestVT: MVT::i64);
770 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::v4f16, Action: Promote);
771 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::v4f16, DestVT: MVT::i64);
772
773 setOperationAction(Op: ISD::STORE, VT: MVT::v4i16, Action: Promote);
774 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
775 setOperationAction(Op: ISD::STORE, VT: MVT::v4f16, Action: Promote);
776 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
777 setOperationAction(Op: ISD::STORE, VT: MVT::v4bf16, Action: Promote);
778 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4bf16, DestVT: MVT::v2i32);
779
780 setOperationAction(Op: ISD::LOAD, VT: MVT::v8i16, Action: Promote);
781 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8i16, DestVT: MVT::v4i32);
782 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f16, Action: Promote);
783 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f16, DestVT: MVT::v4i32);
784 setOperationAction(Op: ISD::LOAD, VT: MVT::v8bf16, Action: Promote);
785 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8bf16, DestVT: MVT::v4i32);
786
787 setOperationAction(Op: ISD::STORE, VT: MVT::v4i16, Action: Promote);
788 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
789 setOperationAction(Op: ISD::STORE, VT: MVT::v4f16, Action: Promote);
790 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
791
792 setOperationAction(Op: ISD::STORE, VT: MVT::v8i16, Action: Promote);
793 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8i16, DestVT: MVT::v4i32);
794 setOperationAction(Op: ISD::STORE, VT: MVT::v8f16, Action: Promote);
795 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f16, DestVT: MVT::v4i32);
796 setOperationAction(Op: ISD::STORE, VT: MVT::v8bf16, Action: Promote);
797 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8bf16, DestVT: MVT::v4i32);
798
799 setOperationAction(Op: ISD::LOAD, VT: MVT::v16i16, Action: Promote);
800 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16i16, DestVT: MVT::v8i32);
801 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f16, Action: Promote);
802 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f16, DestVT: MVT::v8i32);
803 setOperationAction(Op: ISD::LOAD, VT: MVT::v16bf16, Action: Promote);
804 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16bf16, DestVT: MVT::v8i32);
805
806 setOperationAction(Op: ISD::STORE, VT: MVT::v16i16, Action: Promote);
807 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16i16, DestVT: MVT::v8i32);
808 setOperationAction(Op: ISD::STORE, VT: MVT::v16f16, Action: Promote);
809 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f16, DestVT: MVT::v8i32);
810 setOperationAction(Op: ISD::STORE, VT: MVT::v16bf16, Action: Promote);
811 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16bf16, DestVT: MVT::v8i32);
812
813 setOperationAction(Op: ISD::LOAD, VT: MVT::v32i16, Action: Promote);
814 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32i16, DestVT: MVT::v16i32);
815 setOperationAction(Op: ISD::LOAD, VT: MVT::v32f16, Action: Promote);
816 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32f16, DestVT: MVT::v16i32);
817 setOperationAction(Op: ISD::LOAD, VT: MVT::v32bf16, Action: Promote);
818 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32bf16, DestVT: MVT::v16i32);
819
820 setOperationAction(Op: ISD::STORE, VT: MVT::v32i16, Action: Promote);
821 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32i16, DestVT: MVT::v16i32);
822 setOperationAction(Op: ISD::STORE, VT: MVT::v32f16, Action: Promote);
823 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32f16, DestVT: MVT::v16i32);
824 setOperationAction(Op: ISD::STORE, VT: MVT::v32bf16, Action: Promote);
825 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32bf16, DestVT: MVT::v16i32);
826
827 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
828 VT: MVT::v2i32, Action: Expand);
829 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Expand);
830
831 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
832 VT: MVT::v4i32, Action: Expand);
833
834 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
835 VT: MVT::v8i32, Action: Expand);
836
837 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
838 Action: Subtarget->hasVOP3PInsts() ? Legal : Custom);
839
840 setOperationAction(Ops: ISD::FNEG, VTs: {MVT::v2f16, MVT::v2bf16}, Action: Legal);
841 // This isn't really legal, but this avoids the legalizer unrolling it (and
842 // allows matching fneg (fabs x) patterns)
843 setOperationAction(Ops: ISD::FABS, VTs: {MVT::v2f16, MVT::v2bf16}, Action: Legal);
844
845 // Can do this in one BFI plus a constant materialize.
846 setOperationAction(Ops: ISD::FCOPYSIGN,
847 VTs: {MVT::v2f16, MVT::v2bf16, MVT::v4f16, MVT::v4bf16,
848 MVT::v8f16, MVT::v8bf16, MVT::v16f16, MVT::v16bf16,
849 MVT::v32f16, MVT::v32bf16},
850 Action: Custom);
851 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
852 setOperationAction(
853 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
854 VT: MVT::f16, Action: Legal);
855
856 setOperationAction(
857 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
858 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16}, Action: Custom);
859 } else {
860 setOperationAction(
861 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
862 VT: MVT::f16, Action: Custom);
863
864 setOperationAction(Ops: {ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE}, VT: MVT::f16,
865 Action: Legal);
866
867 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE,
868 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
869 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
870 Action: Custom);
871
872 setOperationAction(Ops: {ISD::FMINNUM, ISD::FMAXNUM},
873 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
874 Action: Expand);
875 }
876
877 for (MVT Vec16 :
878 {MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
879 MVT::v16bf16, MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
880 setOperationAction(
881 Ops: {ISD::BUILD_VECTOR, ISD::EXTRACT_VECTOR_ELT, ISD::SCALAR_TO_VECTOR},
882 VT: Vec16, Action: Custom);
883 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec16, Action: Expand);
884 }
885 }
886
887 if (Subtarget->hasVOP3PInsts()) {
888 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::MUL, ISD::SHL, ISD::SRL,
889 ISD::SRA, ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX,
890 ISD::UADDSAT, ISD::USUBSAT, ISD::SADDSAT, ISD::SSUBSAT},
891 VT: MVT::v2i16, Action: Legal);
892
893 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FABS,
894 ISD::FCANONICALIZE},
895 VT: MVT::v2f16, Action: Legal);
896
897 setOperationAction(Ops: ISD::EXTRACT_VECTOR_ELT,
898 VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Action: Custom);
899
900 setOperationAction(Ops: ISD::VECTOR_SHUFFLE,
901 VTs: {MVT::v4f16, MVT::v4i16, MVT::v4bf16, MVT::v8f16,
902 MVT::v8i16, MVT::v8bf16, MVT::v16f16, MVT::v16i16,
903 MVT::v16bf16, MVT::v32f16, MVT::v32i16, MVT::v32bf16},
904 Action: Custom);
905
906 for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16})
907 // Split vector operations.
908 setOperationAction(Ops: {ISD::SHL, ISD::SRA, ISD::SRL, ISD::ADD, ISD::SUB,
909 ISD::MUL, ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN,
910 ISD::UMAX, ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT,
911 ISD::SSUBSAT},
912 VT, Action: Custom);
913
914 for (MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16})
915 // Split vector operations.
916 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FABS,
917 ISD::FCANONICALIZE},
918 VT, Action: Custom);
919
920 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
921 setOperationAction(
922 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
923 VT: MVT::v2f16, Action: Legal);
924 } else {
925 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE}, VT: MVT::v2f16,
926 Action: Legal);
927
928 setOperationAction(
929 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
930 VTs: {MVT::v2f16, MVT::v4f16}, Action: Custom);
931 }
932 setOperationAction(Op: ISD::FEXP, VT: MVT::v2f16, Action: Custom);
933 setOperationAction(Ops: ISD::SELECT, VTs: {MVT::v4i16, MVT::v4f16, MVT::v4bf16},
934 Action: Custom);
935
936 if (Subtarget->hasBF16PackedInsts()) {
937 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMAXNUM, ISD::FMINNUM,
938 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::FMA,
939 ISD::FNEG, ISD::FABS, ISD::FCANONICALIZE},
940 VT: MVT::v2bf16, Action: Legal);
941
942 for (MVT VT : {MVT::v4bf16, MVT::v8bf16, MVT::v16bf16, MVT::v32bf16})
943 // Split vector operations.
944 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FCANONICALIZE,
945 ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM,
946 ISD::FMAXIMUMNUM, ISD::FNEG, ISD::FABS},
947 VT, Action: Custom);
948 }
949
950 if (Subtarget->hasAnyPackedFP32Ops()) {
951 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
952 VT: MVT::v2f32, Action: Legal);
953 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
954 VTs: {MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32},
955 Action: Custom);
956 }
957 if (Subtarget->hasAnyPackedFP64Ops()) {
958 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG,
959 ISD::FCANONICALIZE, ISD::BUILD_VECTOR},
960 VT: MVT::v2f64, Action: Legal);
961 setOperationAction(
962 Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FCANONICALIZE},
963 VTs: {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64}, Action: Custom);
964
965 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
966 setOperationAction(
967 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
968 VT: MVT::v2f64, Action: Legal);
969
970 setOperationAction(
971 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
972 VTs: {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64}, Action: Custom);
973 } else {
974 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE}, VT: MVT::v2f64,
975 Action: Legal);
976 setOperationAction(
977 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
978 VT: MVT::v2f64, Action: Custom);
979 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE, ISD::FMINNUM,
980 ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
981 VTs: {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64},
982 Action: Custom);
983 }
984 }
985
986 if (Subtarget->hasAnyPackedU64Ops()) {
987 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::SHL, ISD::BUILD_VECTOR},
988 VT: MVT::v2i64, Action: Legal);
989 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::SHL},
990 VTs: {MVT::v4i64, MVT::v8i64, MVT::v16i64, MVT::v32i64},
991 Action: Custom);
992 }
993 }
994
995 setOperationAction(Ops: {ISD::FNEG, ISD::FABS}, VT: MVT::v4f16, Action: Custom);
996
997 if (Subtarget->has16BitInsts()) {
998 setOperationAction(Op: ISD::SELECT, VT: MVT::v2i16, Action: Promote);
999 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2i16, DestVT: MVT::i32);
1000 setOperationAction(Op: ISD::SELECT, VT: MVT::v2f16, Action: Promote);
1001 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2f16, DestVT: MVT::i32);
1002 setOperationAction(Op: ISD::SELECT, VT: MVT::v2bf16, Action: Promote);
1003 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2bf16, DestVT: MVT::i32);
1004 } else {
1005 // Legalization hack.
1006 setOperationAction(Ops: ISD::SELECT, VTs: {MVT::v2i16, MVT::v2f16}, Action: Custom);
1007
1008 setOperationAction(Ops: {ISD::FNEG, ISD::FABS}, VT: MVT::v2f16, Action: Custom);
1009 }
1010
1011 setOperationAction(Ops: ISD::SELECT,
1012 VTs: {MVT::v4i16, MVT::v4f16, MVT::v4bf16, MVT::v2i8, MVT::v4i8,
1013 MVT::v8i8, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
1014 MVT::v16i16, MVT::v16f16, MVT::v16bf16, MVT::v32i16,
1015 MVT::v32f16, MVT::v32bf16},
1016 Action: Custom);
1017
1018 setOperationAction(Ops: {ISD::SMULO, ISD::UMULO}, VT: MVT::i64, Action: Custom);
1019
1020 if (Subtarget->useVMulU64Inst())
1021 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Legal);
1022 else if (Subtarget->hasScalarSMulU64())
1023 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Custom);
1024
1025 if (Subtarget->hasMad64_32())
1026 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT: MVT::i32, Action: Custom);
1027
1028 if (Subtarget->hasSafeSmemPrefetch() || Subtarget->hasVmemPrefInsts())
1029 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
1030
1031 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
1032 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM},
1033 VTs: {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16}, Action: Legal);
1034 } else {
1035 // FIXME: For nnan fmaximum, emit the fmaximum3 instead of fmaxnum
1036 if (Subtarget->hasMinimum3Maximum3F32())
1037 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f32, Action: Legal);
1038
1039 if (Subtarget->hasMinimum3Maximum3PKF16()) {
1040 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::v2f16, Action: Legal);
1041
1042 // If only the vector form is available, we need to widen to a vector.
1043 if (!Subtarget->hasMinimum3Maximum3F16())
1044 setOperationPromotedToType(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, OrigVT: MVT::f16,
1045 DestVT: MVT::v2f16);
1046 }
1047 }
1048
1049 if (Subtarget->hasVOP3PInsts()) {
1050 // We want to break these into v2f16 pieces, not scalarize.
1051 setOperationAction(Ops: {ISD::FMINIMUM, ISD::FMAXIMUM},
1052 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
1053 Action: Custom);
1054 }
1055
1056 if (Subtarget->useMinMaxI64Insts())
1057 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT: MVT::i64,
1058 Action: Legal);
1059
1060 setOperationAction(Ops: ISD::INTRINSIC_WO_CHAIN,
1061 VTs: {MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16,
1062 MVT::bf16, MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::i128,
1063 MVT::i8},
1064 Action: Custom);
1065
1066 setOperationAction(Ops: ISD::INTRINSIC_W_CHAIN,
1067 VTs: {MVT::v2f16, MVT::v2i16, MVT::v2bf16, MVT::v3f16,
1068 MVT::v3i16, MVT::v4f16, MVT::v4i16, MVT::v4bf16,
1069 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::Other, MVT::f16,
1070 MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
1071 Action: Custom);
1072
1073 // The s_buffer_load intrinsics accept any result type in IR, but only a few
1074 // of them can be selected. Mark the remaining illegal result types Custom so
1075 // ReplaceNodeResults gets a chance to diagnose them instead of letting the
1076 // type legalizer abort. Its INTRINSIC_WO_CHAIN case dispatches on the
1077 // intrinsic ID, but INTRINSIC_W_CHAIN does not, so remember the types added
1078 // here to keep other chained intrinsics on generic legalization.
1079 for (MVT VT : MVT::all_valuetypes()) {
1080 if (VT.isScalableVector() || isTypeLegal(VT))
1081 continue;
1082 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT, Action: Custom);
1083 if (getOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT) != Custom) {
1084 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT, Action: Custom);
1085 SBufferLoadDiagnosticVTs.set(position: VT.SimpleTy);
1086 }
1087 }
1088
1089 setOperationAction(Ops: ISD::INTRINSIC_VOID,
1090 VTs: {MVT::Other, MVT::v2i16, MVT::v2f16, MVT::v2bf16,
1091 MVT::v3i16, MVT::v3f16, MVT::v4f16, MVT::v4i16,
1092 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
1093 MVT::f16, MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
1094 Action: Custom);
1095
1096 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Custom);
1097 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
1098 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
1099 setOperationAction(Op: ISD::GET_FPENV, VT: MVT::i64, Action: Custom);
1100 setOperationAction(Op: ISD::SET_FPENV, VT: MVT::i64, Action: Custom);
1101
1102 // TODO: Could move this to custom lowering, could benefit from combines on
1103 // extract of relevant bits.
1104 setOperationAction(Op: ISD::GET_FPMODE, VT: MVT::i32, Action: Legal);
1105
1106 setOperationAction(Op: ISD::MUL, VT: MVT::i1, Action: Promote);
1107
1108 if (Subtarget->hasBF16ConversionInsts()) {
1109 setOperationAction(Ops: {ISD::FP_ROUND, ISD::STRICT_FP_ROUND},
1110 VTs: {MVT::bf16, MVT::v2bf16}, Action: Custom);
1111 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2bf16, Action: Legal);
1112 }
1113
1114 if (Subtarget->hasBF16TransInsts()) {
1115 setOperationAction(Ops: {ISD::FEXP2, ISD::FLOG2, ISD::FSQRT}, VT: MVT::bf16, Action: Legal);
1116 }
1117
1118 const bool HasE5M3ConversionInsts =
1119 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
1120 if (Subtarget->hasOCPFP8ConversionInsts() || HasE5M3ConversionInsts) {
1121 setOperationAction(Ops: ISD::CONVERT_FROM_ARBITRARY_FP, VTs: {MVT::f32, MVT::v2f32},
1122 Action: Custom);
1123 setOperationAction(Op: ISD::CONVERT_FROM_ARBITRARY_FP, VT: MVT::v2i8, Action: Custom);
1124
1125 // i8 result promotes to i16, wider vectors split down to v2i8, and v2i8 is
1126 // handled in ReplaceNodeResults before the legalizer splits it per lane.
1127 setOperationAction(Ops: ISD::CONVERT_TO_ARBITRARY_FP, VTs: {MVT::i16, MVT::v2i8},
1128 Action: Custom);
1129 }
1130
1131 if (Subtarget->hasFP8F16ConversionInsts()) {
1132 setOperationAction(Ops: ISD::CONVERT_FROM_ARBITRARY_FP, VTs: {MVT::f16, MVT::v2f16},
1133 Action: Custom);
1134 }
1135
1136 if (Subtarget->hasCvtPkF16F32Inst()) {
1137 setOperationAction(Ops: ISD::FP_ROUND,
1138 VTs: {MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16},
1139 Action: Custom);
1140 }
1141
1142 setTargetDAGCombine({ISD::ADD,
1143 ISD::PTRADD,
1144 ISD::SUB,
1145 ISD::MUL,
1146 ISD::FADD,
1147 ISD::FSUB,
1148 ISD::FDIV,
1149 ISD::FMUL,
1150 ISD::FMINNUM,
1151 ISD::FMAXNUM,
1152 ISD::FMINNUM_IEEE,
1153 ISD::FMAXNUM_IEEE,
1154 ISD::FMINIMUM,
1155 ISD::FMAXIMUM,
1156 ISD::FMINIMUMNUM,
1157 ISD::FMAXIMUMNUM,
1158 ISD::FMA,
1159 ISD::ABS,
1160 ISD::SMIN,
1161 ISD::SMAX,
1162 ISD::UMIN,
1163 ISD::UMAX,
1164 ISD::SETCC,
1165 ISD::SELECT,
1166 ISD::SMIN,
1167 ISD::SMAX,
1168 ISD::UMIN,
1169 ISD::UMAX,
1170 ISD::USUBSAT,
1171 ISD::UADDSAT,
1172 ISD::AND,
1173 ISD::OR,
1174 ISD::XOR,
1175 ISD::SHL,
1176 ISD::SRL,
1177 ISD::SRA,
1178 ISD::FSHR,
1179 ISD::SINT_TO_FP,
1180 ISD::UINT_TO_FP,
1181 ISD::FCANONICALIZE,
1182 ISD::SCALAR_TO_VECTOR,
1183 ISD::ZERO_EXTEND,
1184 ISD::SIGN_EXTEND_INREG,
1185 ISD::ANY_EXTEND,
1186 ISD::EXTRACT_VECTOR_ELT,
1187 ISD::INSERT_VECTOR_ELT,
1188 ISD::FCOPYSIGN});
1189
1190 if (Subtarget->has16BitInsts() && !Subtarget->hasMed3_16())
1191 setTargetDAGCombine(ISD::FP_ROUND);
1192
1193 // All memory operations. Some folding on the pointer operand is done to help
1194 // matching the constant offsets in the addressing modes.
1195 setTargetDAGCombine({ISD::LOAD,
1196 ISD::STORE,
1197 ISD::ATOMIC_LOAD,
1198 ISD::ATOMIC_STORE,
1199 ISD::ATOMIC_CMP_SWAP,
1200 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
1201 ISD::ATOMIC_SWAP,
1202 ISD::ATOMIC_LOAD_ADD,
1203 ISD::ATOMIC_LOAD_SUB,
1204 ISD::ATOMIC_LOAD_AND,
1205 ISD::ATOMIC_LOAD_OR,
1206 ISD::ATOMIC_LOAD_XOR,
1207 ISD::ATOMIC_LOAD_NAND,
1208 ISD::ATOMIC_LOAD_MIN,
1209 ISD::ATOMIC_LOAD_MAX,
1210 ISD::ATOMIC_LOAD_UMIN,
1211 ISD::ATOMIC_LOAD_UMAX,
1212 ISD::ATOMIC_LOAD_FADD,
1213 ISD::ATOMIC_LOAD_FMIN,
1214 ISD::ATOMIC_LOAD_FMAX,
1215 ISD::ATOMIC_LOAD_UINC_WRAP,
1216 ISD::ATOMIC_LOAD_UDEC_WRAP,
1217 ISD::ATOMIC_LOAD_USUB_COND,
1218 ISD::ATOMIC_LOAD_USUB_SAT,
1219 ISD::INTRINSIC_VOID,
1220 ISD::INTRINSIC_W_CHAIN});
1221
1222 // FIXME: In other contexts we pretend this is a per-function property.
1223 setStackPointerRegisterToSaveRestore(AMDGPU::SGPR32);
1224
1225 setSchedulingPreference(Sched::RegPressure);
1226}
1227
1228const GCNSubtarget *SITargetLowering::getSubtarget() const { return Subtarget; }
1229
1230ArrayRef<MCPhysReg> SITargetLowering::getRoundingControlRegisters() const {
1231 static const MCPhysReg RCRegs[] = {AMDGPU::MODE};
1232 return RCRegs;
1233}
1234
1235//===----------------------------------------------------------------------===//
1236// TargetLowering queries
1237//===----------------------------------------------------------------------===//
1238
1239// v_mad_mix* support a conversion from f16 to f32.
1240//
1241// There is only one special case when denormals are enabled we don't currently,
1242// where this is OK to use.
1243bool SITargetLowering::isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
1244 EVT DestVT, EVT SrcVT) const {
1245 return DestVT.getScalarType() == MVT::f32 &&
1246 ((((Opcode == ISD::FMAD && Subtarget->hasMadMixInsts()) ||
1247 (Opcode == ISD::FMA && Subtarget->hasFmaMixInsts())) &&
1248 SrcVT.getScalarType() == MVT::f16) ||
1249 (Opcode == ISD::FMA && Subtarget->hasFmaMixBF16Insts() &&
1250 SrcVT.getScalarType() == MVT::bf16)) &&
1251 // TODO: This probably only requires no input flushing?
1252 denormalModeIsFlushAllF32(MF: DAG.getMachineFunction());
1253}
1254
1255bool SITargetLowering::isFPExtFoldable(const MachineInstr &MI, unsigned Opcode,
1256 LLT DestTy, LLT SrcTy) const {
1257 return ((Opcode == TargetOpcode::G_FMAD && Subtarget->hasMadMixInsts()) ||
1258 (Opcode == TargetOpcode::G_FMA && Subtarget->hasFmaMixInsts())) &&
1259 DestTy.getScalarSizeInBits() == 32 &&
1260 SrcTy.getScalarSizeInBits() == 16 &&
1261 // TODO: This probably only requires no input flushing?
1262 denormalModeIsFlushAllF32(MF: *MI.getMF());
1263}
1264
1265bool SITargetLowering::isShuffleMaskLegal(ArrayRef<int>, EVT) const {
1266 // SI has some legal vector types, but no legal vector operations. Say no
1267 // shuffles are legal in order to prefer scalarizing some vector operations.
1268 return false;
1269}
1270
1271MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
1272 CallingConv::ID CC,
1273 EVT VT) const {
1274 if (CC == CallingConv::AMDGPU_KERNEL)
1275 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1276
1277 if (VT.isVector()) {
1278 EVT ScalarVT = VT.getScalarType();
1279 unsigned Size = ScalarVT.getSizeInBits();
1280 if (Size == 16) {
1281 return Subtarget->has16BitInsts()
1282 ? MVT::getVectorVT(VT: ScalarVT.getSimpleVT(), NumElements: 2)
1283 : MVT::i32;
1284 }
1285
1286 if (Size < 16)
1287 return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32;
1288 return Size == 32 ? ScalarVT.getSimpleVT() : MVT::i32;
1289 }
1290
1291 if (!Subtarget->has16BitInsts() && VT.getSizeInBits() == 16)
1292 return MVT::i32;
1293
1294 if (VT.getSizeInBits() > 32)
1295 return MVT::i32;
1296
1297 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1298}
1299
1300unsigned SITargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
1301 CallingConv::ID CC,
1302 EVT VT) const {
1303 if (CC == CallingConv::AMDGPU_KERNEL)
1304 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1305
1306 if (VT.isVector()) {
1307 unsigned NumElts = VT.getVectorNumElements();
1308 EVT ScalarVT = VT.getScalarType();
1309 unsigned Size = ScalarVT.getSizeInBits();
1310
1311 // FIXME: Should probably promote 8-bit vectors to i16.
1312 if (Size == 16)
1313 return (NumElts + 1) / 2;
1314
1315 if (Size <= 32)
1316 return NumElts;
1317
1318 if (Size > 32)
1319 return NumElts * ((Size + 31) / 32);
1320 } else if (VT.getSizeInBits() > 32)
1321 return (VT.getSizeInBits() + 31) / 32;
1322
1323 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1324}
1325
1326unsigned SITargetLowering::getVectorTypeBreakdownForCallingConv(
1327 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
1328 unsigned &NumIntermediates, MVT &RegisterVT) const {
1329 if (CC != CallingConv::AMDGPU_KERNEL && VT.isVector()) {
1330 unsigned NumElts = VT.getVectorNumElements();
1331 EVT ScalarVT = VT.getScalarType();
1332 unsigned Size = ScalarVT.getSizeInBits();
1333 // FIXME: We should fix the ABI to be the same on targets without 16-bit
1334 // support, but unless we can properly handle 3-vectors, it will be still be
1335 // inconsistent.
1336 if (Size == 16) {
1337 MVT SimpleIntermediateVT =
1338 MVT::getVectorVT(VT: ScalarVT.getSimpleVT(), EC: ElementCount::getFixed(MinVal: 2));
1339 IntermediateVT = SimpleIntermediateVT;
1340 RegisterVT = Subtarget->has16BitInsts() ? SimpleIntermediateVT : MVT::i32;
1341 NumIntermediates = (NumElts + 1) / 2;
1342 return (NumElts + 1) / 2;
1343 }
1344
1345 if (Size == 32) {
1346 RegisterVT = ScalarVT.getSimpleVT();
1347 IntermediateVT = RegisterVT;
1348 NumIntermediates = NumElts;
1349 return NumIntermediates;
1350 }
1351
1352 if (Size < 16 && Subtarget->has16BitInsts()) {
1353 // FIXME: Should probably form v2i16 pieces
1354 RegisterVT = MVT::i16;
1355 IntermediateVT = ScalarVT;
1356 NumIntermediates = NumElts;
1357 return NumIntermediates;
1358 }
1359
1360 if (Size != 16 && Size <= 32) {
1361 RegisterVT = MVT::i32;
1362 IntermediateVT = ScalarVT;
1363 NumIntermediates = NumElts;
1364 return NumIntermediates;
1365 }
1366
1367 if (Size > 32) {
1368 RegisterVT = MVT::i32;
1369 IntermediateVT = RegisterVT;
1370 NumIntermediates = NumElts * ((Size + 31) / 32);
1371 return NumIntermediates;
1372 }
1373 }
1374
1375 return TargetLowering::getVectorTypeBreakdownForCallingConv(
1376 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
1377}
1378
1379static EVT memVTFromLoadIntrData(const SITargetLowering &TLI,
1380 const DataLayout &DL, Type *Ty,
1381 unsigned MaxNumLanes) {
1382 assert(MaxNumLanes != 0);
1383
1384 LLVMContext &Ctx = Ty->getContext();
1385 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
1386 unsigned NumElts = std::min(a: MaxNumLanes, b: VT->getNumElements());
1387 return EVT::getVectorVT(Context&: Ctx, VT: TLI.getValueType(DL, Ty: VT->getElementType()),
1388 NumElements: NumElts);
1389 }
1390
1391 return TLI.getValueType(DL, Ty);
1392}
1393
1394// Peek through TFE struct returns to only use the data size.
1395static EVT memVTFromLoadIntrReturn(const SITargetLowering &TLI,
1396 const DataLayout &DL, Type *Ty,
1397 unsigned MaxNumLanes) {
1398 auto *ST = dyn_cast<StructType>(Val: Ty);
1399 if (!ST)
1400 return memVTFromLoadIntrData(TLI, DL, Ty, MaxNumLanes);
1401
1402 // TFE intrinsics return an aggregate type.
1403 assert(ST->getNumContainedTypes() == 2 &&
1404 ST->getContainedType(1)->isIntegerTy(32));
1405 return memVTFromLoadIntrData(TLI, DL, Ty: ST->getContainedType(i: 0), MaxNumLanes);
1406}
1407
1408/// Map address space 7 to MVT::amdgpuBufferFatPointer because that's its
1409/// in-memory representation. This return value is a custom type because there
1410/// is no MVT::i160 and adding one breaks integer promotion logic. While this
1411/// could cause issues during codegen, these address space 7 pointers will be
1412/// rewritten away by then. Therefore, we can return MVT::amdgpuBufferFatPointer
1413/// in order to allow pre-codegen passes that query TargetTransformInfo, often
1414/// for cost modeling, to work. (This also sets us up decently for doing the
1415/// buffer lowering in GlobalISel if SelectionDAG ever goes away.)
1416MVT SITargetLowering::getPointerTy(const DataLayout &DL, unsigned AS) const {
1417 if (AMDGPUAS::BUFFER_FAT_POINTER == AS && DL.getPointerSizeInBits(AS) == 160)
1418 return MVT::amdgpuBufferFatPointer;
1419 if (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
1420 DL.getPointerSizeInBits(AS) == 192)
1421 return MVT::amdgpuBufferStridedPointer;
1422 return AMDGPUTargetLowering::getPointerTy(DL, AS);
1423}
1424/// Similarly, the in-memory representation of a p7 is {p8, i32}, aka
1425/// v8i32 when padding is added.
1426/// The in-memory representation of a p9 is {p8, i32, i32}, which is
1427/// also v8i32 with padding.
1428MVT SITargetLowering::getPointerMemTy(const DataLayout &DL, unsigned AS) const {
1429 if ((AMDGPUAS::BUFFER_FAT_POINTER == AS &&
1430 DL.getPointerSizeInBits(AS) == 160) ||
1431 (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
1432 DL.getPointerSizeInBits(AS) == 192))
1433 return MVT::v8i32;
1434 return AMDGPUTargetLowering::getPointerMemTy(DL, AS);
1435}
1436
1437static unsigned getIntrMemWidth(unsigned IntrID) {
1438 switch (IntrID) {
1439 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1440 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1441 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1442 return 8;
1443 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1444 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1445 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1446 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
1447 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
1448 case Intrinsic::amdgcn_flat_load_monitor_b32:
1449 case Intrinsic::amdgcn_global_load_monitor_b32:
1450 return 32;
1451 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1452 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1453 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1454 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
1455 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
1456 case Intrinsic::amdgcn_flat_load_monitor_b64:
1457 case Intrinsic::amdgcn_global_load_monitor_b64:
1458 return 64;
1459 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1460 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
1461 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
1462 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
1463 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
1464 case Intrinsic::amdgcn_flat_load_monitor_b128:
1465 case Intrinsic::amdgcn_global_load_monitor_b128:
1466 return 128;
1467 default:
1468 llvm_unreachable("Unknown width");
1469 }
1470}
1471
1472static AtomicOrdering parseAtomicOrderingCABIArg(const CallBase &CI,
1473 unsigned ArgIdx) {
1474 Value *OrderingArg = CI.getArgOperand(i: ArgIdx);
1475 unsigned Ord = cast<ConstantInt>(Val: OrderingArg)->getZExtValue();
1476 switch (AtomicOrderingCABI(Ord)) {
1477 case AtomicOrderingCABI::acquire:
1478 return AtomicOrdering::Acquire;
1479 break;
1480 case AtomicOrderingCABI::release:
1481 return AtomicOrdering::Release;
1482 break;
1483 case AtomicOrderingCABI::seq_cst:
1484 return AtomicOrdering::SequentiallyConsistent;
1485 break;
1486 default:
1487 return AtomicOrdering::Monotonic;
1488 }
1489}
1490
1491static unsigned parseSyncscopeMDArg(const CallBase &CI, unsigned ArgIdx) {
1492 MDNode *ScopeMD = cast<MDNode>(
1493 Val: cast<MetadataAsValue>(Val: CI.getArgOperand(i: ArgIdx))->getMetadata());
1494 StringRef Scope = cast<MDString>(Val: ScopeMD->getOperand(I: 0))->getString();
1495 return CI.getContext().getOrInsertSyncScopeID(SSN: Scope);
1496}
1497
1498void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
1499 const CallBase &CI,
1500 MachineFunction &MF,
1501 unsigned IntrID) const {
1502 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
1503 if (CI.hasMetadata(KindID: LLVMContext::MD_invariant_load))
1504 Flags |= MachineMemOperand::MOInvariant;
1505 if (CI.hasMetadata(KindID: LLVMContext::MD_nontemporal))
1506 Flags |= MachineMemOperand::MONonTemporal;
1507 Flags |= getTargetMMOFlags(I: CI);
1508
1509 if (const AMDGPU::RsrcIntrinsic *RsrcIntr =
1510 AMDGPU::lookupRsrcIntrinsic(Intr: IntrID)) {
1511 AttributeSet Attr =
1512 Intrinsic::getFnAttributes(C&: CI.getContext(), id: (Intrinsic::ID)IntrID);
1513 MemoryEffects ME = Attr.getMemoryEffects();
1514 if (ME.doesNotAccessMemory())
1515 return;
1516
1517 bool IsSPrefetch = IntrID == Intrinsic::amdgcn_s_buffer_prefetch_data;
1518 if (!IsSPrefetch) {
1519 auto *Aux = cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1));
1520 if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE)
1521 Flags |= MachineMemOperand::MOVolatile;
1522 }
1523
1524 Flags |= MachineMemOperand::MODereferenceable;
1525
1526 IntrinsicInfo Info;
1527 // TODO: Should images get their own address space?
1528 Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
1529
1530 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = nullptr;
1531 if (RsrcIntr->IsImage) {
1532 const AMDGPU::ImageDimIntrinsicInfo *Intr =
1533 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrID);
1534 BaseOpcode = AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Intr->BaseOpcode);
1535 Info.align.reset();
1536 }
1537
1538 Value *RsrcArg = CI.getArgOperand(i: RsrcIntr->RsrcArg);
1539 if (auto *RsrcPtrTy = dyn_cast<PointerType>(Val: RsrcArg->getType())) {
1540 if (RsrcPtrTy->getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE)
1541 // We conservatively set the memory operand of a buffer intrinsic to the
1542 // base resource pointer, so that we can access alias information about
1543 // those pointers. Cases like "this points at the same value
1544 // but with a different offset" are handled in
1545 // areMemAccessesTriviallyDisjoint.
1546 Info.ptrVal = RsrcArg;
1547 }
1548
1549 if (ME.onlyReadsMemory()) {
1550 if (RsrcIntr->IsImage) {
1551 unsigned MaxNumLanes = 4;
1552
1553 if (!BaseOpcode->Gather4) {
1554 // If this isn't a gather, we may have excess loaded elements in the
1555 // IR type. Check the dmask for the real number of elements loaded.
1556 unsigned DMask =
1557 cast<ConstantInt>(Val: CI.getArgOperand(i: 0))->getZExtValue();
1558 MaxNumLanes = DMask == 0 ? 1 : llvm::popcount(Value: DMask);
1559 }
1560
1561 Info.memVT = memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(),
1562 Ty: CI.getType(), MaxNumLanes);
1563 } else {
1564 Info.memVT =
1565 memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(), Ty: CI.getType(),
1566 MaxNumLanes: std::numeric_limits<unsigned>::max());
1567 }
1568
1569 // FIXME: What does alignment mean for an image?
1570 Info.opc = ISD::INTRINSIC_W_CHAIN;
1571 Info.flags = Flags | MachineMemOperand::MOLoad;
1572 } else if (ME.onlyWritesMemory()) {
1573 Info.opc = ISD::INTRINSIC_VOID;
1574
1575 Type *DataTy = CI.getArgOperand(i: 0)->getType();
1576 if (RsrcIntr->IsImage) {
1577 unsigned DMask = cast<ConstantInt>(Val: CI.getArgOperand(i: 1))->getZExtValue();
1578 unsigned DMaskLanes = DMask == 0 ? 1 : llvm::popcount(Value: DMask);
1579 Info.memVT = memVTFromLoadIntrData(TLI: *this, DL: MF.getDataLayout(), Ty: DataTy,
1580 MaxNumLanes: DMaskLanes);
1581 } else
1582 Info.memVT = getValueType(DL: MF.getDataLayout(), Ty: DataTy);
1583
1584 Info.flags = Flags | MachineMemOperand::MOStore;
1585 } else {
1586 // Atomic, NoReturn Sampler or prefetch
1587 Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID
1588 : ISD::INTRINSIC_W_CHAIN;
1589
1590 switch (IntrID) {
1591 default:
1592 Info.flags = Flags | MachineMemOperand::MOLoad;
1593 if (!IsSPrefetch)
1594 Info.flags |= MachineMemOperand::MOStore;
1595
1596 if ((RsrcIntr->IsImage && BaseOpcode->NoReturn) || IsSPrefetch) {
1597 // Fake memory access type for no return sampler intrinsics
1598 Info.memVT = MVT::i32;
1599 } else {
1600 // XXX - Should this be volatile without known ordering?
1601 Info.flags |= MachineMemOperand::MOVolatile;
1602 Info.memVT = MVT::getVT(Ty: CI.getArgOperand(i: 0)->getType());
1603 }
1604 break;
1605 case Intrinsic::amdgcn_raw_buffer_load_lds:
1606 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
1607 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
1608 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
1609 case Intrinsic::amdgcn_struct_buffer_load_lds:
1610 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
1611 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
1612 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
1613 unsigned Width = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getZExtValue();
1614
1615 // Entry 0: Load from buffer.
1616 // Don't set an offset, since the pointer value always represents the
1617 // base of the buffer.
1618 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: Width * 8);
1619 Info.flags = Flags | MachineMemOperand::MOLoad;
1620 Infos.push_back(Elt: Info);
1621
1622 // Entry 1: Store to LDS.
1623 // Instruction offset is applied, and an additional per-lane offset
1624 // which we simulate using a larger memory type.
1625 Info.memVT = EVT::getIntegerVT(
1626 Context&: CI.getContext(), BitWidth: Width * 8 * Subtarget->getWavefrontSize());
1627 Info.ptrVal = CI.getArgOperand(i: 1); // LDS destination pointer
1628 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 2))
1629 ->getZExtValue();
1630 Info.fallbackAddressSpace = AMDGPUAS::LOCAL_ADDRESS;
1631 Info.flags = Flags | MachineMemOperand::MOStore;
1632 Infos.push_back(Elt: Info);
1633 return;
1634 }
1635 case Intrinsic::amdgcn_raw_atomic_buffer_load:
1636 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
1637 case Intrinsic::amdgcn_struct_atomic_buffer_load:
1638 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
1639 Info.memVT =
1640 memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(), Ty: CI.getType(),
1641 MaxNumLanes: std::numeric_limits<unsigned>::max());
1642 Info.flags = Flags | MachineMemOperand::MOLoad;
1643 Infos.push_back(Elt: Info);
1644 return;
1645 }
1646 }
1647 }
1648 Infos.push_back(Elt: Info);
1649 return;
1650 }
1651
1652 IntrinsicInfo Info;
1653 switch (IntrID) {
1654 case Intrinsic::amdgcn_ds_ordered_add:
1655 case Intrinsic::amdgcn_ds_ordered_swap: {
1656 Info.opc = ISD::INTRINSIC_W_CHAIN;
1657 Info.memVT = MVT::getVT(Ty: CI.getType());
1658 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1659 Info.align.reset();
1660 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1661
1662 const ConstantInt *Vol = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 4));
1663 if (!Vol->isZero())
1664 Info.flags |= MachineMemOperand::MOVolatile;
1665
1666 Infos.push_back(Elt: Info);
1667 return;
1668 }
1669 case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
1670 case Intrinsic::amdgcn_ds_sub_gs_reg_rtn: {
1671 Info.opc = ISD::INTRINSIC_W_CHAIN;
1672 Info.memVT = MVT::getVT(Ty: CI.getOperand(i_nocapture: 0)->getType());
1673 Info.ptrVal = nullptr;
1674 Info.fallbackAddressSpace = AMDGPUAS::STREAMOUT_REGISTER;
1675 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1676 Infos.push_back(Elt: Info);
1677 return;
1678 }
1679 case Intrinsic::amdgcn_ds_append:
1680 case Intrinsic::amdgcn_ds_consume: {
1681 Info.opc = ISD::INTRINSIC_W_CHAIN;
1682 Info.memVT = MVT::getVT(Ty: CI.getType());
1683 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1684 Info.align.reset();
1685 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1686
1687 const ConstantInt *Vol = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1));
1688 if (!Vol->isZero())
1689 Info.flags |= MachineMemOperand::MOVolatile;
1690
1691 Infos.push_back(Elt: Info);
1692 return;
1693 }
1694 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
1695 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64: {
1696 Info.opc = (IntrID == Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64)
1697 ? ISD::INTRINSIC_W_CHAIN
1698 : ISD::INTRINSIC_VOID;
1699 Info.memVT = MVT::getVT(Ty: CI.getType());
1700 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1701 Info.memVT = MVT::i64;
1702 Info.size = 8;
1703 Info.align.reset();
1704 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1705 Info.order = AtomicOrdering::Monotonic;
1706 Infos.push_back(Elt: Info);
1707 return;
1708 }
1709 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
1710 case Intrinsic::amdgcn_image_bvh_intersect_ray:
1711 case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
1712 Info.opc = ISD::INTRINSIC_W_CHAIN;
1713 Info.memVT =
1714 MVT::getVT(Ty: IntrID == Intrinsic::amdgcn_image_bvh_intersect_ray
1715 ? CI.getType()
1716 : cast<StructType>(Val: CI.getType())
1717 ->getElementType(N: 0)); // XXX: what is correct VT?
1718
1719 Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
1720 Info.align.reset();
1721 Info.flags = Flags | MachineMemOperand::MOLoad |
1722 MachineMemOperand::MODereferenceable;
1723 Infos.push_back(Elt: Info);
1724 return;
1725 }
1726 case Intrinsic::amdgcn_global_atomic_fmin_num:
1727 case Intrinsic::amdgcn_global_atomic_fmax_num:
1728 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1729 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1730 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
1731 Info.opc = ISD::INTRINSIC_W_CHAIN;
1732 Info.memVT = MVT::getVT(Ty: CI.getType());
1733 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1734 Info.align.reset();
1735 Info.flags =
1736 Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
1737 MachineMemOperand::MODereferenceable | MachineMemOperand::MOVolatile;
1738 Infos.push_back(Elt: Info);
1739 return;
1740 }
1741 case Intrinsic::amdgcn_cluster_load_b32:
1742 case Intrinsic::amdgcn_cluster_load_b64:
1743 case Intrinsic::amdgcn_cluster_load_b128:
1744 case Intrinsic::amdgcn_ds_load_tr6_b96:
1745 case Intrinsic::amdgcn_ds_load_tr4_b64:
1746 case Intrinsic::amdgcn_ds_load_tr8_b64:
1747 case Intrinsic::amdgcn_ds_load_tr16_b128:
1748 case Intrinsic::amdgcn_global_load_tr6_b96:
1749 case Intrinsic::amdgcn_global_load_tr4_b64:
1750 case Intrinsic::amdgcn_global_load_tr_b64:
1751 case Intrinsic::amdgcn_global_load_tr_b128:
1752 case Intrinsic::amdgcn_ds_read_tr4_b64:
1753 case Intrinsic::amdgcn_ds_read_tr6_b96:
1754 case Intrinsic::amdgcn_ds_read_tr8_b64:
1755 case Intrinsic::amdgcn_ds_read_tr16_b64: {
1756 Info.opc = ISD::INTRINSIC_W_CHAIN;
1757 Info.memVT = MVT::getVT(Ty: CI.getType());
1758 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1759 Info.align.reset();
1760 Info.flags = Flags | MachineMemOperand::MOLoad;
1761 Infos.push_back(Elt: Info);
1762 return;
1763 }
1764 case Intrinsic::amdgcn_flat_load_monitor_b32:
1765 case Intrinsic::amdgcn_flat_load_monitor_b64:
1766 case Intrinsic::amdgcn_flat_load_monitor_b128:
1767 case Intrinsic::amdgcn_global_load_monitor_b32:
1768 case Intrinsic::amdgcn_global_load_monitor_b64:
1769 case Intrinsic::amdgcn_global_load_monitor_b128: {
1770 Info.opc = ISD::INTRINSIC_W_CHAIN;
1771 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1772 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1773 Info.align.reset();
1774 Info.flags = MachineMemOperand::MOLoad;
1775 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 1);
1776 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 2);
1777 Infos.push_back(Elt: Info);
1778 return;
1779 }
1780 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
1781 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
1782 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
1783 Info.opc = ISD::INTRINSIC_W_CHAIN;
1784 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1785 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1786 Info.align.reset();
1787 Info.flags = (MachineMemOperand::MOLoad | MOCooperative);
1788 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 1);
1789 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 2);
1790 Infos.push_back(Elt: Info);
1791 return;
1792 }
1793 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
1794 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
1795 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
1796 Info.opc = ISD::INTRINSIC_VOID;
1797 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1798 Info.ptrVal = CI.getArgOperand(i: 0);
1799 Info.align.reset();
1800 Info.flags = (MachineMemOperand::MOStore | MOCooperative);
1801 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 2);
1802 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 3);
1803 Infos.push_back(Elt: Info);
1804 return;
1805 }
1806 case Intrinsic::amdgcn_ds_gws_init:
1807 case Intrinsic::amdgcn_ds_gws_barrier:
1808 case Intrinsic::amdgcn_ds_gws_sema_v:
1809 case Intrinsic::amdgcn_ds_gws_sema_br:
1810 case Intrinsic::amdgcn_ds_gws_sema_p:
1811 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
1812 Info.opc = ISD::INTRINSIC_VOID;
1813
1814 const GCNTargetMachine &TM =
1815 static_cast<const GCNTargetMachine &>(getTargetMachine());
1816
1817 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1818 Info.ptrVal = MFI->getGWSPSV(TM);
1819
1820 // This is an abstract access, but we need to specify a type and size.
1821 Info.memVT = MVT::i32;
1822 Info.size = 4;
1823 Info.align = Align(4);
1824
1825 if (IntrID == Intrinsic::amdgcn_ds_gws_barrier)
1826 Info.flags = Flags | MachineMemOperand::MOLoad;
1827 else
1828 Info.flags = Flags | MachineMemOperand::MOStore;
1829 Infos.push_back(Elt: Info);
1830 return;
1831 }
1832 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1833 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1834 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1835 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1836 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1837 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1838 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1839 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
1840 // Entry 0: Load from source (global/flat).
1841 Info.opc = ISD::INTRINSIC_VOID;
1842 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1843 Info.ptrVal = CI.getArgOperand(i: 0); // Global pointer
1844 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getSExtValue();
1845 Info.flags = Flags | MachineMemOperand::MOLoad;
1846 Infos.push_back(Elt: Info);
1847
1848 // Entry 1: Store to LDS (same offset).
1849 Info.flags = Flags | MachineMemOperand::MOStore;
1850 Info.ptrVal = CI.getArgOperand(i: 1); // LDS pointer
1851 Infos.push_back(Elt: Info);
1852 return;
1853 }
1854 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1855 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1856 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1857 case Intrinsic::amdgcn_global_store_async_from_lds_b128: {
1858 // Entry 0: Load from LDS.
1859 Info.opc = ISD::INTRINSIC_VOID;
1860 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1861 Info.ptrVal = CI.getArgOperand(i: 1); // LDS pointer
1862 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getSExtValue();
1863 Info.flags = Flags | MachineMemOperand::MOLoad;
1864 Infos.push_back(Elt: Info);
1865
1866 // Entry 1: Store to global (same offset).
1867 Info.flags = Flags | MachineMemOperand::MOStore;
1868 Info.ptrVal = CI.getArgOperand(i: 0); // Global pointer
1869 Infos.push_back(Elt: Info);
1870 return;
1871 }
1872 case Intrinsic::amdgcn_av_load_b128:
1873 case Intrinsic::amdgcn_av_store_b128: {
1874 bool IsStore = IntrID == Intrinsic::amdgcn_av_store_b128;
1875 Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
1876 Info.memVT = MVT::v4i32;
1877 Info.ptrVal = CI.getArgOperand(i: 0);
1878 Info.align = Align(16);
1879 Info.flags |=
1880 IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
1881 // Pretend to be atomic so that SIMemoryLegalizer::expandStore sets cache
1882 // flags appropriately.
1883 Info.order = AtomicOrdering::Monotonic;
1884
1885 LLVMContext &Ctx = CI.getContext();
1886 unsigned ScopeIdx = CI.arg_size() - 1;
1887 MDNode *ScopeMD = cast<MDNode>(
1888 Val: cast<MetadataAsValue>(Val: CI.getArgOperand(i: ScopeIdx))->getMetadata());
1889 StringRef Scope = cast<MDString>(Val: ScopeMD->getOperand(I: 0))->getString();
1890 Info.ssid = Ctx.getOrInsertSyncScopeID(SSN: Scope);
1891 Infos.push_back(Elt: Info);
1892 return;
1893 }
1894 case Intrinsic::amdgcn_load_to_lds:
1895 case Intrinsic::amdgcn_load_async_to_lds:
1896 case Intrinsic::amdgcn_global_load_lds:
1897 case Intrinsic::amdgcn_global_load_async_lds: {
1898 unsigned Width = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getZExtValue();
1899 auto *Aux = cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1));
1900 bool IsVolatile = Aux->getZExtValue() & AMDGPU::CPol::VOLATILE;
1901 if (IsVolatile)
1902 Flags |= MachineMemOperand::MOVolatile;
1903
1904 // Entry 0: Load from source (global/flat).
1905 Info.opc = ISD::INTRINSIC_VOID;
1906 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: Width * 8);
1907 Info.ptrVal = CI.getArgOperand(i: 0); // Source pointer
1908 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 3))->getSExtValue();
1909 Info.flags = Flags | MachineMemOperand::MOLoad;
1910 Infos.push_back(Elt: Info);
1911
1912 // Entry 1: Store to LDS.
1913 // Same offset from the instruction, but an additional per-lane offset is
1914 // added. Represent that using a wider memory type.
1915 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(),
1916 BitWidth: Width * 8 * Subtarget->getWavefrontSize());
1917 Info.ptrVal = CI.getArgOperand(i: 1); // LDS destination pointer
1918 Info.flags = Flags | MachineMemOperand::MOStore;
1919 Infos.push_back(Elt: Info);
1920 return;
1921 }
1922 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
1923 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
1924 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
1925 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn: {
1926 Info.opc = ISD::INTRINSIC_W_CHAIN;
1927
1928 const GCNTargetMachine &TM =
1929 static_cast<const GCNTargetMachine &>(getTargetMachine());
1930
1931 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1932 Info.ptrVal = MFI->getGWSPSV(TM);
1933
1934 // This is an abstract access, but we need to specify a type and size.
1935 Info.memVT = MVT::i32;
1936 Info.size = 4;
1937 Info.align = Align(4);
1938
1939 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1940 Infos.push_back(Elt: Info);
1941 return;
1942 }
1943 case Intrinsic::amdgcn_s_prefetch_data:
1944 case Intrinsic::amdgcn_s_prefetch_inst:
1945 case Intrinsic::amdgcn_flat_prefetch:
1946 case Intrinsic::amdgcn_global_prefetch: {
1947 Info.opc = ISD::INTRINSIC_VOID;
1948 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: 8);
1949 Info.ptrVal = CI.getArgOperand(i: 0);
1950 Info.flags = Flags | MachineMemOperand::MOLoad;
1951 Infos.push_back(Elt: Info);
1952 return;
1953 }
1954 default:
1955 return;
1956 }
1957}
1958
1959bool SITargetLowering::getAddrModeArguments(const IntrinsicInst *II,
1960 SmallVectorImpl<Value *> &Ops,
1961 Type *&AccessTy) const {
1962 Value *Ptr = nullptr;
1963 switch (II->getIntrinsicID()) {
1964 case Intrinsic::amdgcn_cluster_load_b128:
1965 case Intrinsic::amdgcn_cluster_load_b64:
1966 case Intrinsic::amdgcn_cluster_load_b32:
1967 case Intrinsic::amdgcn_ds_append:
1968 case Intrinsic::amdgcn_ds_consume:
1969 case Intrinsic::amdgcn_ds_load_tr8_b64:
1970 case Intrinsic::amdgcn_ds_load_tr16_b128:
1971 case Intrinsic::amdgcn_ds_load_tr4_b64:
1972 case Intrinsic::amdgcn_ds_load_tr6_b96:
1973 case Intrinsic::amdgcn_ds_read_tr4_b64:
1974 case Intrinsic::amdgcn_ds_read_tr6_b96:
1975 case Intrinsic::amdgcn_ds_read_tr8_b64:
1976 case Intrinsic::amdgcn_ds_read_tr16_b64:
1977 case Intrinsic::amdgcn_ds_ordered_add:
1978 case Intrinsic::amdgcn_ds_ordered_swap:
1979 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
1980 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
1981 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1982 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1983 case Intrinsic::amdgcn_global_atomic_fmax_num:
1984 case Intrinsic::amdgcn_global_atomic_fmin_num:
1985 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1986 case Intrinsic::amdgcn_global_load_tr_b64:
1987 case Intrinsic::amdgcn_global_load_tr_b128:
1988 case Intrinsic::amdgcn_global_load_tr4_b64:
1989 case Intrinsic::amdgcn_global_load_tr6_b96:
1990 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1991 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1992 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1993 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
1994 case Intrinsic::amdgcn_av_load_b128:
1995 case Intrinsic::amdgcn_av_store_b128:
1996 Ptr = II->getArgOperand(i: 0);
1997 break;
1998 case Intrinsic::amdgcn_load_to_lds:
1999 case Intrinsic::amdgcn_load_async_to_lds:
2000 case Intrinsic::amdgcn_global_load_lds:
2001 case Intrinsic::amdgcn_global_load_async_lds:
2002 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
2003 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
2004 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
2005 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
2006 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
2007 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
2008 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
2009 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
2010 Ptr = II->getArgOperand(i: 1);
2011 break;
2012 default:
2013 return false;
2014 }
2015 AccessTy = II->getType();
2016 Ops.push_back(Elt: Ptr);
2017 return true;
2018}
2019
2020bool SITargetLowering::isLegalFlatAddressingMode(const AddrMode &AM,
2021 unsigned AddrSpace) const {
2022 if (!Subtarget->hasFlatInstOffsets()) {
2023 // Flat instructions do not have offsets, and only have the register
2024 // address.
2025 return AM.BaseOffs == 0 && AM.Scale == 0;
2026 }
2027
2028 using AMDGPU::FlatAddrSpace;
2029 FlatAddrSpace FlatVariant =
2030 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS ? FlatAddrSpace::FlatGlobal
2031 : AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ? FlatAddrSpace::FlatScratch
2032 : FlatAddrSpace::FLAT;
2033
2034 return AM.Scale == 0 &&
2035 (AM.BaseOffs == 0 || Subtarget->getInstrInfo()->isLegalFLATOffset(
2036 Offset: AM.BaseOffs, AddrSpace, FlatVariant));
2037}
2038
2039bool SITargetLowering::isLegalGlobalAddressingMode(const AddrMode &AM) const {
2040 if (Subtarget->hasFlatGlobalInsts())
2041 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS);
2042
2043 if (!Subtarget->hasAddr64() || Subtarget->useFlatForGlobal()) {
2044 // Assume the we will use FLAT for all global memory accesses
2045 // on VI.
2046 // FIXME: This assumption is currently wrong. On VI we still use
2047 // MUBUF instructions for the r + i addressing mode. As currently
2048 // implemented, the MUBUF instructions only work on buffer < 4GB.
2049 // It may be possible to support > 4GB buffers with MUBUF instructions,
2050 // by setting the stride value in the resource descriptor which would
2051 // increase the size limit to (stride * 4GB). However, this is risky,
2052 // because it has never been validated.
2053 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::FLAT_ADDRESS);
2054 }
2055
2056 return isLegalMUBUFAddressingMode(AM);
2057}
2058
2059bool SITargetLowering::isLegalMUBUFAddressingMode(const AddrMode &AM) const {
2060 // MUBUF / MTBUF instructions have a 12-bit unsigned byte offset, and
2061 // additionally can do r + r + i with addr64. 32-bit has more addressing
2062 // mode options. Depending on the resource constant, it can also do
2063 // (i64 r0) + (i32 r1) * (i14 i).
2064 //
2065 // Private arrays end up using a scratch buffer most of the time, so also
2066 // assume those use MUBUF instructions. Scratch loads / stores are currently
2067 // implemented as mubuf instructions with offen bit set, so slightly
2068 // different than the normal addr64.
2069 const SIInstrInfo *TII = Subtarget->getInstrInfo();
2070 if (!TII->isLegalMUBUFImmOffset(Imm: AM.BaseOffs))
2071 return false;
2072
2073 // FIXME: Since we can split immediate into soffset and immediate offset,
2074 // would it make sense to allow any immediate?
2075
2076 switch (AM.Scale) {
2077 case 0: // r + i or just i, depending on HasBaseReg.
2078 return true;
2079 case 1:
2080 return true; // We have r + r or r + i.
2081 case 2:
2082 if (AM.HasBaseReg) {
2083 // Reject 2 * r + r.
2084 return false;
2085 }
2086
2087 // Allow 2 * r as r + r
2088 // Or 2 * r + i is allowed as r + r + i.
2089 return true;
2090 default: // Don't allow n * r
2091 return false;
2092 }
2093}
2094
2095bool SITargetLowering::isLegalAddressingMode(const DataLayout &DL,
2096 const AddrMode &AM, Type *Ty,
2097 unsigned AS,
2098 Instruction *I) const {
2099 // No global is ever allowed as a base.
2100 if (AM.BaseGV)
2101 return false;
2102
2103 if (AS == AMDGPUAS::GLOBAL_ADDRESS)
2104 return isLegalGlobalAddressingMode(AM);
2105
2106 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
2107 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
2108 AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE ||
2109 AS == AMDGPUAS::BUFFER_STRIDED_POINTER) {
2110 // If the offset isn't a multiple of 4, it probably isn't going to be
2111 // correctly aligned.
2112 // FIXME: Can we get the real alignment here?
2113 if (AM.BaseOffs % 4 != 0)
2114 return isLegalMUBUFAddressingMode(AM);
2115
2116 if (!Subtarget->hasScalarSubwordLoads()) {
2117 // There are no SMRD extloads, so if we have to do a small type access we
2118 // will use a MUBUF load.
2119 // FIXME?: We also need to do this if unaligned, but we don't know the
2120 // alignment here.
2121 if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4)
2122 return isLegalGlobalAddressingMode(AM);
2123 }
2124
2125 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) {
2126 // SMRD instructions have an 8-bit, dword offset on SI.
2127 if (!isUInt<8>(x: AM.BaseOffs / 4))
2128 return false;
2129 } else if (Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS) {
2130 // On CI+, this can also be a 32-bit literal constant offset. If it fits
2131 // in 8-bits, it can use a smaller encoding.
2132 if (!isUInt<32>(x: AM.BaseOffs / 4))
2133 return false;
2134 } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX9) {
2135 // On VI, these use the SMEM format and the offset is 20-bit in bytes.
2136 if (!isUInt<20>(x: AM.BaseOffs))
2137 return false;
2138 } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX12) {
2139 // On GFX9 the offset is signed 21-bit in bytes (but must not be negative
2140 // for S_BUFFER_* instructions).
2141 if (!isInt<21>(x: AM.BaseOffs))
2142 return false;
2143 } else {
2144 // On GFX12, all offsets are signed 24-bit in bytes.
2145 if (!isInt<24>(x: AM.BaseOffs))
2146 return false;
2147 }
2148
2149 if ((AS == AMDGPUAS::CONSTANT_ADDRESS ||
2150 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
2151 AM.BaseOffs < 0) {
2152 // Scalar (non-buffer) loads can only use a negative offset if
2153 // soffset+offset is non-negative. Since the compiler can only prove that
2154 // in a few special cases, it is safer to claim that negative offsets are
2155 // not supported.
2156 return false;
2157 }
2158
2159 if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
2160 return true;
2161
2162 if (AM.Scale == 1 && AM.HasBaseReg)
2163 return true;
2164
2165 return false;
2166 }
2167
2168 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
2169 return Subtarget->hasFlatScratchEnabled()
2170 ? isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS)
2171 : isLegalMUBUFAddressingMode(AM);
2172
2173 if (AS == AMDGPUAS::LOCAL_ADDRESS ||
2174 (AS == AMDGPUAS::REGION_ADDRESS && Subtarget->hasGDS())) {
2175 // Basic, single offset DS instructions allow a 16-bit unsigned immediate
2176 // field.
2177 // XXX - If doing a 4-byte aligned 8-byte type access, we effectively have
2178 // an 8-bit dword offset but we don't know the alignment here.
2179 if (!isUInt<16>(x: AM.BaseOffs))
2180 return false;
2181
2182 if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
2183 return true;
2184
2185 if (AM.Scale == 1 && AM.HasBaseReg)
2186 return true;
2187
2188 return false;
2189 }
2190
2191 if (AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::UNKNOWN_ADDRESS_SPACE) {
2192 // For an unknown address space, this usually means that this is for some
2193 // reason being used for pure arithmetic, and not based on some addressing
2194 // computation. We don't have instructions that compute pointers with any
2195 // addressing modes, so treat them as having no offset like flat
2196 // instructions.
2197 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::FLAT_ADDRESS);
2198 }
2199
2200 // Assume a user alias of global for unknown address spaces.
2201 return isLegalGlobalAddressingMode(AM);
2202}
2203
2204bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
2205 const MachineFunction &MF) const {
2206 if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS)
2207 return (MemVT.getSizeInBits() <= 4 * 32);
2208 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
2209 unsigned MaxPrivateBits = 8 * getSubtarget()->getMaxPrivateElementSize();
2210 return (MemVT.getSizeInBits() <= MaxPrivateBits);
2211 }
2212 if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS)
2213 return (MemVT.getSizeInBits() <= 2 * 32);
2214 return true;
2215}
2216
2217bool SITargetLowering::allowsMisalignedMemoryAccessesImpl(
2218 unsigned Size, unsigned AddrSpace, Align Alignment,
2219 MachineMemOperand::Flags Flags, unsigned *IsFast) const {
2220 if (IsFast)
2221 *IsFast = 0;
2222
2223 if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
2224 AddrSpace == AMDGPUAS::REGION_ADDRESS) {
2225 // Check if alignment requirements for ds_read/write instructions are
2226 // disabled.
2227 if (!Subtarget->hasUnalignedDSAccessEnabled() && Alignment < Align(4))
2228 return false;
2229
2230 Align RequiredAlignment(
2231 PowerOf2Ceil(A: divideCeil(Numerator: Size, Denominator: 8))); // Natural alignment.
2232 if (Subtarget->hasLDSMisalignedBugInWGPMode() && Size > 32 &&
2233 Alignment < RequiredAlignment)
2234 return false;
2235
2236 // Either, the alignment requirements are "enabled", or there is an
2237 // unaligned LDS access related hardware bug though alignment requirements
2238 // are "disabled". In either case, we need to check for proper alignment
2239 // requirements.
2240 //
2241 switch (Size) {
2242 case 64:
2243 // SI has a hardware bug in the LDS / GDS bounds checking: if the base
2244 // address is negative, then the instruction is incorrectly treated as
2245 // out-of-bounds even if base + offsets is in bounds. Split vectorized
2246 // loads here to avoid emitting ds_read2_b32. We may re-combine the
2247 // load later in the SILoadStoreOptimizer.
2248 if (!Subtarget->hasUsableDSOffset() && Alignment < Align(8))
2249 return false;
2250
2251 // 8 byte accessing via ds_read/write_b64 require 8-byte alignment, but we
2252 // can do a 4 byte aligned, 8 byte access in a single operation using
2253 // ds_read2/write2_b32 with adjacent offsets.
2254 RequiredAlignment = Align(4);
2255
2256 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2257 // We will either select ds_read_b64/ds_write_b64 or ds_read2_b32/
2258 // ds_write2_b32 depending on the alignment. In either case with either
2259 // alignment there is no faster way of doing this.
2260
2261 // The numbers returned here and below are not additive, it is a 'speed
2262 // rank'. They are just meant to be compared to decide if a certain way
2263 // of lowering an operation is faster than another. For that purpose
2264 // naturally aligned operation gets it bitsize to indicate that "it
2265 // operates with a speed comparable to N-bit wide load". With the full
2266 // alignment ds128 is slower than ds96 for example. If underaligned it
2267 // is comparable to a speed of a single dword access, which would then
2268 // mean 32 < 128 and it is faster to issue a wide load regardless.
2269 // 1 is simply "slow, don't do it". I.e. comparing an aligned load to a
2270 // wider load which will not be aligned anymore the latter is slower.
2271 if (IsFast)
2272 *IsFast = (Alignment >= RequiredAlignment) ? 64
2273 : (Alignment < Align(4)) ? 32
2274 : 1;
2275 return true;
2276 }
2277
2278 break;
2279 case 96:
2280 if (!Subtarget->hasDS96AndDS128())
2281 return false;
2282
2283 // 12 byte accessing via ds_read/write_b96 require 16-byte alignment on
2284 // gfx8 and older.
2285
2286 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2287 // Naturally aligned access is fastest. However, also report it is Fast
2288 // if memory is aligned less than DWORD. A narrow load or store will be
2289 // be equally slow as a single ds_read_b96/ds_write_b96, but there will
2290 // be more of them, so overall we will pay less penalty issuing a single
2291 // instruction.
2292
2293 // See comment on the values above.
2294 if (IsFast)
2295 *IsFast = (Alignment >= RequiredAlignment) ? 96
2296 : (Alignment < Align(4)) ? 32
2297 : 1;
2298 return true;
2299 }
2300
2301 break;
2302 case 128:
2303 if (!Subtarget->hasDS96AndDS128() || !Subtarget->useDS128())
2304 return false;
2305
2306 // 16 byte accessing via ds_read/write_b128 require 16-byte alignment on
2307 // gfx8 and older, but we can do a 8 byte aligned, 16 byte access in a
2308 // single operation using ds_read2/write2_b64.
2309 RequiredAlignment = Align(8);
2310
2311 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2312 // Naturally aligned access is fastest. However, also report it is Fast
2313 // if memory is aligned less than DWORD. A narrow load or store will be
2314 // be equally slow as a single ds_read_b128/ds_write_b128, but there
2315 // will be more of them, so overall we will pay less penalty issuing a
2316 // single instruction.
2317
2318 // See comment on the values above.
2319 if (IsFast)
2320 *IsFast = (Alignment >= RequiredAlignment) ? 128
2321 : (Alignment < Align(4)) ? 32
2322 : 1;
2323 return true;
2324 }
2325
2326 break;
2327 default:
2328 if (Size > 32)
2329 return false;
2330
2331 break;
2332 }
2333
2334 // See comment on the values above.
2335 // Note that we have a single-dword or sub-dword here, so if underaligned
2336 // it is a slowest possible access, hence returned value is 0.
2337 if (IsFast)
2338 *IsFast = (Alignment >= RequiredAlignment) ? Size : 0;
2339
2340 return Alignment >= RequiredAlignment ||
2341 Subtarget->hasUnalignedDSAccessEnabled();
2342 }
2343
2344 // FIXME: We have to be conservative here and assume that flat operations
2345 // will access scratch. If we had access to the IR function, then we
2346 // could determine if any private memory was used in the function.
2347 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ||
2348 AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
2349 bool AlignedBy4 = Alignment >= Align(4);
2350 if (Subtarget->hasUnalignedScratchAccessEnabled()) {
2351 if (IsFast)
2352 *IsFast = AlignedBy4 ? Size : 1;
2353 return true;
2354 }
2355
2356 if (IsFast)
2357 *IsFast = AlignedBy4;
2358
2359 return AlignedBy4;
2360 }
2361
2362 // So long as they are correct, wide global memory operations perform better
2363 // than multiple smaller memory ops -- even when misaligned
2364 if (AMDGPU::isExtendedGlobalAddrSpace(AS: AddrSpace)) {
2365 if (IsFast)
2366 *IsFast = Size;
2367
2368 return Alignment >= Align(4) ||
2369 Subtarget->hasUnalignedBufferAccessEnabled();
2370 }
2371
2372 // Ensure robust out-of-bounds guarantees for buffer accesses are met when the
2373 // "amdgpu.buffer.oob.mode" module flag has not enabled relaxed untyped-buffer
2374 // OOB semantics. Normally hardware will ensure proper
2375 // out-of-bounds behavior, but in the edge case where an access starts
2376 // out-of-bounds and then enters in-bounds, the entire access would be treated
2377 // as out-of-bounds. Prevent misaligned memory accesses by requiring the
2378 // natural alignment of buffer accesses.
2379 if (AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
2380 AddrSpace == AMDGPUAS::BUFFER_RESOURCE ||
2381 AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) {
2382 if (!Subtarget->hasRelaxedBufferOOBMode() &&
2383 Alignment < Align(PowerOf2Ceil(A: divideCeil(Numerator: Size, Denominator: 8))))
2384 return false;
2385 }
2386
2387 // Smaller than dword value must be aligned.
2388 if (Size < 32)
2389 return false;
2390
2391 // 8.1.6 - For Dword or larger reads or writes, the two LSBs of the
2392 // byte-address are ignored, thus forcing Dword alignment.
2393 // This applies to private, global, and constant memory.
2394 if (IsFast)
2395 *IsFast = 1;
2396
2397 return Size >= 32 && Alignment >= Align(4);
2398}
2399
2400bool SITargetLowering::allowsMisalignedMemoryAccesses(
2401 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
2402 unsigned *IsFast) const {
2403 return allowsMisalignedMemoryAccessesImpl(Size: VT.getSizeInBits(), AddrSpace,
2404 Alignment, Flags, IsFast);
2405}
2406
2407EVT SITargetLowering::getOptimalMemOpType(
2408 LLVMContext &Context, const MemOp &Op,
2409 const AttributeList &FuncAttributes) const {
2410 // FIXME: Should account for address space here.
2411
2412 // The default fallback uses the private pointer size as a guess for a type to
2413 // use. Make sure we switch these to 64-bit accesses.
2414
2415 if (Op.size() >= 16 &&
2416 Op.isDstAligned(AlignCheck: Align(4))) // XXX: Should only do for global
2417 return MVT::v4i32;
2418
2419 if (Op.size() >= 8 && Op.isDstAligned(AlignCheck: Align(4)))
2420 return MVT::v2i32;
2421
2422 // Use the default.
2423 return MVT::Other;
2424}
2425
2426bool SITargetLowering::isMemOpHasNoClobberedMemOperand(const SDNode *N) const {
2427 const MemSDNode *MemNode = cast<MemSDNode>(Val: N);
2428 return MemNode->getMemOperand()->getFlags() & MONoClobber;
2429}
2430
2431bool SITargetLowering::isNonGlobalAddrSpace(unsigned AS) {
2432 return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS ||
2433 AS == AMDGPUAS::PRIVATE_ADDRESS;
2434}
2435
2436bool SITargetLowering::isFreeAddrSpaceCast(unsigned SrcAS,
2437 unsigned DestAS) const {
2438 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
2439 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
2440 Subtarget->hasGloballyAddressableScratch()) {
2441 // Flat -> private requires subtracting src_flat_scratch_base_lo.
2442 return false;
2443 }
2444
2445 // Flat -> private/local is a simple truncate.
2446 // Flat -> global is no-op
2447 return true;
2448 }
2449
2450 const GCNTargetMachine &TM =
2451 static_cast<const GCNTargetMachine &>(getTargetMachine());
2452 return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
2453}
2454
2455TargetLoweringBase::LegalizeTypeAction
2456SITargetLowering::getPreferredVectorAction(MVT VT) const {
2457 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
2458 VT.getScalarType().bitsLE(VT: MVT::i16))
2459 return VT.isPow2VectorType() ? TypeSplitVector : TypeWidenVector;
2460 return TargetLoweringBase::getPreferredVectorAction(VT);
2461}
2462
2463bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
2464 Type *Ty) const {
2465 // FIXME: Could be smarter if called for vector constants.
2466 return true;
2467}
2468
2469TargetLowering::ExtractSubvectorCost
2470SITargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
2471 unsigned Index) const {
2472 if (!isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: ResVT))
2473 return ExtractSubvectorCost::Expensive;
2474
2475 // TODO: Add more cases that are cheap.
2476 if (Index == 0)
2477 return ExtractSubvectorCost::Free;
2478 return ExtractSubvectorCost::Expensive;
2479}
2480
2481bool SITargetLowering::isExtractVecEltCheap(EVT VT, unsigned Index) const {
2482 // TODO: This should be more aggressive, particular for 16-bit element
2483 // vectors. However there are some mixed improvements and regressions.
2484 EVT EltTy = VT.getVectorElementType();
2485 unsigned MinAlign = Subtarget->useRealTrue16Insts() ? 16 : 32;
2486 return EltTy.getSizeInBits() % MinAlign == 0;
2487}
2488
2489bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT) const {
2490 if (Subtarget->has16BitInsts() && VT == MVT::i16) {
2491 switch (Op) {
2492 case ISD::LOAD:
2493 case ISD::STORE:
2494 return true;
2495 default:
2496 return false;
2497 }
2498 }
2499
2500 // SimplifySetCC uses this function to determine whether or not it should
2501 // create setcc with i1 operands. We don't have instructions for i1 setcc.
2502 if (VT == MVT::i1 && Op == ISD::SETCC)
2503 return false;
2504
2505 return TargetLowering::isTypeDesirableForOp(Op, VT);
2506}
2507
2508MachinePointerInfo
2509SITargetLowering::getKernargSegmentPtrInfo(MachineFunction &MF) const {
2510 // This isn't really a constant pool but close enough.
2511 MachinePointerInfo PtrInfo(MF.getPSVManager().getConstantPool());
2512 PtrInfo.AddrSpace = AMDGPUAS::CONSTANT_ADDRESS;
2513 return PtrInfo;
2514}
2515
2516SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
2517 const SDLoc &SL,
2518 SDValue Chain,
2519 uint64_t Offset) const {
2520 const DataLayout &DL = DAG.getDataLayout();
2521 MachineFunction &MF = DAG.getMachineFunction();
2522 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
2523 MVT PtrVT = getPointerTy(DL, AS: AMDGPUAS::CONSTANT_ADDRESS);
2524
2525 auto [InputPtrReg, RC, ArgTy] =
2526 Info->getPreloadedValue(Value: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
2527
2528 // We may not have the kernarg segment argument if we have no kernel
2529 // arguments.
2530 if (!InputPtrReg)
2531 return DAG.getConstant(Val: Offset, DL: SL, VT: PtrVT);
2532
2533 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
2534 SDValue BasePtr = DAG.getCopyFromReg(
2535 Chain, dl: SL, Reg: MRI.getLiveInVirtReg(PReg: InputPtrReg->getRegister()), VT: PtrVT);
2536
2537 return DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
2538}
2539
2540SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG,
2541 const SDLoc &SL) const {
2542 uint64_t Offset =
2543 getImplicitParameterOffset(MF: DAG.getMachineFunction(), Param: FIRST_IMPLICIT);
2544 return lowerKernArgParameterPtr(DAG, SL, Chain: DAG.getEntryNode(), Offset);
2545}
2546
2547SDValue SITargetLowering::getLDSKernelId(SelectionDAG &DAG,
2548 const SDLoc &SL) const {
2549
2550 Function &F = DAG.getMachineFunction().getFunction();
2551 std::optional<uint32_t> KnownSize =
2552 AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F);
2553 if (KnownSize.has_value())
2554 return DAG.getConstant(Val: *KnownSize, DL: SL, VT: MVT::i32);
2555 return SDValue();
2556}
2557
2558SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
2559 const SDLoc &SL, SDValue Val,
2560 bool Signed,
2561 const ISD::InputArg *Arg) const {
2562 // First, if it is a widened vector, narrow it.
2563 if (VT.isVector() &&
2564 VT.getVectorNumElements() != MemVT.getVectorNumElements()) {
2565 EVT NarrowedVT =
2566 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MemVT.getVectorElementType(),
2567 NumElements: VT.getVectorNumElements());
2568 Val = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: NarrowedVT, N1: Val,
2569 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
2570 }
2571
2572 // Then convert the vector elements or scalar value.
2573 if (Arg && (Arg->Flags.isSExt() || Arg->Flags.isZExt()) && VT.bitsLT(VT: MemVT)) {
2574 unsigned Opc = Arg->Flags.isZExt() ? ISD::AssertZext : ISD::AssertSext;
2575 Val = DAG.getNode(Opcode: Opc, DL: SL, VT: MemVT, N1: Val, N2: DAG.getValueType(VT));
2576 }
2577
2578 if (MemVT.isFloatingPoint()) {
2579 if (VT.isFloatingPoint()) {
2580 Val = getFPExtOrFPRound(DAG, Op: Val, DL: SL, VT);
2581 } else {
2582 assert(!MemVT.isVector());
2583 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
2584 SDValue Cast = DAG.getBitcast(VT: IntVT, V: Val);
2585 Val = DAG.getAnyExtOrTrunc(Op: Cast, DL: SL, VT);
2586 }
2587 } else if (Signed)
2588 Val = DAG.getSExtOrTrunc(Op: Val, DL: SL, VT);
2589 else
2590 Val = DAG.getZExtOrTrunc(Op: Val, DL: SL, VT);
2591
2592 return Val;
2593}
2594
2595SDValue SITargetLowering::lowerKernargMemParameter(
2596 SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain,
2597 uint64_t Offset, Align Alignment, bool Signed,
2598 const ISD::InputArg *Arg) const {
2599
2600 MachinePointerInfo PtrInfo =
2601 getKernargSegmentPtrInfo(MF&: DAG.getMachineFunction());
2602
2603 // Try to avoid using an extload by loading earlier than the argument address,
2604 // and extracting the relevant bits. The load should hopefully be merged with
2605 // the previous argument.
2606 if (MemVT.getStoreSize() < 4 && Alignment < 4) {
2607 // TODO: Handle align < 4 and size >= 4 (can happen with packed structs).
2608 int64_t AlignDownOffset = alignDown(Value: Offset, Align: 4);
2609 int64_t OffsetDiff = Offset - AlignDownOffset;
2610
2611 EVT IntVT = MemVT.changeTypeToInteger();
2612
2613 // TODO: If we passed in the base kernel offset we could have a better
2614 // alignment than 4, but we don't really need it.
2615 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset: AlignDownOffset);
2616 SDValue Load = DAG.getLoad(VT: MVT::i32, dl: SL, Chain, Ptr,
2617 PtrInfo: PtrInfo.getWithOffset(O: AlignDownOffset), Alignment: Align(4),
2618 MMOFlags: MachineMemOperand::MODereferenceable |
2619 MachineMemOperand::MOInvariant);
2620
2621 SDValue ShiftAmt = DAG.getConstant(Val: OffsetDiff * 8, DL: SL, VT: MVT::i32);
2622 SDValue Extract = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: Load, N2: ShiftAmt);
2623
2624 SDValue ArgVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: IntVT, Operand: Extract);
2625 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MemVT, Operand: ArgVal);
2626 ArgVal = convertArgType(DAG, VT, MemVT, SL, Val: ArgVal, Signed, Arg);
2627
2628 return DAG.getMergeValues(Ops: {ArgVal, Load.getValue(R: 1)}, dl: SL);
2629 }
2630
2631 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
2632 SDValue Load = DAG.getLoad(
2633 VT: MemVT, dl: SL, Chain, Ptr, PtrInfo: PtrInfo.getWithOffset(O: Offset), Alignment,
2634 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
2635
2636 SDValue Val = convertArgType(DAG, VT, MemVT, SL, Val: Load, Signed, Arg);
2637 return DAG.getMergeValues(Ops: {Val, Load.getValue(R: 1)}, dl: SL);
2638}
2639
2640/// Coerce an argument which was passed in a different ABI type to the original
2641/// expected value type.
2642SDValue SITargetLowering::convertABITypeToValueType(SelectionDAG &DAG,
2643 SDValue Val,
2644 CCValAssign &VA,
2645 const SDLoc &SL) const {
2646 EVT ValVT = VA.getValVT();
2647
2648 // If this is an 8 or 16-bit value, it is really passed promoted
2649 // to 32 bits. Insert an assert[sz]ext to capture this, then
2650 // truncate to the right size.
2651 switch (VA.getLocInfo()) {
2652 case CCValAssign::Full:
2653 return Val;
2654 case CCValAssign::BCvt:
2655 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ValVT, Operand: Val);
2656 case CCValAssign::SExt:
2657 Val = DAG.getNode(Opcode: ISD::AssertSext, DL: SL, VT: VA.getLocVT(), N1: Val,
2658 N2: DAG.getValueType(ValVT));
2659 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2660 case CCValAssign::ZExt:
2661 Val = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: VA.getLocVT(), N1: Val,
2662 N2: DAG.getValueType(ValVT));
2663 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2664 case CCValAssign::AExt:
2665 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2666 default:
2667 llvm_unreachable("Unknown loc info!");
2668 }
2669}
2670
2671SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG,
2672 CCValAssign &VA, const SDLoc &SL,
2673 SDValue Chain,
2674 const ISD::InputArg &Arg) const {
2675 MachineFunction &MF = DAG.getMachineFunction();
2676 MachineFrameInfo &MFI = MF.getFrameInfo();
2677
2678 if (Arg.Flags.isByVal()) {
2679 unsigned Size = Arg.Flags.getByValSize();
2680 int FrameIdx = MFI.CreateFixedObject(Size, SPOffset: VA.getLocMemOffset(), IsImmutable: false);
2681 return DAG.getFrameIndex(FI: FrameIdx, VT: MVT::i32);
2682 }
2683
2684 unsigned ArgOffset = VA.getLocMemOffset();
2685 unsigned ArgSize = VA.getValVT().getStoreSize();
2686
2687 int FI = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: true);
2688
2689 // Create load nodes to retrieve arguments from the stack.
2690 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
2691
2692 // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
2693 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
2694 MVT MemVT = VA.getValVT();
2695
2696 switch (VA.getLocInfo()) {
2697 default:
2698 break;
2699 case CCValAssign::BCvt:
2700 MemVT = VA.getLocVT();
2701 break;
2702 case CCValAssign::SExt:
2703 ExtType = ISD::SEXTLOAD;
2704 break;
2705 case CCValAssign::ZExt:
2706 ExtType = ISD::ZEXTLOAD;
2707 break;
2708 case CCValAssign::AExt:
2709 ExtType = ISD::EXTLOAD;
2710 break;
2711 }
2712
2713 SDValue ArgValue = DAG.getExtLoad(
2714 ExtType, dl: SL, VT: VA.getLocVT(), Chain, Ptr: FIN,
2715 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), MemVT);
2716
2717 SDValue ConvertedVal = convertABITypeToValueType(DAG, Val: ArgValue, VA, SL);
2718 if (ConvertedVal == ArgValue)
2719 return ConvertedVal;
2720
2721 return DAG.getMergeValues(Ops: {ConvertedVal, ArgValue.getValue(R: 1)}, dl: SL);
2722}
2723
2724SDValue SITargetLowering::lowerWorkGroupId(
2725 SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
2726 AMDGPUFunctionArgInfo::PreloadedValue WorkGroupIdPV,
2727 AMDGPUFunctionArgInfo::PreloadedValue ClusterMaxIdPV,
2728 AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const {
2729 if (!Subtarget->hasClusters())
2730 return getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
2731
2732 // Clusters are supported. Return the global position in the grid. If clusters
2733 // are enabled, WorkGroupIdPV returns the cluster ID not the workgroup ID.
2734
2735 // WorkGroupIdXYZ = ClusterId == 0 ?
2736 // ClusterIdXYZ :
2737 // ClusterIdXYZ * (ClusterMaxIdXYZ + 1) + ClusterWorkGroupIdXYZ
2738 SDValue ClusterIdXYZ = getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
2739 SDLoc SL(ClusterIdXYZ);
2740 SDValue ClusterMaxIdXYZ = getPreloadedValue(DAG, MFI, VT, ClusterMaxIdPV);
2741 SDValue One = DAG.getConstant(Val: 1, DL: SL, VT);
2742 SDValue ClusterSizeXYZ = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ClusterMaxIdXYZ, N2: One);
2743 SDValue ClusterWorkGroupIdXYZ =
2744 getPreloadedValue(DAG, MFI, VT, ClusterWorkGroupIdPV);
2745 SDValue GlobalIdXYZ =
2746 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ClusterWorkGroupIdXYZ,
2747 N2: DAG.getNode(Opcode: ISD::MUL, DL: SL, VT, N1: ClusterIdXYZ, N2: ClusterSizeXYZ));
2748
2749 switch (MFI.getClusterDims().getKind()) {
2750 case AMDGPU::ClusterDimsAttr::Kind::FixedDims:
2751 case AMDGPU::ClusterDimsAttr::Kind::VariableDims:
2752 return GlobalIdXYZ;
2753 case AMDGPU::ClusterDimsAttr::Kind::NoCluster:
2754 return ClusterIdXYZ;
2755 case AMDGPU::ClusterDimsAttr::Kind::Unknown: {
2756 using namespace AMDGPU::Hwreg;
2757 SDValue ClusterIdField =
2758 DAG.getTargetConstant(Val: HwregEncoding::encode(Values: ID_IB_STS2, Values: 6, Values: 4), DL: SL, VT);
2759 SDNode *GetReg =
2760 DAG.getMachineNode(Opcode: AMDGPU::S_GETREG_B32_const, dl: SL, VT, Op1: ClusterIdField);
2761 SDValue ClusterId(GetReg, 0);
2762 SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT);
2763 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SL, VT, N1: ClusterId, N2: Zero, N3: ClusterIdXYZ,
2764 N4: GlobalIdXYZ, N5: DAG.getCondCode(Cond: ISD::SETEQ));
2765 }
2766 }
2767
2768 llvm_unreachable("nothing should reach here");
2769}
2770
2771SDValue SITargetLowering::getPreloadedValue(
2772 SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
2773 AMDGPUFunctionArgInfo::PreloadedValue PVID) const {
2774 const ArgDescriptor *Reg = nullptr;
2775 const TargetRegisterClass *RC = nullptr;
2776 LLT Ty;
2777
2778 CallingConv::ID CC = DAG.getMachineFunction().getFunction().getCallingConv();
2779 const ArgDescriptor WorkGroupIDX =
2780 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP9);
2781 // If GridZ is not programmed in an entry function then the hardware will set
2782 // it to all zeros, so there is no need to mask the GridY value in the low
2783 // order bits.
2784 const ArgDescriptor WorkGroupIDY = ArgDescriptor::createRegister(
2785 Reg: AMDGPU::TTMP7,
2786 Mask: AMDGPU::isEntryFunctionCC(CC) && !MFI.hasWorkGroupIDZ() ? ~0u : 0xFFFFu);
2787 const ArgDescriptor WorkGroupIDZ =
2788 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP7, Mask: 0xFFFF0000u);
2789 const ArgDescriptor ClusterWorkGroupIDX =
2790 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0000000Fu);
2791 const ArgDescriptor ClusterWorkGroupIDY =
2792 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x000000F0u);
2793 const ArgDescriptor ClusterWorkGroupIDZ =
2794 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x00000F00u);
2795 const ArgDescriptor ClusterWorkGroupMaxIDX =
2796 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0000F000u);
2797 const ArgDescriptor ClusterWorkGroupMaxIDY =
2798 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x000F0000u);
2799 const ArgDescriptor ClusterWorkGroupMaxIDZ =
2800 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x00F00000u);
2801 const ArgDescriptor ClusterWorkGroupMaxFlatID =
2802 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0F000000u);
2803
2804 auto LoadConstant = [&](unsigned N) {
2805 return DAG.getConstant(Val: N, DL: SDLoc(), VT);
2806 };
2807
2808 if (Subtarget->hasArchitectedSGPRs() &&
2809 (AMDGPU::isCompute(CC) || CC == CallingConv::AMDGPU_Gfx)) {
2810 AMDGPU::ClusterDimsAttr ClusterDims = MFI.getClusterDims();
2811 bool HasFixedDims = ClusterDims.isFixedDims();
2812
2813 switch (PVID) {
2814 case AMDGPUFunctionArgInfo::WORKGROUP_ID_X:
2815 Reg = &WorkGroupIDX;
2816 RC = &AMDGPU::SReg_32RegClass;
2817 Ty = LLT::scalar(SizeInBits: 32);
2818 break;
2819 case AMDGPUFunctionArgInfo::WORKGROUP_ID_Y:
2820 Reg = &WorkGroupIDY;
2821 RC = &AMDGPU::SReg_32RegClass;
2822 Ty = LLT::scalar(SizeInBits: 32);
2823 break;
2824 case AMDGPUFunctionArgInfo::WORKGROUP_ID_Z:
2825 Reg = &WorkGroupIDZ;
2826 RC = &AMDGPU::SReg_32RegClass;
2827 Ty = LLT::scalar(SizeInBits: 32);
2828 break;
2829 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X:
2830 if (HasFixedDims && ClusterDims.getDims()[0] == 1)
2831 return LoadConstant(0);
2832 Reg = &ClusterWorkGroupIDX;
2833 RC = &AMDGPU::SReg_32RegClass;
2834 Ty = LLT::scalar(SizeInBits: 32);
2835 break;
2836 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y:
2837 if (HasFixedDims && ClusterDims.getDims()[1] == 1)
2838 return LoadConstant(0);
2839 Reg = &ClusterWorkGroupIDY;
2840 RC = &AMDGPU::SReg_32RegClass;
2841 Ty = LLT::scalar(SizeInBits: 32);
2842 break;
2843 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z:
2844 if (HasFixedDims && ClusterDims.getDims()[2] == 1)
2845 return LoadConstant(0);
2846 Reg = &ClusterWorkGroupIDZ;
2847 RC = &AMDGPU::SReg_32RegClass;
2848 Ty = LLT::scalar(SizeInBits: 32);
2849 break;
2850 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X:
2851 if (HasFixedDims)
2852 return LoadConstant(ClusterDims.getDims()[0] - 1);
2853 Reg = &ClusterWorkGroupMaxIDX;
2854 RC = &AMDGPU::SReg_32RegClass;
2855 Ty = LLT::scalar(SizeInBits: 32);
2856 break;
2857 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y:
2858 if (HasFixedDims)
2859 return LoadConstant(ClusterDims.getDims()[1] - 1);
2860 Reg = &ClusterWorkGroupMaxIDY;
2861 RC = &AMDGPU::SReg_32RegClass;
2862 Ty = LLT::scalar(SizeInBits: 32);
2863 break;
2864 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z:
2865 if (HasFixedDims)
2866 return LoadConstant(ClusterDims.getDims()[2] - 1);
2867 Reg = &ClusterWorkGroupMaxIDZ;
2868 RC = &AMDGPU::SReg_32RegClass;
2869 Ty = LLT::scalar(SizeInBits: 32);
2870 break;
2871 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID:
2872 Reg = &ClusterWorkGroupMaxFlatID;
2873 RC = &AMDGPU::SReg_32RegClass;
2874 Ty = LLT::scalar(SizeInBits: 32);
2875 break;
2876 default:
2877 break;
2878 }
2879 }
2880
2881 if (!Reg)
2882 std::tie(args&: Reg, args&: RC, args&: Ty) = MFI.getPreloadedValue(Value: PVID);
2883 if (!Reg) {
2884 if (PVID == AMDGPUFunctionArgInfo::PreloadedValue::KERNARG_SEGMENT_PTR) {
2885 // It's possible for a kernarg intrinsic call to appear in a kernel with
2886 // no allocated segment, in which case we do not add the user sgpr
2887 // argument, so just return null.
2888 return DAG.getConstant(Val: 0, DL: SDLoc(), VT);
2889 }
2890
2891 // It's undefined behavior if a function marked with the amdgpu-no-*
2892 // attributes uses the corresponding intrinsic.
2893 return DAG.getPOISON(VT);
2894 }
2895
2896 return loadInputValue(DAG, RC, VT, SL: SDLoc(DAG.getEntryNode()), Arg: *Reg);
2897}
2898
2899static void processPSInputArgs(SmallVectorImpl<ISD::InputArg> &Splits,
2900 CallingConv::ID CallConv,
2901 ArrayRef<ISD::InputArg> Ins, BitVector &Skipped,
2902 FunctionType *FType,
2903 SIMachineFunctionInfo *Info) {
2904 for (unsigned I = 0, E = Ins.size(), PSInputNum = 0; I != E; ++I) {
2905 const ISD::InputArg *Arg = &Ins[I];
2906
2907 assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
2908 "vector type argument should have been split");
2909
2910 // First check if it's a PS input addr.
2911 if (CallConv == CallingConv::AMDGPU_PS && !Arg->Flags.isInReg() &&
2912 PSInputNum <= 15) {
2913 bool SkipArg = !Arg->Used && !Info->isPSInputAllocated(Index: PSInputNum);
2914
2915 // Inconveniently only the first part of the split is marked as isSplit,
2916 // so skip to the end. We only want to increment PSInputNum once for the
2917 // entire split argument.
2918 if (Arg->Flags.isSplit()) {
2919 while (!Arg->Flags.isSplitEnd()) {
2920 assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
2921 "unexpected vector split in ps argument type");
2922 if (!SkipArg)
2923 Splits.push_back(Elt: *Arg);
2924 Arg = &Ins[++I];
2925 }
2926 }
2927
2928 if (SkipArg) {
2929 // We can safely skip PS inputs.
2930 Skipped.set(Arg->getOrigArgIndex());
2931 ++PSInputNum;
2932 continue;
2933 }
2934
2935 Info->markPSInputAllocated(Index: PSInputNum);
2936 if (Arg->Used)
2937 Info->markPSInputEnabled(Index: PSInputNum);
2938
2939 ++PSInputNum;
2940 }
2941
2942 Splits.push_back(Elt: *Arg);
2943 }
2944}
2945
2946// Allocate special inputs passed in VGPRs.
2947void SITargetLowering::allocateSpecialEntryInputVGPRs(
2948 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
2949 SIMachineFunctionInfo &Info) const {
2950 const LLT I32 = LLT::integer(SizeInBits: 32);
2951 MachineRegisterInfo &MRI = MF.getRegInfo();
2952
2953 if (Info.hasWorkItemIDX()) {
2954 Register Reg = AMDGPU::VGPR0;
2955 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: I32);
2956
2957 CCInfo.AllocateReg(Reg);
2958 unsigned Mask =
2959 (Subtarget->hasPackedTID() && Info.hasWorkItemIDY()) ? 0x3ff : ~0u;
2960 Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
2961 }
2962
2963 if (Info.hasWorkItemIDY()) {
2964 assert(Info.hasWorkItemIDX());
2965 if (Subtarget->hasPackedTID()) {
2966 Info.setWorkItemIDY(
2967 ArgDescriptor::createRegister(Reg: AMDGPU::VGPR0, Mask: 0x3ff << 10));
2968 } else {
2969 unsigned Reg = AMDGPU::VGPR1;
2970 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: I32);
2971
2972 CCInfo.AllocateReg(Reg);
2973 Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg));
2974 }
2975 }
2976
2977 if (Info.hasWorkItemIDZ()) {
2978 assert(Info.hasWorkItemIDX() && Info.hasWorkItemIDY());
2979 if (Subtarget->hasPackedTID()) {
2980 Info.setWorkItemIDZ(
2981 ArgDescriptor::createRegister(Reg: AMDGPU::VGPR0, Mask: 0x3ff << 20));
2982 } else {
2983 unsigned Reg = AMDGPU::VGPR2;
2984 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: I32);
2985
2986 CCInfo.AllocateReg(Reg);
2987 Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg));
2988 }
2989 }
2990}
2991
2992// Try to allocate a VGPR at the end of the argument list, or if no argument
2993// VGPRs are left allocating a stack slot.
2994// If \p Mask is given it indicates bitfield position in the register.
2995// If \p Arg is given use it with new ]p Mask instead of allocating new.
2996static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u,
2997 ArgDescriptor Arg = ArgDescriptor()) {
2998 if (Arg.isSet())
2999 return ArgDescriptor::createArg(Arg, Mask);
3000
3001 ArrayRef<MCPhysReg> ArgVGPRs = ArrayRef(AMDGPU::VGPR_32RegClass.begin(), 32);
3002 unsigned RegIdx = CCInfo.getFirstUnallocated(Regs: ArgVGPRs);
3003 if (RegIdx == ArgVGPRs.size()) {
3004 // Spill to stack required.
3005 int64_t Offset = CCInfo.AllocateStack(Size: 4, Alignment: Align(4));
3006
3007 return ArgDescriptor::createStack(Offset, Mask);
3008 }
3009
3010 unsigned Reg = ArgVGPRs[RegIdx];
3011 Reg = CCInfo.AllocateReg(Reg);
3012 assert(Reg != AMDGPU::NoRegister);
3013
3014 MachineFunction &MF = CCInfo.getMachineFunction();
3015 Register LiveInVReg = MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass);
3016 MF.getRegInfo().setType(VReg: LiveInVReg, Ty: LLT::scalar(SizeInBits: 32));
3017 return ArgDescriptor::createRegister(Reg, Mask);
3018}
3019
3020static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo,
3021 const TargetRegisterClass *RC,
3022 unsigned NumArgRegs) {
3023 ArrayRef<MCPhysReg> ArgSGPRs = ArrayRef(RC->begin(), 32);
3024 unsigned RegIdx = CCInfo.getFirstUnallocated(Regs: ArgSGPRs);
3025 if (RegIdx == ArgSGPRs.size())
3026 report_fatal_error(reason: "ran out of SGPRs for arguments");
3027
3028 unsigned Reg = ArgSGPRs[RegIdx];
3029 Reg = CCInfo.AllocateReg(Reg);
3030 assert(Reg != AMDGPU::NoRegister);
3031
3032 MachineFunction &MF = CCInfo.getMachineFunction();
3033 MF.addLiveIn(PReg: Reg, RC);
3034 return ArgDescriptor::createRegister(Reg);
3035}
3036
3037// If this has a fixed position, we still should allocate the register in the
3038// CCInfo state. Technically we could get away with this for values passed
3039// outside of the normal argument range.
3040static void allocateFixedSGPRInputImpl(CCState &CCInfo,
3041 const TargetRegisterClass *RC,
3042 MCRegister Reg) {
3043 Reg = CCInfo.AllocateReg(Reg);
3044 assert(Reg != AMDGPU::NoRegister);
3045 MachineFunction &MF = CCInfo.getMachineFunction();
3046 MF.addLiveIn(PReg: Reg, RC);
3047}
3048
3049static void allocateSGPR32Input(CCState &CCInfo, ArgDescriptor &Arg) {
3050 if (Arg) {
3051 allocateFixedSGPRInputImpl(CCInfo, RC: &AMDGPU::SGPR_32RegClass,
3052 Reg: Arg.getRegister());
3053 } else
3054 Arg = allocateSGPR32InputImpl(CCInfo, RC: &AMDGPU::SGPR_32RegClass, NumArgRegs: 32);
3055}
3056
3057static void allocateSGPR64Input(CCState &CCInfo, ArgDescriptor &Arg) {
3058 if (Arg) {
3059 allocateFixedSGPRInputImpl(CCInfo, RC: &AMDGPU::SGPR_64RegClass,
3060 Reg: Arg.getRegister());
3061 } else
3062 Arg = allocateSGPR32InputImpl(CCInfo, RC: &AMDGPU::SGPR_64RegClass, NumArgRegs: 16);
3063}
3064
3065/// Allocate implicit function VGPR arguments at the end of allocated user
3066/// arguments.
3067void SITargetLowering::allocateSpecialInputVGPRs(
3068 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
3069 SIMachineFunctionInfo &Info) const {
3070 const unsigned Mask = 0x3ff;
3071 ArgDescriptor Arg;
3072
3073 if (Info.hasWorkItemIDX()) {
3074 Arg = allocateVGPR32Input(CCInfo, Mask);
3075 Info.setWorkItemIDX(Arg);
3076 }
3077
3078 if (Info.hasWorkItemIDY()) {
3079 Arg = allocateVGPR32Input(CCInfo, Mask: Mask << 10, Arg);
3080 Info.setWorkItemIDY(Arg);
3081 }
3082
3083 if (Info.hasWorkItemIDZ())
3084 Info.setWorkItemIDZ(allocateVGPR32Input(CCInfo, Mask: Mask << 20, Arg));
3085}
3086
3087/// Allocate implicit function VGPR arguments in fixed registers.
3088void SITargetLowering::allocateSpecialInputVGPRsFixed(
3089 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
3090 SIMachineFunctionInfo &Info) const {
3091 Register Reg = CCInfo.AllocateReg(Reg: AMDGPU::VGPR31);
3092 if (!Reg)
3093 report_fatal_error(reason: "failed to allocate VGPR for implicit arguments");
3094
3095 const unsigned Mask = 0x3ff;
3096 Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
3097 Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg, Mask: Mask << 10));
3098 Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg, Mask: Mask << 20));
3099}
3100
3101void SITargetLowering::allocateSpecialInputSGPRs(
3102 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
3103 SIMachineFunctionInfo &Info) const {
3104 auto &ArgInfo = Info.getArgInfo();
3105 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
3106
3107 // TODO: Unify handling with private memory pointers.
3108 if (UserSGPRInfo.hasDispatchPtr())
3109 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.DispatchPtr);
3110
3111 if (UserSGPRInfo.hasQueuePtr())
3112 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.QueuePtr);
3113
3114 // Implicit arg ptr takes the place of the kernarg segment pointer. This is a
3115 // constant offset from the kernarg segment.
3116 if (Info.hasImplicitArgPtr())
3117 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.ImplicitArgPtr);
3118
3119 if (UserSGPRInfo.hasDispatchID())
3120 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.DispatchID);
3121
3122 // flat_scratch_init is not applicable for non-kernel functions.
3123
3124 if (Info.hasWorkGroupIDX())
3125 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDX);
3126
3127 if (Info.hasWorkGroupIDY())
3128 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDY);
3129
3130 if (Info.hasWorkGroupIDZ())
3131 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDZ);
3132
3133 if (Info.hasLDSKernelId())
3134 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.LDSKernelId);
3135}
3136
3137// Allocate special inputs passed in user SGPRs.
3138void SITargetLowering::allocateHSAUserSGPRs(CCState &CCInfo,
3139 MachineFunction &MF,
3140 const SIRegisterInfo &TRI,
3141 SIMachineFunctionInfo &Info) const {
3142 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
3143 if (UserSGPRInfo.hasImplicitBufferPtr()) {
3144 Register ImplicitBufferPtrReg = Info.addImplicitBufferPtr(TRI);
3145 MF.addLiveIn(PReg: ImplicitBufferPtrReg, RC: &AMDGPU::SGPR_64RegClass);
3146 CCInfo.AllocateReg(Reg: ImplicitBufferPtrReg);
3147 }
3148
3149 // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
3150 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
3151 Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
3152 MF.addLiveIn(PReg: PrivateSegmentBufferReg, RC: &AMDGPU::SGPR_128RegClass);
3153 CCInfo.AllocateReg(Reg: PrivateSegmentBufferReg);
3154 }
3155
3156 if (UserSGPRInfo.hasDispatchPtr()) {
3157 Register DispatchPtrReg = Info.addDispatchPtr(TRI);
3158 MF.addLiveIn(PReg: DispatchPtrReg, RC: &AMDGPU::SGPR_64RegClass);
3159 CCInfo.AllocateReg(Reg: DispatchPtrReg);
3160 }
3161
3162 if (UserSGPRInfo.hasQueuePtr()) {
3163 Register QueuePtrReg = Info.addQueuePtr(TRI);
3164 MF.addLiveIn(PReg: QueuePtrReg, RC: &AMDGPU::SGPR_64RegClass);
3165 CCInfo.AllocateReg(Reg: QueuePtrReg);
3166 }
3167
3168 if (UserSGPRInfo.hasKernargSegmentPtr()) {
3169 MachineRegisterInfo &MRI = MF.getRegInfo();
3170 Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
3171 CCInfo.AllocateReg(Reg: InputPtrReg);
3172
3173 Register VReg = MF.addLiveIn(PReg: InputPtrReg, RC: &AMDGPU::SGPR_64RegClass);
3174 MRI.setType(VReg, Ty: LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64));
3175 }
3176
3177 if (UserSGPRInfo.hasDispatchID()) {
3178 Register DispatchIDReg = Info.addDispatchID(TRI);
3179 MF.addLiveIn(PReg: DispatchIDReg, RC: &AMDGPU::SGPR_64RegClass);
3180 CCInfo.AllocateReg(Reg: DispatchIDReg);
3181 }
3182
3183 if (UserSGPRInfo.hasFlatScratchInit() && !getSubtarget()->isAmdPalOS()) {
3184 Register FlatScratchInitReg = Info.addFlatScratchInit(TRI);
3185 MF.addLiveIn(PReg: FlatScratchInitReg, RC: &AMDGPU::SGPR_64RegClass);
3186 CCInfo.AllocateReg(Reg: FlatScratchInitReg);
3187 }
3188
3189 if (UserSGPRInfo.hasPrivateSegmentSize()) {
3190 Register PrivateSegmentSizeReg = Info.addPrivateSegmentSize(TRI);
3191 MF.addLiveIn(PReg: PrivateSegmentSizeReg, RC: &AMDGPU::SGPR_32RegClass);
3192 CCInfo.AllocateReg(Reg: PrivateSegmentSizeReg);
3193 }
3194
3195 // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
3196 // these from the dispatch pointer.
3197}
3198
3199// Allocate pre-loaded kernel arguemtns. Arguments to be preloading must be
3200// sequential starting from the first argument.
3201void SITargetLowering::allocatePreloadKernArgSGPRs(
3202 CCState &CCInfo, SmallVectorImpl<CCValAssign> &ArgLocs,
3203 const SmallVectorImpl<ISD::InputArg> &Ins, MachineFunction &MF,
3204 const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const {
3205 Function &F = MF.getFunction();
3206 unsigned LastExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
3207 GCNUserSGPRUsageInfo &SGPRInfo = Info.getUserSGPRInfo();
3208 bool InPreloadSequence = true;
3209 unsigned InIdx = 0;
3210 bool AlignedForImplictArgs = false;
3211 unsigned ImplicitArgOffset = 0;
3212 for (auto &Arg : F.args()) {
3213 if (!InPreloadSequence || !Arg.hasInRegAttr())
3214 break;
3215
3216 unsigned ArgIdx = Arg.getArgNo();
3217 // Don't preload non-original args or parts not in the current preload
3218 // sequence.
3219 if (InIdx < Ins.size() &&
3220 (!Ins[InIdx].isOrigArg() || Ins[InIdx].getOrigArgIndex() != ArgIdx))
3221 break;
3222
3223 for (; InIdx < Ins.size() && Ins[InIdx].isOrigArg() &&
3224 Ins[InIdx].getOrigArgIndex() == ArgIdx;
3225 InIdx++) {
3226 assert(ArgLocs[ArgIdx].isMemLoc());
3227 auto &ArgLoc = ArgLocs[InIdx];
3228 const Align KernelArgBaseAlign = Align(16);
3229 unsigned ArgOffset = ArgLoc.getLocMemOffset();
3230 Align Alignment = commonAlignment(A: KernelArgBaseAlign, Offset: ArgOffset);
3231 unsigned NumAllocSGPRs =
3232 alignTo(Value: ArgLoc.getLocVT().getFixedSizeInBits(), Align: 32) / 32;
3233
3234 // Fix alignment for hidden arguments.
3235 if (Arg.hasAttribute(Kind: "amdgpu-hidden-argument")) {
3236 if (!AlignedForImplictArgs) {
3237 ImplicitArgOffset =
3238 alignTo(Size: LastExplicitArgOffset,
3239 A: Subtarget->getAlignmentForImplicitArgPtr()) -
3240 LastExplicitArgOffset;
3241 AlignedForImplictArgs = true;
3242 }
3243 ArgOffset += ImplicitArgOffset;
3244 }
3245
3246 // Arg is preloaded into the previous SGPR.
3247 if (ArgLoc.getLocVT().getStoreSize() < 4 && Alignment < 4) {
3248 assert(InIdx >= 1 && "No previous SGPR");
3249 Info.getArgInfo().PreloadKernArgs[InIdx].Regs.push_back(
3250 Elt: Info.getArgInfo().PreloadKernArgs[InIdx - 1].Regs[0]);
3251 continue;
3252 }
3253
3254 unsigned Padding = ArgOffset - LastExplicitArgOffset;
3255 unsigned PaddingSGPRs = alignTo(Value: Padding, Align: 4) / 4;
3256 // Check for free user SGPRs for preloading.
3257 if (PaddingSGPRs + NumAllocSGPRs > SGPRInfo.getNumFreeUserSGPRs()) {
3258 InPreloadSequence = false;
3259 break;
3260 }
3261
3262 // Preload this argument.
3263 const TargetRegisterClass *RC =
3264 TRI.getSGPRClassForBitWidth(BitWidth: NumAllocSGPRs * 32);
3265 SmallVectorImpl<MCRegister> *PreloadRegs =
3266 Info.addPreloadedKernArg(TRI, RC, AllocSizeDWord: NumAllocSGPRs, KernArgIdx: InIdx, PaddingSGPRs);
3267
3268 if (PreloadRegs->size() > 1)
3269 RC = &AMDGPU::SGPR_32RegClass;
3270 for (auto &Reg : *PreloadRegs) {
3271 assert(Reg);
3272 MF.addLiveIn(PReg: Reg, RC);
3273 CCInfo.AllocateReg(Reg);
3274 }
3275
3276 LastExplicitArgOffset = NumAllocSGPRs * 4 + ArgOffset;
3277 }
3278 }
3279}
3280
3281void SITargetLowering::allocateLDSKernelId(CCState &CCInfo, MachineFunction &MF,
3282 const SIRegisterInfo &TRI,
3283 SIMachineFunctionInfo &Info) const {
3284 // Always allocate this last since it is a synthetic preload.
3285 if (Info.hasLDSKernelId()) {
3286 Register Reg = Info.addLDSKernelId();
3287 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3288 CCInfo.AllocateReg(Reg);
3289 }
3290}
3291
3292// Allocate special input registers that are initialized per-wave.
3293void SITargetLowering::allocateSystemSGPRs(CCState &CCInfo, MachineFunction &MF,
3294 SIMachineFunctionInfo &Info,
3295 CallingConv::ID CallConv,
3296 bool IsShader) const {
3297 bool HasArchitectedSGPRs = Subtarget->hasArchitectedSGPRs();
3298 if (Subtarget->hasUserSGPRInit16BugInWave32() && !IsShader) {
3299 // Note: user SGPRs are handled by the front-end for graphics shaders
3300 // Pad up the used user SGPRs with dead inputs.
3301
3302 // TODO: NumRequiredSystemSGPRs computation should be adjusted appropriately
3303 // before enabling architected SGPRs for workgroup IDs.
3304 assert(!HasArchitectedSGPRs && "Unhandled feature for the subtarget");
3305
3306 unsigned CurrentUserSGPRs = Info.getNumUserSGPRs();
3307 // Note we do not count the PrivateSegmentWaveByteOffset. We do not want to
3308 // rely on it to reach 16 since if we end up having no stack usage, it will
3309 // not really be added.
3310 unsigned NumRequiredSystemSGPRs =
3311 Info.hasWorkGroupIDX() + Info.hasWorkGroupIDY() +
3312 Info.hasWorkGroupIDZ() + Info.hasWorkGroupInfo();
3313 for (unsigned i = NumRequiredSystemSGPRs + CurrentUserSGPRs; i < 16; ++i) {
3314 Register Reg = Info.addReservedUserSGPR();
3315 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3316 CCInfo.AllocateReg(Reg);
3317 }
3318 }
3319
3320 if (!HasArchitectedSGPRs) {
3321 if (Info.hasWorkGroupIDX()) {
3322 Register Reg = Info.addWorkGroupIDX();
3323 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3324 CCInfo.AllocateReg(Reg);
3325 }
3326
3327 if (Info.hasWorkGroupIDY()) {
3328 Register Reg = Info.addWorkGroupIDY();
3329 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3330 CCInfo.AllocateReg(Reg);
3331 }
3332
3333 if (Info.hasWorkGroupIDZ()) {
3334 Register Reg = Info.addWorkGroupIDZ();
3335 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3336 CCInfo.AllocateReg(Reg);
3337 }
3338 }
3339
3340 if (Info.hasWorkGroupInfo()) {
3341 Register Reg = Info.addWorkGroupInfo();
3342 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3343 CCInfo.AllocateReg(Reg);
3344 }
3345
3346 if (Info.hasPrivateSegmentWaveByteOffset()) {
3347 // Scratch wave offset passed in system SGPR.
3348 unsigned PrivateSegmentWaveByteOffsetReg;
3349
3350 if (IsShader) {
3351 PrivateSegmentWaveByteOffsetReg =
3352 Info.getPrivateSegmentWaveByteOffsetSystemSGPR();
3353
3354 // This is true if the scratch wave byte offset doesn't have a fixed
3355 // location.
3356 if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) {
3357 PrivateSegmentWaveByteOffsetReg = findFirstFreeSGPR(CCInfo);
3358 Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
3359 }
3360 } else
3361 PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset();
3362
3363 MF.addLiveIn(PReg: PrivateSegmentWaveByteOffsetReg, RC: &AMDGPU::SGPR_32RegClass);
3364 CCInfo.AllocateReg(Reg: PrivateSegmentWaveByteOffsetReg);
3365 }
3366
3367 assert(!Subtarget->hasUserSGPRInit16BugInWave32() || IsShader ||
3368 Info.getNumPreloadedSGPRs() >= 16);
3369}
3370
3371static void reservePrivateMemoryRegs(const TargetMachine &TM,
3372 MachineFunction &MF,
3373 const SIRegisterInfo &TRI,
3374 SIMachineFunctionInfo &Info) {
3375 // Now that we've figured out where the scratch register inputs are, see if
3376 // should reserve the arguments and use them directly.
3377 MachineFrameInfo &MFI = MF.getFrameInfo();
3378 bool HasStackObjects = MFI.hasStackObjects();
3379 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
3380
3381 // Record that we know we have non-spill stack objects so we don't need to
3382 // check all stack objects later.
3383 if (HasStackObjects)
3384 Info.setHasNonSpillStackObjects(true);
3385
3386 // Everything live out of a block is spilled with fast regalloc, so it's
3387 // almost certain that spilling will be required.
3388 if (TM.getOptLevel() == CodeGenOptLevel::None)
3389 HasStackObjects = true;
3390
3391 // For now assume stack access is needed in any callee functions, so we need
3392 // the scratch registers to pass in.
3393 bool RequiresStackAccess = HasStackObjects || MFI.hasCalls();
3394
3395 if (!ST.hasFlatScratchEnabled()) {
3396 if (RequiresStackAccess && ST.isAmdHsaOrMesa(F: MF.getFunction())) {
3397 // If we have stack objects, we unquestionably need the private buffer
3398 // resource. For the Code Object V2 ABI, this will be the first 4 user
3399 // SGPR inputs. We can reserve those and use them directly.
3400
3401 Register PrivateSegmentBufferReg =
3402 Info.getPreloadedReg(Value: AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
3403 Info.setScratchRSrcReg(PrivateSegmentBufferReg);
3404 } else {
3405 unsigned ReservedBufferReg = TRI.reservedPrivateSegmentBufferReg(MF);
3406 // We tentatively reserve the last registers (skipping the last registers
3407 // which may contain VCC, FLAT_SCR, and XNACK). After register allocation,
3408 // we'll replace these with the ones immediately after those which were
3409 // really allocated. In the prologue copies will be inserted from the
3410 // argument to these reserved registers.
3411
3412 // Without HSA, relocations are used for the scratch pointer and the
3413 // buffer resource setup is always inserted in the prologue. Scratch wave
3414 // offset is still in an input SGPR.
3415 Info.setScratchRSrcReg(ReservedBufferReg);
3416 }
3417 }
3418
3419 MachineRegisterInfo &MRI = MF.getRegInfo();
3420
3421 // For entry functions we have to set up the stack pointer if we use it,
3422 // whereas non-entry functions get this "for free". This means there is no
3423 // intrinsic advantage to using S32 over S34 in cases where we do not have
3424 // calls but do need a frame pointer (i.e. if we are requested to have one
3425 // because frame pointer elimination is disabled). To keep things simple we
3426 // only ever use S32 as the call ABI stack pointer, and so using it does not
3427 // imply we need a separate frame pointer.
3428 //
3429 // Try to use s32 as the SP, but move it if it would interfere with input
3430 // arguments. This won't work with calls though.
3431 //
3432 // FIXME: Move SP to avoid any possible inputs, or find a way to spill input
3433 // registers.
3434 if (!MRI.isLiveIn(Reg: AMDGPU::SGPR32)) {
3435 Info.setStackPtrOffsetReg(AMDGPU::SGPR32);
3436 } else {
3437 assert(AMDGPU::isShader(MF.getFunction().getCallingConv()));
3438
3439 if (MFI.hasCalls())
3440 report_fatal_error(reason: "call in graphics shader with too many input SGPRs");
3441
3442 for (unsigned Reg : AMDGPU::SGPR_32RegClass) {
3443 if (!MRI.isLiveIn(Reg)) {
3444 Info.setStackPtrOffsetReg(Reg);
3445 break;
3446 }
3447 }
3448
3449 if (Info.getStackPtrOffsetReg() == AMDGPU::SP_REG)
3450 report_fatal_error(reason: "failed to find register for SP");
3451 }
3452
3453 // hasFP should be accurate for entry functions even before the frame is
3454 // finalized, because it does not rely on the known stack size, only
3455 // properties like whether variable sized objects are present.
3456 if (ST.getFrameLowering()->hasFP(MF)) {
3457 Info.setFrameOffsetReg(AMDGPU::SGPR33);
3458 }
3459}
3460
3461bool SITargetLowering::supportSplitCSR(MachineFunction *MF) const {
3462 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
3463 return !Info->isEntryFunction();
3464}
3465
3466void SITargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {}
3467
3468void SITargetLowering::insertCopiesSplitCSR(
3469 MachineBasicBlock *Entry,
3470 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
3471 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3472
3473 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(MF: Entry->getParent());
3474 if (!IStart)
3475 return;
3476
3477 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
3478 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
3479 MachineBasicBlock::iterator MBBI = Entry->begin();
3480 for (const MCPhysReg *I = IStart; *I; ++I) {
3481 const TargetRegisterClass *RC = nullptr;
3482 if (AMDGPU::SReg_64RegClass.contains(Reg: *I))
3483 RC = &AMDGPU::SGPR_64RegClass;
3484 else if (AMDGPU::SReg_32RegClass.contains(Reg: *I))
3485 RC = &AMDGPU::SGPR_32RegClass;
3486 else
3487 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3488
3489 Register NewVR = MRI->createVirtualRegister(RegClass: RC);
3490 // Create copy from CSR to a virtual register.
3491 Entry->addLiveIn(PhysReg: *I);
3492 BuildMI(BB&: *Entry, I: MBBI, MIMD: DebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewVR)
3493 .addReg(RegNo: *I);
3494
3495 // Insert the copy-back instructions right before the terminator.
3496 for (auto *Exit : Exits)
3497 BuildMI(BB&: *Exit, I: Exit->getFirstTerminator(), MIMD: DebugLoc(),
3498 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: *I)
3499 .addReg(RegNo: NewVR);
3500 }
3501}
3502
3503SDValue SITargetLowering::LowerFormalArguments(
3504 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3505 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3506 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3507 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3508
3509 MachineFunction &MF = DAG.getMachineFunction();
3510 const Function &Fn = MF.getFunction();
3511 FunctionType *FType = MF.getFunction().getFunctionType();
3512 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
3513 bool IsError = false;
3514
3515 if (Subtarget->isAmdHsaOS() && AMDGPU::isGraphics(CC: CallConv)) {
3516 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
3517 Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc()));
3518 IsError = true;
3519 }
3520
3521 SmallVector<ISD::InputArg, 16> Splits;
3522 SmallVector<CCValAssign, 16> ArgLocs;
3523 BitVector Skipped(Fn.arg_size());
3524 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
3525 *DAG.getContext());
3526
3527 bool IsGraphics = AMDGPU::isGraphics(CC: CallConv);
3528 bool IsKernel = AMDGPU::isKernel(CC: CallConv);
3529 bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CC: CallConv);
3530
3531 if (IsGraphics) {
3532 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info->getUserSGPRInfo();
3533 assert(!UserSGPRInfo.hasDispatchPtr() &&
3534 !UserSGPRInfo.hasKernargSegmentPtr() && !Info->hasWorkGroupInfo() &&
3535 !Info->hasLDSKernelId() && !Info->hasWorkItemIDX() &&
3536 !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ());
3537 (void)UserSGPRInfo;
3538 if (!Subtarget->hasFlatScratchEnabled())
3539 assert(!UserSGPRInfo.hasFlatScratchInit());
3540 if ((CallConv != CallingConv::AMDGPU_CS &&
3541 CallConv != CallingConv::AMDGPU_Gfx &&
3542 CallConv != CallingConv::AMDGPU_Gfx_WholeWave) ||
3543 !Subtarget->hasArchitectedSGPRs())
3544 assert(!Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() &&
3545 !Info->hasWorkGroupIDZ());
3546 }
3547
3548 bool IsWholeWaveFunc = Info->isWholeWaveFunction();
3549
3550 if (CallConv == CallingConv::AMDGPU_PS) {
3551 processPSInputArgs(Splits, CallConv, Ins, Skipped, FType, Info);
3552
3553 // At least one interpolation mode must be enabled or else the GPU will
3554 // hang.
3555 //
3556 // Check PSInputAddr instead of PSInputEnable. The idea is that if the user
3557 // set PSInputAddr, the user wants to enable some bits after the compilation
3558 // based on run-time states. Since we can't know what the final PSInputEna
3559 // will look like, so we shouldn't do anything here and the user should take
3560 // responsibility for the correct programming.
3561 //
3562 // Otherwise, the following restrictions apply:
3563 // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
3564 // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
3565 // enabled too.
3566 if ((Info->getPSInputAddr() & 0x7F) == 0 ||
3567 ((Info->getPSInputAddr() & 0xF) == 0 && Info->isPSInputAllocated(Index: 11))) {
3568 CCInfo.AllocateReg(Reg: AMDGPU::VGPR0);
3569 CCInfo.AllocateReg(Reg: AMDGPU::VGPR1);
3570 Info->markPSInputAllocated(Index: 0);
3571 Info->markPSInputEnabled(Index: 0);
3572 }
3573 if (Subtarget->isAmdPalOS()) {
3574 // For isAmdPalOS, the user does not enable some bits after compilation
3575 // based on run-time states; the register values being generated here are
3576 // the final ones set in hardware. Therefore we need to apply the
3577 // workaround to PSInputAddr and PSInputEnable together. (The case where
3578 // a bit is set in PSInputAddr but not PSInputEnable is where the
3579 // frontend set up an input arg for a particular interpolation mode, but
3580 // nothing uses that input arg. Really we should have an earlier pass
3581 // that removes such an arg.)
3582 unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
3583 if ((PsInputBits & 0x7F) == 0 ||
3584 ((PsInputBits & 0xF) == 0 && (PsInputBits >> 11 & 1)))
3585 Info->markPSInputEnabled(Index: llvm::countr_zero(Val: Info->getPSInputAddr()));
3586 }
3587 } else if (IsKernel) {
3588 assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX());
3589 } else {
3590 Splits.append(in_start: IsWholeWaveFunc ? std::next(x: Ins.begin()) : Ins.begin(),
3591 in_end: Ins.end());
3592 }
3593
3594 if (IsKernel)
3595 analyzeFormalArgumentsCompute(State&: CCInfo, Ins);
3596
3597 if (IsEntryFunc) {
3598 allocateSpecialEntryInputVGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3599 allocateHSAUserSGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3600 if (IsKernel && Subtarget->hasKernargPreload())
3601 allocatePreloadKernArgSGPRs(CCInfo, ArgLocs, Ins, MF, TRI: *TRI, Info&: *Info);
3602
3603 allocateLDSKernelId(CCInfo, MF, TRI: *TRI, Info&: *Info);
3604 } else if (!IsGraphics) {
3605 // For the fixed ABI, pass workitem IDs in the last argument register.
3606 allocateSpecialInputVGPRsFixed(CCInfo, MF, TRI: *TRI, Info&: *Info);
3607
3608 // FIXME: Sink this into allocateSpecialInputSGPRs
3609 if (!Subtarget->hasFlatScratchEnabled())
3610 CCInfo.AllocateReg(Reg: Info->getScratchRSrcReg());
3611
3612 allocateSpecialInputSGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3613 }
3614
3615 if (!IsKernel) {
3616 CCAssignFn *AssignFn = CCAssignFnForCall(CC: CallConv, IsVarArg: isVarArg);
3617 CCInfo.AnalyzeFormalArguments(Ins: Splits, Fn: AssignFn);
3618
3619 // This assumes the registers are allocated by CCInfo in ascending order
3620 // with no gaps.
3621 Info->setNumWaveDispatchSGPRs(
3622 CCInfo.getFirstUnallocated(Regs: AMDGPU::SGPR_32RegClass.getRegisters()));
3623 Info->setNumWaveDispatchVGPRs(
3624 CCInfo.getFirstUnallocated(Regs: AMDGPU::VGPR_32RegClass.getRegisters()));
3625 } else if (Info->getNumKernargPreloadedSGPRs()) {
3626 Info->setNumWaveDispatchSGPRs(Info->getNumUserSGPRs());
3627 }
3628
3629 SmallVector<SDValue, 16> Chains;
3630
3631 if (IsWholeWaveFunc) {
3632 SDValue Setup = DAG.getNode(Opcode: AMDGPUISD::WHOLE_WAVE_SETUP, DL,
3633 ResultTys: {MVT::i1, MVT::Other}, Ops: Chain);
3634 InVals.push_back(Elt: Setup.getValue(R: 0));
3635 Chains.push_back(Elt: Setup.getValue(R: 1));
3636 }
3637
3638 // FIXME: This is the minimum kernel argument alignment. We should improve
3639 // this to the maximum alignment of the arguments.
3640 //
3641 // FIXME: Alignment of explicit arguments totally broken with non-0 explicit
3642 // kern arg offset.
3643 const Align KernelArgBaseAlign = Align(16);
3644
3645 for (unsigned i = IsWholeWaveFunc ? 1 : 0, e = Ins.size(), ArgIdx = 0; i != e;
3646 ++i) {
3647 const ISD::InputArg &Arg = Ins[i];
3648 if ((Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) || IsError) {
3649 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
3650 continue;
3651 }
3652
3653 CCValAssign &VA = ArgLocs[ArgIdx++];
3654 MVT VT = VA.getLocVT();
3655
3656 if (IsEntryFunc && VA.isMemLoc()) {
3657 VT = Ins[i].VT;
3658 EVT MemVT = VA.getLocVT();
3659
3660 const uint64_t Offset = VA.getLocMemOffset();
3661 Align Alignment = commonAlignment(A: KernelArgBaseAlign, Offset);
3662
3663 if (Arg.Flags.isByRef()) {
3664 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL: DL, Chain, Offset);
3665
3666 const GCNTargetMachine &TM =
3667 static_cast<const GCNTargetMachine &>(getTargetMachine());
3668 if (!TM.isNoopAddrSpaceCast(SrcAS: AMDGPUAS::CONSTANT_ADDRESS,
3669 DestAS: Arg.Flags.getPointerAddrSpace())) {
3670 Ptr = DAG.getAddrSpaceCast(dl: DL, VT, Ptr, SrcAS: AMDGPUAS::CONSTANT_ADDRESS,
3671 DestAS: Arg.Flags.getPointerAddrSpace());
3672 }
3673
3674 InVals.push_back(Elt: Ptr);
3675 continue;
3676 }
3677
3678 SDValue NewArg;
3679 if (Arg.isOrigArg() && Info->getArgInfo().PreloadKernArgs.count(Val: i)) {
3680 if (MemVT.getStoreSize() < 4 && Alignment < 4) {
3681 // In this case the argument is packed into the previous preload SGPR.
3682 int64_t AlignDownOffset = alignDown(Value: Offset, Align: 4);
3683 int64_t OffsetDiff = Offset - AlignDownOffset;
3684 EVT IntVT = MemVT.changeTypeToInteger();
3685
3686 const SIMachineFunctionInfo *Info =
3687 MF.getInfo<SIMachineFunctionInfo>();
3688 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
3689 Register Reg =
3690 Info->getArgInfo().PreloadKernArgs.find(Val: i)->getSecond().Regs[0];
3691
3692 assert(Reg);
3693 Register VReg = MRI.getLiveInVirtReg(PReg: Reg);
3694 SDValue Copy = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::i32);
3695
3696 SDValue ShiftAmt = DAG.getConstant(Val: OffsetDiff * 8, DL, VT: MVT::i32);
3697 SDValue Extract = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Copy, N2: ShiftAmt);
3698
3699 SDValue ArgVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: Extract);
3700 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MemVT, Operand: ArgVal);
3701 NewArg = convertArgType(DAG, VT, MemVT, SL: DL, Val: ArgVal,
3702 Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3703
3704 NewArg = DAG.getMergeValues(Ops: {NewArg, Copy.getValue(R: 1)}, dl: DL);
3705 } else {
3706 const SIMachineFunctionInfo *Info =
3707 MF.getInfo<SIMachineFunctionInfo>();
3708 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
3709 const SmallVectorImpl<MCRegister> &PreloadRegs =
3710 Info->getArgInfo().PreloadKernArgs.find(Val: i)->getSecond().Regs;
3711
3712 SDValue Copy;
3713 if (PreloadRegs.size() == 1) {
3714 Register VReg = MRI.getLiveInVirtReg(PReg: PreloadRegs[0]);
3715 const TargetRegisterClass *RC = MRI.getRegClass(Reg: VReg);
3716 NewArg = DAG.getCopyFromReg(
3717 Chain, dl: DL, Reg: VReg,
3718 VT: EVT::getIntegerVT(Context&: *DAG.getContext(),
3719 BitWidth: TRI->getRegSizeInBits(RC: *RC)));
3720
3721 } else {
3722 // If the kernarg alignment does not match the alignment of the SGPR
3723 // tuple RC that can accommodate this argument, it will be built up
3724 // via copies from from the individual SGPRs that the argument was
3725 // preloaded to.
3726 SmallVector<SDValue, 4> Elts;
3727 for (auto Reg : PreloadRegs) {
3728 Register VReg = MRI.getLiveInVirtReg(PReg: Reg);
3729 Copy = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::i32);
3730 Elts.push_back(Elt: Copy);
3731 }
3732 NewArg =
3733 DAG.getBuildVector(VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
3734 NumElements: PreloadRegs.size()),
3735 DL, Ops: Elts);
3736 }
3737
3738 // If the argument was preloaded to multiple consecutive 32-bit
3739 // registers because of misalignment between addressable SGPR tuples
3740 // and the argument size, we can still assume that because of kernarg
3741 // segment alignment restrictions that NewArg's size is the same as
3742 // MemVT and just do a bitcast. If MemVT is less than 32-bits we add a
3743 // truncate since we cannot preload to less than a single SGPR and the
3744 // MemVT may be smaller.
3745 EVT MemVTInt =
3746 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
3747 if (MemVT.bitsLT(VT: NewArg.getSimpleValueType()))
3748 NewArg = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MemVTInt, Operand: NewArg);
3749
3750 NewArg = DAG.getBitcast(VT: MemVT, V: NewArg);
3751 NewArg = convertArgType(DAG, VT, MemVT, SL: DL, Val: NewArg,
3752 Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3753 NewArg = DAG.getMergeValues(Ops: {NewArg, Chain}, dl: DL);
3754 }
3755 } else {
3756 // Hidden arguments that are in the kernel signature must be preloaded
3757 // to user SGPRs. Print a diagnostic error if a hidden argument is in
3758 // the argument list and is not preloaded.
3759 if (Arg.isOrigArg()) {
3760 Argument *OrigArg = Fn.getArg(i: Arg.getOrigArgIndex());
3761 if (OrigArg->hasAttribute(Kind: "amdgpu-hidden-argument")) {
3762 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
3763 *OrigArg->getParent(),
3764 "hidden argument in kernel signature was not preloaded",
3765 DL.getDebugLoc()));
3766 }
3767 }
3768
3769 NewArg =
3770 lowerKernargMemParameter(DAG, VT, MemVT, SL: DL, Chain, Offset,
3771 Alignment, Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3772 }
3773 Chains.push_back(Elt: NewArg.getValue(R: 1));
3774
3775 auto *ParamTy =
3776 dyn_cast<PointerType>(Val: FType->getParamType(i: Ins[i].getOrigArgIndex()));
3777 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS &&
3778 ParamTy &&
3779 (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
3780 ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS)) {
3781 // On SI local pointers are just offsets into LDS, so they are always
3782 // less than 16-bits. On CI and newer they could potentially be
3783 // real pointers, so we can't guarantee their size.
3784 NewArg = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: NewArg.getValueType(), N1: NewArg,
3785 N2: DAG.getValueType(MVT::i16));
3786 }
3787
3788 InVals.push_back(Elt: NewArg);
3789 continue;
3790 }
3791 if (!IsEntryFunc && VA.isMemLoc()) {
3792 SDValue Val = lowerStackParameter(DAG, VA, SL: DL, Chain, Arg);
3793 InVals.push_back(Elt: Val);
3794 if (!Arg.Flags.isByVal())
3795 Chains.push_back(Elt: Val.getValue(R: 1));
3796 continue;
3797 }
3798
3799 assert(VA.isRegLoc() && "Parameter must be in a register!");
3800
3801 Register Reg = VA.getLocReg();
3802 const TargetRegisterClass *RC = nullptr;
3803 if (AMDGPU::VGPR_32RegClass.contains(Reg))
3804 RC = &AMDGPU::VGPR_32RegClass;
3805 else if (AMDGPU::SGPR_32RegClass.contains(Reg))
3806 RC = &AMDGPU::SGPR_32RegClass;
3807 else
3808 llvm_unreachable("Unexpected register class in LowerFormalArguments!");
3809
3810 Reg = MF.addLiveIn(PReg: Reg, RC);
3811 SDValue Val = DAG.getCopyFromReg(Chain, dl: DL, Reg, VT);
3812 if (Arg.Flags.isInReg() && RC == &AMDGPU::VGPR_32RegClass) {
3813 // FIXME: Need to forward the chains created by `CopyFromReg`s, make sure
3814 // they will read physical regs before any side effect instructions.
3815 SDValue ReadFirstLane =
3816 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
3817 Val = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Val.getValueType(),
3818 N1: ReadFirstLane, N2: Val);
3819 }
3820
3821 if (Arg.Flags.isSRet()) {
3822 // The return object should be reasonably addressable.
3823 Val = annotateStackObjectPointer(Ptr: Val, DAG, DL,
3824 Alignment: Arg.Flags.getNonZeroMemAlign());
3825 }
3826
3827 Val = convertABITypeToValueType(DAG, Val, VA, SL: DL);
3828 InVals.push_back(Elt: Val);
3829 }
3830
3831 // Start adding system SGPRs.
3832 if (IsEntryFunc)
3833 allocateSystemSGPRs(CCInfo, MF, Info&: *Info, CallConv, IsShader: IsGraphics);
3834
3835 unsigned StackArgSize = CCInfo.getStackSize();
3836 Info->setBytesInStackArgArea(StackArgSize);
3837
3838 return Chains.empty() ? Chain
3839 : DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
3840}
3841
3842// TODO: If return values can't fit in registers, we should return as many as
3843// possible in registers before passing on stack.
3844bool SITargetLowering::CanLowerReturn(
3845 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
3846 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
3847 const Type *RetTy) const {
3848 // Replacing returns with sret/stack usage doesn't make sense for shaders.
3849 // FIXME: Also sort of a workaround for custom vector splitting in LowerReturn
3850 // for shaders. Vector types should be explicitly handled by CC.
3851 if (AMDGPU::isEntryFunctionCC(CC: CallConv))
3852 return true;
3853
3854 SmallVector<CCValAssign, 16> RVLocs;
3855 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
3856 if (!CCInfo.CheckReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, IsVarArg)))
3857 return false;
3858
3859 // We must use the stack if return would require unavailable registers.
3860 unsigned MaxNumVGPRs = Subtarget->getMaxNumVGPRs(MF);
3861 unsigned TotalNumVGPRs = Subtarget->getAddressableNumArchVGPRs();
3862 for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i)
3863 if (CCInfo.isAllocated(Reg: AMDGPU::VGPR_32RegClass.getRegister(i)))
3864 return false;
3865
3866 return true;
3867}
3868
3869SDValue
3870SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
3871 bool isVarArg,
3872 const SmallVectorImpl<ISD::OutputArg> &Outs,
3873 const SmallVectorImpl<SDValue> &OutVals,
3874 const SDLoc &DL, SelectionDAG &DAG) const {
3875 MachineFunction &MF = DAG.getMachineFunction();
3876 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
3877 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3878
3879 if (AMDGPU::isKernel(CC: CallConv)) {
3880 return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs,
3881 OutVals, DL, DAG);
3882 }
3883
3884 bool IsShader = AMDGPU::isShader(CC: CallConv);
3885
3886 Info->setIfReturnsVoid(Outs.empty());
3887 bool IsWaveEnd = Info->returnsVoid() && IsShader;
3888
3889 // CCValAssign - represent the assignment of the return value to a location.
3890 SmallVector<CCValAssign, 48> RVLocs;
3891
3892 // CCState - Info about the registers and stack slots.
3893 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
3894 *DAG.getContext());
3895
3896 // Analyze outgoing return values.
3897 CCInfo.AnalyzeReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, IsVarArg: isVarArg));
3898
3899 SDValue Glue;
3900 SmallVector<SDValue, 48> RetOps;
3901 RetOps.push_back(Elt: Chain); // Operand #0 = Chain (updated below)
3902
3903 SDValue ReadFirstLane =
3904 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
3905 // Copy the result values into the output registers.
3906 for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E;
3907 ++I, ++RealRVLocIdx) {
3908 CCValAssign &VA = RVLocs[I];
3909 assert(VA.isRegLoc() && "Can only return in registers!");
3910 // TODO: Partially return in registers if return values don't fit.
3911 SDValue Arg = OutVals[RealRVLocIdx];
3912
3913 // Copied from other backends.
3914 switch (VA.getLocInfo()) {
3915 case CCValAssign::Full:
3916 break;
3917 case CCValAssign::BCvt:
3918 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getLocVT(), Operand: Arg);
3919 break;
3920 case CCValAssign::SExt:
3921 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3922 break;
3923 case CCValAssign::ZExt:
3924 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3925 break;
3926 case CCValAssign::AExt:
3927 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3928 break;
3929 default:
3930 llvm_unreachable("Unknown loc info!");
3931 }
3932 if (TRI->isSGPRPhysReg(Reg: VA.getLocReg()))
3933 Arg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Arg.getValueType(),
3934 N1: ReadFirstLane, N2: Arg);
3935 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VA.getLocReg(), N: Arg, Glue);
3936 Glue = Chain.getValue(R: 1);
3937 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
3938 }
3939
3940 // FIXME: Does sret work properly?
3941 if (!Info->isEntryFunction()) {
3942 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
3943 const MCPhysReg *I =
3944 TRI->getCalleeSavedRegsViaCopy(MF: &DAG.getMachineFunction());
3945 if (I) {
3946 for (; *I; ++I) {
3947 if (AMDGPU::SReg_64RegClass.contains(Reg: *I))
3948 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::i64));
3949 else if (AMDGPU::SReg_32RegClass.contains(Reg: *I))
3950 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::i32));
3951 else
3952 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3953 }
3954 }
3955 }
3956
3957 // Update chain and glue.
3958 RetOps[0] = Chain;
3959 if (Glue.getNode())
3960 RetOps.push_back(Elt: Glue);
3961
3962 unsigned Opc = AMDGPUISD::ENDPGM;
3963 if (!IsWaveEnd)
3964 Opc = Info->isWholeWaveFunction() ? AMDGPUISD::WHOLE_WAVE_RETURN
3965 : IsShader ? AMDGPUISD::RETURN_TO_EPILOG
3966 : AMDGPUISD::RET_GLUE;
3967 return DAG.getNode(Opcode: Opc, DL, VT: MVT::Other, Ops: RetOps);
3968}
3969
3970SDValue SITargetLowering::LowerCallResult(
3971 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool IsVarArg,
3972 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3973 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn,
3974 SDValue ThisVal) const {
3975 CCAssignFn *RetCC = CCAssignFnForReturn(CC: CallConv, IsVarArg);
3976
3977 // Assign locations to each value returned by this call.
3978 SmallVector<CCValAssign, 16> RVLocs;
3979 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
3980 *DAG.getContext());
3981 CCInfo.AnalyzeCallResult(Ins, Fn: RetCC);
3982
3983 // Copy all of the result registers out of their specified physreg.
3984 for (CCValAssign VA : RVLocs) {
3985 SDValue Val;
3986
3987 if (VA.isRegLoc()) {
3988 Val =
3989 DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue: InGlue);
3990 Chain = Val.getValue(R: 1);
3991 InGlue = Val.getValue(R: 2);
3992 } else if (VA.isMemLoc()) {
3993 report_fatal_error(reason: "TODO: return values in memory");
3994 } else
3995 llvm_unreachable("unknown argument location type");
3996
3997 switch (VA.getLocInfo()) {
3998 case CCValAssign::Full:
3999 break;
4000 case CCValAssign::BCvt:
4001 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Val);
4002 break;
4003 case CCValAssign::ZExt:
4004 Val = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: VA.getLocVT(), N1: Val,
4005 N2: DAG.getValueType(VA.getValVT()));
4006 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
4007 break;
4008 case CCValAssign::SExt:
4009 Val = DAG.getNode(Opcode: ISD::AssertSext, DL, VT: VA.getLocVT(), N1: Val,
4010 N2: DAG.getValueType(VA.getValVT()));
4011 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
4012 break;
4013 case CCValAssign::AExt:
4014 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
4015 break;
4016 default:
4017 llvm_unreachable("Unknown loc info!");
4018 }
4019
4020 InVals.push_back(Elt: Val);
4021 }
4022
4023 return Chain;
4024}
4025
4026// Add code to pass special inputs required depending on used features separate
4027// from the explicit user arguments present in the IR.
4028void SITargetLowering::passSpecialInputs(
4029 CallLoweringInfo &CLI, CCState &CCInfo, const SIMachineFunctionInfo &Info,
4030 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
4031 SmallVectorImpl<SDValue> &MemOpChains, SDValue Chain) const {
4032 // If we don't have a call site, this was a call inserted by
4033 // legalization. These can never use special inputs.
4034 if (!CLI.CB)
4035 return;
4036
4037 SelectionDAG &DAG = CLI.DAG;
4038 const SDLoc &DL = CLI.DL;
4039 const Function &F = DAG.getMachineFunction().getFunction();
4040
4041 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
4042 const AMDGPUFunctionArgInfo &CallerArgInfo = Info.getArgInfo();
4043
4044 const AMDGPUFunctionArgInfo &CalleeArgInfo =
4045 AMDGPUFunctionArgInfo::FixedABIFunctionInfo;
4046
4047 // TODO: Unify with private memory register handling. This is complicated by
4048 // the fact that at least in kernels, the input argument is not necessarily
4049 // in the same location as the input.
4050 // clang-format off
4051 static constexpr std::pair<AMDGPUFunctionArgInfo::PreloadedValue,
4052 std::array<StringLiteral, 2>> ImplicitAttrs[] = {
4053 {AMDGPUFunctionArgInfo::DISPATCH_PTR, {"amdgpu-no-dispatch-ptr", ""}},
4054 {AMDGPUFunctionArgInfo::QUEUE_PTR, {"amdgpu-no-queue-ptr", ""}},
4055 {AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR, {"amdgpu-no-implicitarg-ptr", ""}},
4056 {AMDGPUFunctionArgInfo::DISPATCH_ID, {"amdgpu-no-dispatch-id", ""}},
4057 {AMDGPUFunctionArgInfo::WORKGROUP_ID_X, {"amdgpu-no-workgroup-id-x", "amdgpu-no-cluster-id-x"}},
4058 {AMDGPUFunctionArgInfo::WORKGROUP_ID_Y, {"amdgpu-no-workgroup-id-y", "amdgpu-no-cluster-id-y"}},
4059 {AMDGPUFunctionArgInfo::WORKGROUP_ID_Z, {"amdgpu-no-workgroup-id-z", "amdgpu-no-cluster-id-z"}},
4060 {AMDGPUFunctionArgInfo::LDS_KERNEL_ID, {"amdgpu-no-lds-kernel-id", ""}},
4061 };
4062 // clang-format on
4063
4064 for (auto [InputID, Attrs] : ImplicitAttrs) {
4065 // If the callee does not use the attribute value, skip copying the value.
4066 if (all_of(Range&: Attrs, P: [&](StringRef Attr) {
4067 return Attr.empty() || CLI.CB->hasFnAttr(Kind: Attr);
4068 }))
4069 continue;
4070
4071 const auto [OutgoingArg, ArgRC, ArgTy] =
4072 CalleeArgInfo.getPreloadedValue(Value: InputID);
4073 if (!OutgoingArg)
4074 continue;
4075
4076 const auto [IncomingArg, IncomingArgRC, Ty] =
4077 CallerArgInfo.getPreloadedValue(Value: InputID);
4078 assert(IncomingArgRC == ArgRC);
4079
4080 // All special arguments are ints for now.
4081 EVT ArgVT = TRI->getSpillSize(RC: *ArgRC) == 8 ? MVT::i64 : MVT::i32;
4082 SDValue InputReg;
4083
4084 if (IncomingArg) {
4085 InputReg = loadInputValue(DAG, RC: ArgRC, VT: ArgVT, SL: DL, Arg: *IncomingArg);
4086 } else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) {
4087 // The implicit arg ptr is special because it doesn't have a corresponding
4088 // input for kernels, and is computed from the kernarg segment pointer.
4089 InputReg = getImplicitArgPtr(DAG, SL: DL);
4090 } else if (InputID == AMDGPUFunctionArgInfo::LDS_KERNEL_ID) {
4091 std::optional<uint32_t> Id =
4092 AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F);
4093 if (Id.has_value()) {
4094 InputReg = DAG.getConstant(Val: *Id, DL, VT: ArgVT);
4095 } else {
4096 InputReg = DAG.getPOISON(VT: ArgVT);
4097 }
4098 } else {
4099 // We may have proven the input wasn't needed, although the ABI is
4100 // requiring it. We just need to allocate the register appropriately.
4101 InputReg = DAG.getPOISON(VT: ArgVT);
4102 }
4103
4104 if (OutgoingArg->isRegister()) {
4105 RegsToPass.emplace_back(Args: OutgoingArg->getRegister(), Args&: InputReg);
4106 if (!CCInfo.AllocateReg(Reg: OutgoingArg->getRegister()))
4107 report_fatal_error(reason: "failed to allocate implicit input argument");
4108 } else {
4109 unsigned SpecialArgOffset =
4110 CCInfo.AllocateStack(Size: ArgVT.getStoreSize(), Alignment: Align(4));
4111 SDValue ArgStore =
4112 storeStackInputValue(DAG, SL: DL, Chain, ArgVal: InputReg, Offset: SpecialArgOffset);
4113 MemOpChains.push_back(Elt: ArgStore);
4114 }
4115 }
4116
4117 // Pack workitem IDs into a single register or pass it as is if already
4118 // packed.
4119
4120 auto [OutgoingArg, ArgRC, Ty] =
4121 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_X);
4122 if (!OutgoingArg)
4123 std::tie(args&: OutgoingArg, args&: ArgRC, args&: Ty) =
4124 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
4125 if (!OutgoingArg)
4126 std::tie(args&: OutgoingArg, args&: ArgRC, args&: Ty) =
4127 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
4128 if (!OutgoingArg)
4129 return;
4130
4131 const ArgDescriptor *IncomingArgX = std::get<0>(
4132 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_X));
4133 const ArgDescriptor *IncomingArgY = std::get<0>(
4134 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Y));
4135 const ArgDescriptor *IncomingArgZ = std::get<0>(
4136 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Z));
4137
4138 SDValue InputReg;
4139 SDLoc SL;
4140
4141 const bool NeedWorkItemIDX = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-x");
4142 const bool NeedWorkItemIDY = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-y");
4143 const bool NeedWorkItemIDZ = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-z");
4144
4145 // If incoming ids are not packed we need to pack them.
4146 if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo.WorkItemIDX &&
4147 NeedWorkItemIDX) {
4148 if (Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 0) != 0) {
4149 InputReg = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgX);
4150 } else {
4151 InputReg = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
4152 }
4153 }
4154
4155 if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo.WorkItemIDY &&
4156 NeedWorkItemIDY && Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 1) != 0) {
4157 SDValue Y = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgY);
4158 Y = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Y,
4159 N2: DAG.getShiftAmountConstant(Val: 10, VT: MVT::i32, DL: SL));
4160 InputReg = InputReg.getNode()
4161 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: InputReg, N2: Y)
4162 : Y;
4163 }
4164
4165 if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo.WorkItemIDZ &&
4166 NeedWorkItemIDZ && Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 2) != 0) {
4167 SDValue Z = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgZ);
4168 Z = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Z,
4169 N2: DAG.getShiftAmountConstant(Val: 20, VT: MVT::i32, DL: SL));
4170 InputReg = InputReg.getNode()
4171 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: InputReg, N2: Z)
4172 : Z;
4173 }
4174
4175 if (!InputReg && (NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ)) {
4176 if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {
4177 // We're in a situation where the outgoing function requires the workitem
4178 // ID, but the calling function does not have it (e.g a graphics function
4179 // calling a C calling convention function). This is illegal, but we need
4180 // to produce something.
4181 InputReg = DAG.getPOISON(VT: MVT::i32);
4182 } else {
4183 // Workitem ids are already packed, any of present incoming arguments
4184 // will carry all required fields.
4185 ArgDescriptor IncomingArg =
4186 ArgDescriptor::createArg(Arg: IncomingArgX ? *IncomingArgX
4187 : IncomingArgY ? *IncomingArgY
4188 : *IncomingArgZ,
4189 Mask: ~0u);
4190 InputReg = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: IncomingArg);
4191 }
4192 }
4193
4194 if (OutgoingArg->isRegister()) {
4195 if (InputReg)
4196 RegsToPass.emplace_back(Args: OutgoingArg->getRegister(), Args&: InputReg);
4197
4198 CCInfo.AllocateReg(Reg: OutgoingArg->getRegister());
4199 } else {
4200 unsigned SpecialArgOffset = CCInfo.AllocateStack(Size: 4, Alignment: Align(4));
4201 if (InputReg) {
4202 SDValue ArgStore =
4203 storeStackInputValue(DAG, SL: DL, Chain, ArgVal: InputReg, Offset: SpecialArgOffset);
4204 MemOpChains.push_back(Elt: ArgStore);
4205 }
4206 }
4207}
4208
4209bool SITargetLowering::isEligibleForTailCallOptimization(
4210 SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
4211 const SmallVectorImpl<ISD::OutputArg> &Outs,
4212 const SmallVectorImpl<SDValue> &OutVals,
4213 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
4214 if (AMDGPU::isChainCC(CC: CalleeCC))
4215 return true;
4216
4217 if (!AMDGPU::mayTailCallThisCC(CC: CalleeCC))
4218 return false;
4219
4220 // For a divergent call target, we need to do a waterfall loop over the
4221 // possible callees which precludes us from using a simple jump.
4222 if (Callee->isDivergent())
4223 return false;
4224
4225 MachineFunction &MF = DAG.getMachineFunction();
4226 const Function &CallerF = MF.getFunction();
4227 CallingConv::ID CallerCC = CallerF.getCallingConv();
4228 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
4229 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
4230
4231 // Kernels aren't callable, and don't have a live in return address so it
4232 // doesn't make sense to do a tail call with entry functions.
4233 if (!CallerPreserved)
4234 return false;
4235
4236 bool CCMatch = CallerCC == CalleeCC;
4237
4238 if (DAG.getTarget().Options.GuaranteedTailCallOpt) {
4239 if (AMDGPU::canGuaranteeTCO(CC: CalleeCC) && CCMatch)
4240 return true;
4241 return false;
4242 }
4243
4244 // TODO: Can we handle var args?
4245 if (IsVarArg)
4246 return false;
4247
4248 for (const Argument &Arg : CallerF.args()) {
4249 if (Arg.hasByValAttr())
4250 return false;
4251 }
4252
4253 LLVMContext &Ctx = *DAG.getContext();
4254
4255 // Check that the call results are passed in the same way.
4256 if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C&: Ctx, Ins,
4257 CalleeFn: CCAssignFnForCall(CC: CalleeCC, IsVarArg),
4258 CallerFn: CCAssignFnForCall(CC: CallerCC, IsVarArg)))
4259 return false;
4260
4261 // The callee has to preserve all registers the caller needs to preserve.
4262 if (!CCMatch) {
4263 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
4264 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved))
4265 return false;
4266 }
4267
4268 // Nothing more to check if the callee is taking no arguments.
4269 if (Outs.empty())
4270 return true;
4271
4272 SmallVector<CCValAssign, 16> ArgLocs;
4273 CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx);
4274
4275 // FIXME: We are not allocating special input registers, so we will be
4276 // deciding based on incorrect register assignments.
4277 CCInfo.AnalyzeCallOperands(Outs, Fn: CCAssignFnForCall(CC: CalleeCC, IsVarArg));
4278
4279 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
4280 // If the stack arguments for this call do not fit into our own save area then
4281 // the call cannot be made tail.
4282 // TODO: Is this really necessary?
4283 if (CCInfo.getStackSize() > FuncInfo->getBytesInStackArgArea())
4284 return false;
4285
4286 for (const auto &[CCVA, ArgVal] : zip_equal(t&: ArgLocs, u: OutVals)) {
4287 // FIXME: What about inreg arguments that end up passed in memory?
4288 if (!CCVA.isRegLoc())
4289 continue;
4290
4291 // If we are passing an argument in an SGPR, and the value is divergent,
4292 // this call requires a waterfall loop.
4293 if (ArgVal->isDivergent() && TRI->isSGPRPhysReg(Reg: CCVA.getLocReg())) {
4294 LLVM_DEBUG(
4295 dbgs() << "Cannot tail call due to divergent outgoing argument in "
4296 << printReg(CCVA.getLocReg(), TRI) << '\n');
4297 return false;
4298 }
4299 }
4300
4301 const MachineRegisterInfo &MRI = MF.getRegInfo();
4302 return parametersInCSRMatch(MRI, CallerPreservedMask: CallerPreserved, ArgLocs, OutVals);
4303}
4304
4305bool SITargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
4306 if (!CI->isTailCall())
4307 return false;
4308
4309 const Function *ParentFn = CI->getFunction();
4310 if (AMDGPU::isEntryFunctionCC(CC: ParentFn->getCallingConv()))
4311 return false;
4312 return true;
4313}
4314
4315namespace {
4316// Chain calls have special arguments that we need to handle. These are
4317// tagging along at the end of the arguments list(s), after the SGPR and VGPR
4318// arguments (index 0 and 1 respectively).
4319enum ChainCallArgIdx {
4320 Exec = 2,
4321 Flags,
4322 NumVGPRs,
4323 FallbackExec,
4324 FallbackCallee
4325};
4326} // anonymous namespace
4327
4328// The wave scratch offset register is used as the global base pointer.
4329SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
4330 SmallVectorImpl<SDValue> &InVals) const {
4331 CallingConv::ID CallConv = CLI.CallConv;
4332 bool IsChainCallConv = AMDGPU::isChainCC(CC: CallConv);
4333
4334 SelectionDAG &DAG = CLI.DAG;
4335
4336 const SDLoc &DL = CLI.DL;
4337 SDValue Chain = CLI.Chain;
4338 SDValue Callee = CLI.Callee;
4339
4340 llvm::SmallVector<SDValue, 6> ChainCallSpecialArgs;
4341 bool UsesDynamicVGPRs = false;
4342 if (IsChainCallConv) {
4343 // The last arguments should be the value that we need to put in EXEC,
4344 // followed by the flags and any other arguments with special meanings.
4345 // Pop them out of CLI.Outs and CLI.OutVals before we do any processing so
4346 // we don't treat them like the "real" arguments.
4347 auto RequestedExecIt =
4348 llvm::find_if(Range&: CLI.Outs, P: [](const ISD::OutputArg &Arg) {
4349 return Arg.OrigArgIndex == 2;
4350 });
4351 assert(RequestedExecIt != CLI.Outs.end() && "No node for EXEC");
4352
4353 size_t SpecialArgsBeginIdx = RequestedExecIt - CLI.Outs.begin();
4354 CLI.OutVals.erase(CS: CLI.OutVals.begin() + SpecialArgsBeginIdx,
4355 CE: CLI.OutVals.end());
4356 CLI.Outs.erase(CS: RequestedExecIt, CE: CLI.Outs.end());
4357
4358 assert(CLI.Outs.back().OrigArgIndex < 2 &&
4359 "Haven't popped all the special args");
4360
4361 TargetLowering::ArgListEntry RequestedExecArg =
4362 CLI.Args[ChainCallArgIdx::Exec];
4363 if (!RequestedExecArg.Ty->isIntegerTy(BitWidth: Subtarget->getWavefrontSize()))
4364 return lowerUnhandledCall(CLI, InVals, Reason: "Invalid value for EXEC");
4365
4366 // Convert constants into TargetConstants, so they become immediate operands
4367 // instead of being selected into S_MOV.
4368 auto PushNodeOrTargetConstant = [&](TargetLowering::ArgListEntry Arg) {
4369 if (const auto *ArgNode = dyn_cast<ConstantSDNode>(Val&: Arg.Node)) {
4370 ChainCallSpecialArgs.push_back(Elt: DAG.getTargetConstant(
4371 Val: ArgNode->getAPIntValue(), DL, VT: ArgNode->getValueType(ResNo: 0)));
4372 } else
4373 ChainCallSpecialArgs.push_back(Elt: Arg.Node);
4374 };
4375
4376 PushNodeOrTargetConstant(RequestedExecArg);
4377
4378 // Process any other special arguments depending on the value of the flags.
4379 TargetLowering::ArgListEntry Flags = CLI.Args[ChainCallArgIdx::Flags];
4380
4381 const APInt &FlagsValue = cast<ConstantSDNode>(Val&: Flags.Node)->getAPIntValue();
4382 if (FlagsValue.isZero()) {
4383 if (CLI.Args.size() > ChainCallArgIdx::Flags + 1)
4384 return lowerUnhandledCall(CLI, InVals,
4385 Reason: "no additional args allowed if flags == 0");
4386 } else if (FlagsValue.isOneBitSet(BitNo: 0)) {
4387 if (CLI.Args.size() != ChainCallArgIdx::FallbackCallee + 1) {
4388 return lowerUnhandledCall(CLI, InVals, Reason: "expected 3 additional args");
4389 }
4390
4391 if (!Subtarget->isWave32()) {
4392 return lowerUnhandledCall(
4393 CLI, InVals, Reason: "dynamic VGPR mode is only supported for wave32");
4394 }
4395
4396 UsesDynamicVGPRs = true;
4397 std::for_each(first: CLI.Args.begin() + ChainCallArgIdx::NumVGPRs,
4398 last: CLI.Args.end(), f: PushNodeOrTargetConstant);
4399 }
4400 }
4401
4402 SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
4403 SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
4404 SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
4405 bool &IsTailCall = CLI.IsTailCall;
4406 bool IsVarArg = CLI.IsVarArg;
4407 bool IsSibCall = false;
4408 MachineFunction &MF = DAG.getMachineFunction();
4409
4410 if (Callee.isUndef() || isNullConstant(V: Callee)) {
4411 if (!CLI.IsTailCall) {
4412 for (ISD::InputArg &Arg : CLI.Ins)
4413 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
4414 }
4415
4416 return Chain;
4417 }
4418
4419 if (IsVarArg) {
4420 return lowerUnhandledCall(CLI, InVals,
4421 Reason: "unsupported call to variadic function ");
4422 }
4423
4424 if (!CLI.CB)
4425 return lowerUnhandledCall(CLI, InVals, Reason: "unsupported libcall legalization");
4426
4427 if (IsTailCall && MF.getTarget().Options.GuaranteedTailCallOpt) {
4428 return lowerUnhandledCall(CLI, InVals,
4429 Reason: "unsupported required tail call to function ");
4430 }
4431
4432 if (IsTailCall) {
4433 IsTailCall = isEligibleForTailCallOptimization(Callee, CalleeCC: CallConv, IsVarArg,
4434 Outs, OutVals, Ins, DAG);
4435 if (!IsTailCall &&
4436 ((CLI.CB && CLI.CB->isMustTailCall()) || IsChainCallConv)) {
4437 report_fatal_error(reason: "failed to perform tail call elimination on a call "
4438 "site marked musttail or on llvm.amdgcn.cs.chain");
4439 }
4440
4441 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
4442
4443 // A sibling call is one where we're under the usual C ABI and not planning
4444 // to change that but can still do a tail call:
4445 if (!TailCallOpt && IsTailCall)
4446 IsSibCall = true;
4447
4448 if (IsTailCall)
4449 ++NumTailCalls;
4450 }
4451
4452 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
4453 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
4454 SmallVector<SDValue, 8> MemOpChains;
4455
4456 // Analyze operands of the call, assigning locations to each operand.
4457 SmallVector<CCValAssign, 16> ArgLocs;
4458 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
4459 CCAssignFn *AssignFn = CCAssignFnForCall(CC: CallConv, IsVarArg);
4460
4461 if (CallConv != CallingConv::AMDGPU_Gfx && !AMDGPU::isChainCC(CC: CallConv) &&
4462 CallConv != CallingConv::AMDGPU_Gfx_WholeWave) {
4463 // With a fixed ABI, allocate fixed registers before user arguments.
4464 passSpecialInputs(CLI, CCInfo, Info: *Info, RegsToPass, MemOpChains, Chain);
4465 }
4466
4467 // Mark the scratch resource descriptor as allocated so the CC analysis
4468 // does not assign user arguments to these registers, matching the callee.
4469 if (!Subtarget->hasFlatScratchEnabled())
4470 CCInfo.AllocateReg(Reg: Info->getScratchRSrcReg());
4471
4472 CCInfo.AnalyzeCallOperands(Outs, Fn: AssignFn);
4473
4474 // Get a count of how many bytes are to be pushed on the stack.
4475 unsigned NumBytes = CCInfo.getStackSize();
4476
4477 if (IsSibCall) {
4478 // Since we're not changing the ABI to make this a tail call, the memory
4479 // operands are already available in the caller's incoming argument space.
4480 NumBytes = 0;
4481 }
4482
4483 // FPDiff is the byte offset of the call's argument area from the callee's.
4484 // Stores to callee stack arguments will be placed in FixedStackSlots offset
4485 // by this amount for a tail call. In a sibling call it must be 0 because the
4486 // caller will deallocate the entire stack and the callee still expects its
4487 // arguments to begin at SP+0. Completely unused for non-tail calls.
4488 int32_t FPDiff = 0;
4489 MachineFrameInfo &MFI = MF.getFrameInfo();
4490 auto *TRI = Subtarget->getRegisterInfo();
4491
4492 // Adjust the stack pointer for the new arguments...
4493 // These operations are automatically eliminated by the prolog/epilog pass
4494 if (!IsSibCall)
4495 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL);
4496
4497 if (!IsSibCall || IsChainCallConv) {
4498 if (!Subtarget->hasFlatScratchEnabled()) {
4499 SmallVector<SDValue, 4> CopyFromChains;
4500
4501 // In the HSA case, this should be an identity copy.
4502 SDValue ScratchRSrcReg =
4503 DAG.getCopyFromReg(Chain, dl: DL, Reg: Info->getScratchRSrcReg(), VT: MVT::v4i32);
4504 RegsToPass.emplace_back(Args: IsChainCallConv
4505 ? AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51
4506 : AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3,
4507 Args&: ScratchRSrcReg);
4508 CopyFromChains.push_back(Elt: ScratchRSrcReg.getValue(R: 1));
4509 Chain = DAG.getTokenFactor(DL, Vals&: CopyFromChains);
4510 }
4511 }
4512
4513 const unsigned NumSpecialInputs = RegsToPass.size();
4514
4515 MVT PtrVT = MVT::i32;
4516
4517 // Walk the register/memloc assignments, inserting copies/loads.
4518 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4519 CCValAssign &VA = ArgLocs[i];
4520 SDValue Arg = OutVals[i];
4521
4522 // Promote the value if needed.
4523 switch (VA.getLocInfo()) {
4524 case CCValAssign::Full:
4525 break;
4526 case CCValAssign::BCvt:
4527 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getLocVT(), Operand: Arg);
4528 break;
4529 case CCValAssign::ZExt:
4530 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4531 break;
4532 case CCValAssign::SExt:
4533 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4534 break;
4535 case CCValAssign::AExt:
4536 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4537 break;
4538 case CCValAssign::FPExt:
4539 Arg = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4540 break;
4541 default:
4542 llvm_unreachable("Unknown loc info!");
4543 }
4544
4545 if (VA.isRegLoc()) {
4546 RegsToPass.push_back(Elt: std::pair(VA.getLocReg(), Arg));
4547 } else {
4548 assert(VA.isMemLoc());
4549
4550 SDValue DstAddr;
4551 MachinePointerInfo DstInfo;
4552
4553 unsigned LocMemOffset = VA.getLocMemOffset();
4554 int32_t Offset = LocMemOffset;
4555
4556 SDValue PtrOff = DAG.getConstant(Val: Offset, DL, VT: PtrVT);
4557 MaybeAlign Alignment;
4558
4559 if (IsTailCall) {
4560 ISD::ArgFlagsTy Flags = Outs[i].Flags;
4561 unsigned OpSize = Flags.isByVal() ? Flags.getByValSize()
4562 : VA.getValVT().getStoreSize();
4563
4564 // FIXME: We can have better than the minimum byval required alignment.
4565 Alignment =
4566 Flags.isByVal()
4567 ? Flags.getNonZeroByValAlign()
4568 : commonAlignment(A: Subtarget->getStackAlignment(), Offset);
4569
4570 Offset = Offset + FPDiff;
4571 int FI = MFI.CreateFixedObject(Size: OpSize, SPOffset: Offset, IsImmutable: true);
4572
4573 DstAddr = DAG.getFrameIndex(FI, VT: PtrVT);
4574 DstInfo = MachinePointerInfo::getFixedStack(MF, FI);
4575
4576 // Make sure any stack arguments overlapping with where we're storing
4577 // are loaded before this eventual operation. Otherwise they'll be
4578 // clobbered.
4579
4580 // FIXME: Why is this really necessary? This seems to just result in a
4581 // lot of code to copy the stack and write them back to the same
4582 // locations, which are supposed to be immutable?
4583 Chain = addTokenForArgument(Chain, DAG, MFI, ClobberedFI: FI);
4584 } else {
4585 // Stores to the argument stack area are relative to the stack pointer.
4586 SDValue SP = DAG.getCopyFromReg(Chain, dl: DL, Reg: Info->getStackPtrOffsetReg(),
4587 VT: MVT::i32);
4588 DstAddr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SP, N2: PtrOff);
4589 DstInfo = MachinePointerInfo::getStack(MF, Offset: LocMemOffset);
4590 Alignment =
4591 commonAlignment(A: Subtarget->getStackAlignment(), Offset: LocMemOffset);
4592 }
4593
4594 if (Outs[i].Flags.isByVal()) {
4595 SDValue SizeNode =
4596 DAG.getConstant(Val: Outs[i].Flags.getByValSize(), DL, VT: MVT::i32);
4597 SDValue Cpy =
4598 DAG.getMemcpy(Chain, dl: DL, Dst: DstAddr, Src: Arg, Size: SizeNode,
4599 DstAlign: Outs[i].Flags.getNonZeroByValAlign(),
4600 SrcAlign: Outs[i].Flags.getNonZeroByValAlign(),
4601 /*isVol = */ false, /*AlwaysInline = */ true,
4602 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: DstInfo,
4603 SrcPtrInfo: MachinePointerInfo(AMDGPUAS::PRIVATE_ADDRESS));
4604
4605 MemOpChains.push_back(Elt: Cpy);
4606 } else {
4607 SDValue Store =
4608 DAG.getStore(Chain, dl: DL, Val: Arg, Ptr: DstAddr, PtrInfo: DstInfo, Alignment);
4609 MemOpChains.push_back(Elt: Store);
4610 }
4611 }
4612 }
4613
4614 if (!MemOpChains.empty())
4615 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
4616
4617 SDValue ReadFirstLaneID =
4618 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
4619
4620 SDValue TokenGlue;
4621 if (CLI.ConvergenceControlToken) {
4622 TokenGlue = DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL, VT: MVT::Glue,
4623 Operand: CLI.ConvergenceControlToken);
4624 }
4625
4626 // Build a sequence of copy-to-reg nodes chained together with token chain
4627 // and flag operands which copy the outgoing args into the appropriate regs.
4628 SDValue InGlue;
4629
4630 unsigned ArgIdx = 0;
4631 for (auto [Reg, Val] : RegsToPass) {
4632 if (ArgIdx++ >= NumSpecialInputs &&
4633 (IsChainCallConv || !Val->isDivergent()) && TRI->isSGPRPhysReg(Reg)) {
4634 // For chain calls, the inreg arguments are required to be
4635 // uniform. Speculatively Insert a readfirstlane in case we cannot prove
4636 // they are uniform.
4637 //
4638 // For other calls, if an inreg arguments is known to be uniform,
4639 // speculatively insert a readfirstlane in case it is in a VGPR.
4640 //
4641 // FIXME: We need to execute this in a waterfall loop if it is a divergent
4642 // value, so let that continue to produce invalid code.
4643
4644 SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Val});
4645 if (TokenGlue)
4646 ReadfirstlaneArgs.push_back(Elt: TokenGlue);
4647 Val = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Val.getValueType(),
4648 Ops: ReadfirstlaneArgs);
4649 }
4650
4651 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N: Val, Glue: InGlue);
4652 InGlue = Chain.getValue(R: 1);
4653 }
4654
4655 // We don't usually want to end the call-sequence here because we would tidy
4656 // the frame up *after* the call, however in the ABI-changing tail-call case
4657 // we've carefully laid out the parameters so that when sp is reset they'll be
4658 // in the correct location.
4659 if (IsTailCall && !IsSibCall) {
4660 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue: InGlue, DL);
4661 InGlue = Chain.getValue(R: 1);
4662 }
4663
4664 std::vector<SDValue> Ops({Chain});
4665
4666 // Add a redundant copy of the callee global which will not be legalized, as
4667 // we need direct access to the callee later.
4668 if (GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
4669 const GlobalValue *GV = GSD->getGlobal();
4670 Ops.push_back(x: Callee);
4671 Ops.push_back(x: DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i64));
4672 } else {
4673 if (IsTailCall) {
4674 // isEligibleForTailCallOptimization considered whether the call target is
4675 // divergent, but we may still end up with a uniform value in a VGPR.
4676 // Insert a readfirstlane just in case.
4677 SDValue ReadFirstLaneID =
4678 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
4679
4680 SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Callee});
4681 if (TokenGlue)
4682 ReadfirstlaneArgs.push_back(Elt: TokenGlue); // Wire up convergence token.
4683 Callee = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Callee.getValueType(),
4684 Ops: ReadfirstlaneArgs);
4685 }
4686
4687 Ops.push_back(x: Callee);
4688 Ops.push_back(x: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i64));
4689 }
4690
4691 if (IsTailCall) {
4692 // Each tail call may have to adjust the stack by a different amount, so
4693 // this information must travel along with the operation for eventual
4694 // consumption by emitEpilogue.
4695 Ops.push_back(x: DAG.getTargetConstant(Val: FPDiff, DL, VT: MVT::i32));
4696 }
4697
4698 if (IsChainCallConv)
4699 llvm::append_range(C&: Ops, R&: ChainCallSpecialArgs);
4700
4701 // Add argument registers to the end of the list so that they are known live
4702 // into the call.
4703 for (auto &[Reg, Val] : RegsToPass)
4704 Ops.push_back(x: DAG.getRegister(Reg, VT: Val.getValueType()));
4705
4706 // Add a register mask operand representing the call-preserved registers.
4707 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
4708 assert(Mask && "Missing call preserved mask for calling convention");
4709 Ops.push_back(x: DAG.getRegisterMask(RegMask: Mask));
4710
4711 if (SDValue Token = CLI.ConvergenceControlToken) {
4712 SmallVector<SDValue, 2> GlueOps;
4713 GlueOps.push_back(Elt: Token);
4714 if (InGlue)
4715 GlueOps.push_back(Elt: InGlue);
4716
4717 InGlue = SDValue(DAG.getMachineNode(Opcode: TargetOpcode::CONVERGENCECTRL_GLUE, dl: DL,
4718 VT: MVT::Glue, Ops: GlueOps),
4719 0);
4720 }
4721
4722 if (InGlue)
4723 Ops.push_back(x: InGlue);
4724
4725 // If we're doing a tall call, use a TC_RETURN here rather than an
4726 // actual call instruction.
4727 if (IsTailCall) {
4728 MFI.setHasTailCall();
4729 unsigned OPC = AMDGPUISD::TC_RETURN;
4730 switch (CallConv) {
4731 case CallingConv::AMDGPU_Gfx:
4732 OPC = AMDGPUISD::TC_RETURN_GFX;
4733 break;
4734 case CallingConv::AMDGPU_CS_Chain:
4735 case CallingConv::AMDGPU_CS_ChainPreserve:
4736 OPC = UsesDynamicVGPRs ? AMDGPUISD::TC_RETURN_CHAIN_DVGPR
4737 : AMDGPUISD::TC_RETURN_CHAIN;
4738 break;
4739 }
4740
4741 // If the caller is a whole wave function, we need to use a special opcode
4742 // so we can patch up EXEC.
4743 if (Info->isWholeWaveFunction())
4744 OPC = AMDGPUISD::TC_RETURN_GFX_WholeWave;
4745
4746 return DAG.getNode(Opcode: OPC, DL, VT: MVT::Other, Ops);
4747 }
4748
4749 // Returns a chain and a flag for retval copy to use.
4750 SDValue Call = DAG.getNode(Opcode: AMDGPUISD::CALL, DL, ResultTys: {MVT::Other, MVT::Glue}, Ops);
4751 Chain = Call.getValue(R: 0);
4752 InGlue = Call.getValue(R: 1);
4753
4754 uint64_t CalleePopBytes = NumBytes;
4755 Chain = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: CalleePopBytes, Glue: InGlue, DL);
4756 if (!Ins.empty())
4757 InGlue = Chain.getValue(R: 1);
4758
4759 // Handle result values, copying them out of physregs into vregs that we
4760 // return.
4761 return LowerCallResult(Chain, InGlue, CallConv, IsVarArg, Ins, DL, DAG,
4762 InVals, /*IsThisReturn=*/false, ThisVal: SDValue());
4763}
4764
4765// This is similar to the default implementation in ExpandDYNAMIC_STACKALLOC,
4766// except for:
4767// 1. Stack growth direction(default: downwards, AMDGPU: upwards), and
4768// 2. Scale size where, scale = wave-reduction(alloca-size) * wave-size
4769SDValue SITargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
4770 SelectionDAG &DAG) const {
4771 const MachineFunction &MF = DAG.getMachineFunction();
4772 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
4773
4774 SDLoc dl(Op);
4775 EVT VT = Op.getValueType();
4776 SDValue Chain = Op.getOperand(i: 0);
4777 Register SPReg = Info->getStackPtrOffsetReg();
4778
4779 // Chain the dynamic stack allocation so that it doesn't modify the stack
4780 // pointer when other instructions are using the stack.
4781 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
4782
4783 SDValue Size = Op.getOperand(i: 1);
4784 SDValue BaseAddr = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
4785 Align Alignment = cast<ConstantSDNode>(Val: Op.getOperand(i: 2))->getAlignValue();
4786
4787 const TargetFrameLowering *TFL = Subtarget->getFrameLowering();
4788 assert(TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp &&
4789 "Stack grows upwards for AMDGPU");
4790
4791 Chain = BaseAddr.getValue(R: 1);
4792 // When using flat-scratch, the stack offset is unscaled.
4793 const bool HasFlatScratch = Subtarget->hasFlatScratchEnabled();
4794 const unsigned WavefrontSizeLog2 = Subtarget->getWavefrontSizeLog2();
4795
4796 Align StackAlign = TFL->getStackAlign();
4797 if (Alignment > StackAlign) {
4798 uint64_t ScaledAlignment = Alignment.value()
4799 << (HasFlatScratch ? 0 : WavefrontSizeLog2);
4800 uint64_t StackAlignMask = ScaledAlignment - 1;
4801 SDValue TmpAddr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr,
4802 N2: DAG.getConstant(Val: StackAlignMask, DL: dl, VT));
4803 BaseAddr = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: TmpAddr,
4804 N2: DAG.getSignedConstant(Val: -ScaledAlignment, DL: dl, VT));
4805 }
4806
4807 assert(Size.getValueType() == MVT::i32 && "Size must be 32-bit");
4808 SDValue NewSP;
4809 if (isa<ConstantSDNode>(Val: Size)) {
4810 // Increase the stack pointer by the size of the alloca.
4811 // If not using flat-scratch, we have to scale the size by the wave-size.
4812 SDValue ScaledSize =
4813 HasFlatScratch
4814 ? Size
4815 : DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Size,
4816 N2: DAG.getConstant(Val: WavefrontSizeLog2, DL: dl, VT: MVT::i32));
4817 NewSP = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr, N2: ScaledSize); // Value
4818 } else {
4819 // For dynamic sized alloca, perform wave-wide reduction to get max of
4820 // alloca size(divergent), and then scale it (when not using flat-scratch)
4821 // by wave-size.
4822 SDValue WaveReduction =
4823 DAG.getTargetConstant(Val: Intrinsic::amdgcn_wave_reduce_umax, DL: dl, VT: MVT::i32);
4824 Size = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::i32, N1: WaveReduction,
4825 N2: Size, N3: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32));
4826 SDValue ScaledSize = Size;
4827 if (!HasFlatScratch) {
4828 ScaledSize =
4829 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Size,
4830 N2: DAG.getConstant(Val: WavefrontSizeLog2, DL: dl, VT: MVT::i32));
4831 }
4832 NewSP =
4833 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr, N2: ScaledSize); // Value in vgpr.
4834 SDValue ReadFirstLaneID =
4835 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: dl, VT: MVT::i32);
4836 NewSP = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::i32, N1: ReadFirstLaneID,
4837 N2: NewSP);
4838 }
4839
4840 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: NewSP); // Output chain
4841 SDValue CallSeqEnd = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
4842
4843 return DAG.getMergeValues(Ops: {BaseAddr, CallSeqEnd}, dl);
4844}
4845
4846SDValue SITargetLowering::LowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const {
4847 if (Op.getValueType() != MVT::i32)
4848 return Op; // Defer to cannot select error.
4849
4850 Register SP = getStackPointerRegisterToSaveRestore();
4851 SDLoc SL(Op);
4852
4853 SDValue CopyFromSP = DAG.getCopyFromReg(Chain: Op->getOperand(Num: 0), dl: SL, Reg: SP, VT: MVT::i32);
4854
4855 // Convert from wave uniform to swizzled vector address. This should protect
4856 // from any edge cases where the stacksave result isn't directly used with
4857 // stackrestore.
4858 SDValue VectorAddress =
4859 DAG.getNode(Opcode: AMDGPUISD::WAVE_ADDRESS, DL: SL, VT: MVT::i32, Operand: CopyFromSP);
4860 return DAG.getMergeValues(Ops: {VectorAddress, CopyFromSP.getValue(R: 1)}, dl: SL);
4861}
4862
4863SDValue SITargetLowering::lowerGET_ROUNDING(SDValue Op,
4864 SelectionDAG &DAG) const {
4865 SDLoc SL(Op);
4866 assert(Op.getValueType() == MVT::i32);
4867
4868 uint32_t BothRoundHwReg =
4869 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
4870 SDValue GetRoundBothImm = DAG.getTargetConstant(Val: BothRoundHwReg, DL: SL, VT: MVT::i32);
4871
4872 SDValue IntrinID =
4873 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_getreg, DL: SL, VT: MVT::i32);
4874 SDValue GetReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList: Op->getVTList(),
4875 N1: Op.getOperand(i: 0), N2: IntrinID, N3: GetRoundBothImm);
4876
4877 // There are two rounding modes, one for f32 and one for f64/f16. We only
4878 // report in the standard value range if both are the same.
4879 //
4880 // The raw values also differ from the expected FLT_ROUNDS values. Nearest
4881 // ties away from zero is not supported, and the other values are rotated by
4882 // 1.
4883 //
4884 // If the two rounding modes are not the same, report a target defined value.
4885
4886 // Mode register rounding mode fields:
4887 //
4888 // [1:0] Single-precision round mode.
4889 // [3:2] Double/Half-precision round mode.
4890 //
4891 // 0=nearest even; 1= +infinity; 2= -infinity, 3= toward zero.
4892 //
4893 // Hardware Spec
4894 // Toward-0 3 0
4895 // Nearest Even 0 1
4896 // +Inf 1 2
4897 // -Inf 2 3
4898 // NearestAway0 N/A 4
4899 //
4900 // We have to handle 16 permutations of a 4-bit value, so we create a 64-bit
4901 // table we can index by the raw hardware mode.
4902 //
4903 // (trunc (FltRoundConversionTable >> MODE.fp_round)) & 0xf
4904
4905 SDValue BitTable =
4906 DAG.getConstant(Val: AMDGPU::FltRoundConversionTable, DL: SL, VT: MVT::i64);
4907
4908 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4909 SDValue RoundModeTimesNumBits =
4910 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: GetReg, N2: Two);
4911
4912 // TODO: We could possibly avoid a 64-bit shift and use a simpler table if we
4913 // knew only one mode was demanded.
4914 SDValue TableValue =
4915 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i64, N1: BitTable, N2: RoundModeTimesNumBits);
4916 SDValue TruncTable = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: TableValue);
4917
4918 SDValue EntryMask = DAG.getConstant(Val: 0xf, DL: SL, VT: MVT::i32);
4919 SDValue TableEntry =
4920 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: TruncTable, N2: EntryMask);
4921
4922 // There's a gap in the 4-bit encoded table and actual enum values, so offset
4923 // if it's an extended value.
4924 SDValue Four = DAG.getConstant(Val: 4, DL: SL, VT: MVT::i32);
4925 SDValue IsStandardValue =
4926 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: TableEntry, RHS: Four, Cond: ISD::SETULT);
4927 SDValue EnumOffset = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: TableEntry, N2: Four);
4928 SDValue Result = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i32, N1: IsStandardValue,
4929 N2: TableEntry, N3: EnumOffset);
4930
4931 return DAG.getMergeValues(Ops: {Result, GetReg.getValue(R: 1)}, dl: SL);
4932}
4933
4934SDValue SITargetLowering::lowerSET_ROUNDING(SDValue Op,
4935 SelectionDAG &DAG) const {
4936 SDLoc SL(Op);
4937
4938 SDValue NewMode = Op.getOperand(i: 1);
4939 assert(NewMode.getValueType() == MVT::i32);
4940
4941 // Index a table of 4-bit entries mapping from the C FLT_ROUNDS values to the
4942 // hardware MODE.fp_round values.
4943 if (auto *ConstMode = dyn_cast<ConstantSDNode>(Val&: NewMode)) {
4944 uint32_t ClampedVal = std::min(
4945 a: static_cast<uint32_t>(ConstMode->getZExtValue()),
4946 b: static_cast<uint32_t>(AMDGPU::TowardZeroF32_TowardNegativeF64));
4947 NewMode = DAG.getConstant(
4948 Val: AMDGPU::decodeFltRoundToHWConversionTable(FltRounds: ClampedVal), DL: SL, VT: MVT::i32);
4949 } else {
4950 // If we know the input can only be one of the supported standard modes in
4951 // the range 0-3, we can use a simplified mapping to hardware values.
4952 KnownBits KB = DAG.computeKnownBits(Op: NewMode);
4953 const bool UseReducedTable = KB.countMinLeadingZeros() >= 30;
4954 // The supported standard values are 0-3. The extended values start at 8. We
4955 // need to offset by 4 if the value is in the extended range.
4956
4957 if (UseReducedTable) {
4958 // Truncate to the low 32-bits.
4959 SDValue BitTable = DAG.getConstant(
4960 Val: AMDGPU::FltRoundToHWConversionTable & 0xffff, DL: SL, VT: MVT::i32);
4961
4962 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4963 SDValue RoundModeTimesNumBits =
4964 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: NewMode, N2: Two);
4965
4966 NewMode =
4967 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: BitTable, N2: RoundModeTimesNumBits);
4968
4969 // TODO: SimplifyDemandedBits on the setreg source here can likely reduce
4970 // the table extracted bits into inline immediates.
4971 } else {
4972 // table_index = umin(value, value - 4)
4973 // MODE.fp_round = (bit_table >> (table_index << 2)) & 0xf
4974 SDValue BitTable =
4975 DAG.getConstant(Val: AMDGPU::FltRoundToHWConversionTable, DL: SL, VT: MVT::i64);
4976
4977 SDValue Four = DAG.getConstant(Val: 4, DL: SL, VT: MVT::i32);
4978 SDValue OffsetEnum = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: NewMode, N2: Four);
4979 SDValue IndexVal =
4980 DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewMode, N2: OffsetEnum);
4981
4982 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4983 SDValue RoundModeTimesNumBits =
4984 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: IndexVal, N2: Two);
4985
4986 SDValue TableValue =
4987 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i64, N1: BitTable, N2: RoundModeTimesNumBits);
4988 SDValue TruncTable = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: TableValue);
4989
4990 // No need to mask out the high bits since the setreg will ignore them
4991 // anyway.
4992 NewMode = TruncTable;
4993 }
4994
4995 // Insert a readfirstlane in case the value is a VGPR. We could do this
4996 // earlier and keep more operations scalar, but that interferes with
4997 // combining the source.
4998 SDValue ReadFirstLaneID =
4999 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: SL, VT: MVT::i32);
5000 NewMode = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
5001 N1: ReadFirstLaneID, N2: NewMode);
5002 }
5003
5004 // N.B. The setreg will be later folded into s_round_mode on supported
5005 // targets.
5006 SDValue IntrinID =
5007 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_setreg, DL: SL, VT: MVT::i32);
5008 uint32_t BothRoundHwReg =
5009 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
5010 SDValue RoundBothImm = DAG.getTargetConstant(Val: BothRoundHwReg, DL: SL, VT: MVT::i32);
5011
5012 SDValue SetReg =
5013 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VTList: Op->getVTList(), N1: Op.getOperand(i: 0),
5014 N2: IntrinID, N3: RoundBothImm, N4: NewMode);
5015
5016 return SetReg;
5017}
5018
5019SDValue SITargetLowering::lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const {
5020 if (Op->isDivergent() &&
5021 (!Subtarget->hasVmemPrefInsts() || !Op.getConstantOperandVal(i: 4)))
5022 // Cannot do I$ prefetch with divergent pointer.
5023 return SDValue();
5024
5025 switch (cast<MemSDNode>(Val&: Op)->getAddressSpace()) {
5026 case AMDGPUAS::FLAT_ADDRESS:
5027 case AMDGPUAS::GLOBAL_ADDRESS:
5028 case AMDGPUAS::CONSTANT_ADDRESS:
5029 break;
5030 case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
5031 if (Subtarget->hasSafeSmemPrefetch())
5032 break;
5033 [[fallthrough]];
5034 default:
5035 return SDValue();
5036 }
5037
5038 // I$ prefetch
5039 if (!Subtarget->hasSafeSmemPrefetch() && !Op.getConstantOperandVal(i: 4))
5040 return SDValue();
5041
5042 return Op;
5043}
5044
5045// Work around DAG legality rules only based on the result type.
5046SDValue SITargetLowering::lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
5047 bool IsStrict = Op.getOpcode() == ISD::STRICT_FP_EXTEND;
5048 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
5049 EVT SrcVT = Src.getValueType();
5050
5051 if (SrcVT.getScalarType() != MVT::bf16)
5052 return Op;
5053
5054 SDLoc SL(Op);
5055 SDValue BitCast =
5056 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: SrcVT.changeTypeToInteger(), Operand: Src);
5057
5058 EVT DstVT = Op.getValueType();
5059 if (IsStrict)
5060 llvm_unreachable("Need STRICT_BF16_TO_FP");
5061
5062 return DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SL, VT: DstVT, Operand: BitCast);
5063}
5064
5065SDValue SITargetLowering::lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const {
5066 SDLoc SL(Op);
5067 if (Op.getValueType() != MVT::i64)
5068 return Op;
5069
5070 uint32_t ModeHwReg =
5071 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 23);
5072 SDValue ModeHwRegImm = DAG.getTargetConstant(Val: ModeHwReg, DL: SL, VT: MVT::i32);
5073 uint32_t TrapHwReg =
5074 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: 0, Values: 5);
5075 SDValue TrapHwRegImm = DAG.getTargetConstant(Val: TrapHwReg, DL: SL, VT: MVT::i32);
5076
5077 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
5078 SDValue IntrinID =
5079 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_getreg, DL: SL, VT: MVT::i32);
5080 SDValue GetModeReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList,
5081 N1: Op.getOperand(i: 0), N2: IntrinID, N3: ModeHwRegImm);
5082 SDValue GetTrapReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList,
5083 N1: Op.getOperand(i: 0), N2: IntrinID, N3: TrapHwRegImm);
5084 SDValue TokenReg =
5085 DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: GetModeReg.getValue(R: 1),
5086 N2: GetTrapReg.getValue(R: 1));
5087
5088 SDValue CvtPtr =
5089 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: GetModeReg, N2: GetTrapReg);
5090 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
5091
5092 return DAG.getMergeValues(Ops: {Result, TokenReg}, dl: SL);
5093}
5094
5095SDValue SITargetLowering::lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const {
5096 SDLoc SL(Op);
5097 if (Op.getOperand(i: 1).getValueType() != MVT::i64)
5098 return Op;
5099
5100 SDValue Input = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
5101 SDValue NewModeReg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Input,
5102 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
5103 SDValue NewTrapReg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Input,
5104 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
5105
5106 SDValue ReadFirstLaneID =
5107 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: SL, VT: MVT::i32);
5108 NewModeReg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
5109 N1: ReadFirstLaneID, N2: NewModeReg);
5110 NewTrapReg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
5111 N1: ReadFirstLaneID, N2: NewTrapReg);
5112
5113 unsigned ModeHwReg =
5114 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 23);
5115 SDValue ModeHwRegImm = DAG.getTargetConstant(Val: ModeHwReg, DL: SL, VT: MVT::i32);
5116 unsigned TrapHwReg =
5117 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: 0, Values: 5);
5118 SDValue TrapHwRegImm = DAG.getTargetConstant(Val: TrapHwReg, DL: SL, VT: MVT::i32);
5119
5120 SDValue IntrinID =
5121 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_setreg, DL: SL, VT: MVT::i32);
5122 SDValue SetModeReg =
5123 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VT: MVT::Other, N1: Op.getOperand(i: 0),
5124 N2: IntrinID, N3: ModeHwRegImm, N4: NewModeReg);
5125 SDValue SetTrapReg =
5126 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VT: MVT::Other, N1: Op.getOperand(i: 0),
5127 N2: IntrinID, N3: TrapHwRegImm, N4: NewTrapReg);
5128 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: SetTrapReg, N2: SetModeReg);
5129}
5130
5131Register SITargetLowering::getRegisterByName(const char *RegName, LLT VT,
5132 const MachineFunction &MF) const {
5133 const Function &Fn = MF.getFunction();
5134
5135 Register Reg =
5136 StringSwitch<Register>(RegName)
5137 .Case(S: "m0", Value: AMDGPU::M0)
5138 .Case(S: "exec", Value: AMDGPU::EXEC)
5139 .Case(S: "exec_lo", Value: AMDGPU::EXEC_LO)
5140 .Case(S: "exec_hi", Value: AMDGPU::EXEC_HI)
5141 .Case(S: "flat_scratch", Value: AMDGPU::FLAT_SCR)
5142 .Case(S: "flat_scratch_lo", Value: AMDGPU::FLAT_SCR_LO)
5143 .Case(S: "flat_scratch_hi", Value: AMDGPU::FLAT_SCR_HI)
5144 .Case(S: "src_flat_scratch_base", Value: AMDGPU::SRC_FLAT_SCRATCH_BASE)
5145 .Case(S: "src_flat_scratch_base_lo", Value: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
5146 .Case(S: "src_flat_scratch_base_hi", Value: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
5147 .Default(Value: Register());
5148 if (!Reg)
5149 return Reg;
5150
5151 if (!Subtarget->hasFlatScrRegister() &&
5152 Subtarget->getRegisterInfo()->regsOverlap(RegA: Reg, RegB: AMDGPU::FLAT_SCR)) {
5153 Fn.getContext().emitError(ErrorStr: Twine("invalid register \"" + StringRef(RegName) +
5154 "\" for subtarget."));
5155 }
5156
5157 if (!Subtarget->hasGloballyAddressableScratch() &&
5158 Subtarget->getRegisterInfo()->regsOverlap(
5159 RegA: Reg, RegB: AMDGPU::SRC_FLAT_SCRATCH_BASE)) {
5160 Fn.getContext().emitError(ErrorStr: Twine("invalid register \"" + StringRef(RegName) +
5161 "\" for subtarget."));
5162 }
5163
5164 switch (Reg) {
5165 case AMDGPU::M0:
5166 case AMDGPU::EXEC_LO:
5167 case AMDGPU::EXEC_HI:
5168 case AMDGPU::FLAT_SCR_LO:
5169 case AMDGPU::FLAT_SCR_HI:
5170 case AMDGPU::SRC_FLAT_SCRATCH_BASE_LO:
5171 case AMDGPU::SRC_FLAT_SCRATCH_BASE_HI:
5172 if (VT.getSizeInBits() == 32)
5173 return Reg;
5174 break;
5175 case AMDGPU::EXEC:
5176 case AMDGPU::FLAT_SCR:
5177 case AMDGPU::SRC_FLAT_SCRATCH_BASE:
5178 if (VT.getSizeInBits() == 64)
5179 return Reg;
5180 break;
5181 default:
5182 llvm_unreachable("missing register type checking");
5183 }
5184
5185 report_fatal_error(
5186 reason: Twine("invalid type for register \"" + StringRef(RegName) + "\"."));
5187}
5188
5189// If kill is not the last instruction, split the block so kill is always a
5190// proper terminator.
5191MachineBasicBlock *
5192SITargetLowering::splitKillBlock(MachineInstr &MI,
5193 MachineBasicBlock *BB) const {
5194 MachineBasicBlock *SplitBB = BB->splitAt(SplitInst&: MI, /*UpdateLiveIns=*/true);
5195 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5196 MI.setDesc(TII->getKillTerminatorFromPseudo(Opcode: MI.getOpcode()));
5197 return SplitBB;
5198}
5199
5200// Split block \p MBB at \p MI, as to insert a loop. If \p InstInLoop is true,
5201// \p MI will be the only instruction in the loop body block. Otherwise, it will
5202// be the first instruction in the remainder block.
5203//
5204/// \returns { LoopBody, Remainder }
5205static std::pair<MachineBasicBlock *, MachineBasicBlock *>
5206splitBlockForLoop(MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop) {
5207 MachineFunction *MF = MBB.getParent();
5208 MachineBasicBlock::iterator I(&MI);
5209
5210 // To insert the loop we need to split the block. Move everything after this
5211 // point to a new block, and insert a new empty block between the two.
5212 MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
5213 MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
5214 MachineFunction::iterator MBBI(MBB);
5215 ++MBBI;
5216
5217 MF->insert(MBBI, MBB: LoopBB);
5218 MF->insert(MBBI, MBB: RemainderBB);
5219
5220 LoopBB->addSuccessor(Succ: LoopBB);
5221 LoopBB->addSuccessor(Succ: RemainderBB);
5222
5223 // Move the rest of the block into a new block.
5224 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
5225
5226 if (InstInLoop) {
5227 auto Next = std::next(x: I);
5228
5229 // Move instruction to loop body.
5230 LoopBB->splice(Where: LoopBB->begin(), Other: &MBB, From: I, To: Next);
5231
5232 // Move the rest of the block.
5233 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: Next, To: MBB.end());
5234 } else {
5235 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: I, To: MBB.end());
5236 }
5237
5238 MBB.addSuccessor(Succ: LoopBB);
5239
5240 return std::pair(LoopBB, RemainderBB);
5241}
5242
5243/// Insert \p MI into a BUNDLE with an S_WAITCNT 0 immediately following it.
5244void SITargetLowering::bundleInstWithWaitcnt(MachineInstr &MI) const {
5245 MachineBasicBlock *MBB = MI.getParent();
5246 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5247 auto I = MI.getIterator();
5248 auto E = std::next(x: I);
5249
5250 // clang-format off
5251 BuildMI(BB&: *MBB, I: E, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_WAITCNT))
5252 .addImm(Val: 0);
5253 // clang-format on
5254
5255 MIBundleBuilder Bundler(*MBB, I, E);
5256 finalizeBundle(MBB&: *MBB, FirstMI: Bundler.begin());
5257}
5258
5259MachineBasicBlock *
5260SITargetLowering::emitGWSMemViolTestLoop(MachineInstr &MI,
5261 MachineBasicBlock *BB) const {
5262 const DebugLoc &DL = MI.getDebugLoc();
5263
5264 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
5265
5266 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5267
5268 // Apparently kill flags are only valid if the def is in the same block?
5269 if (MachineOperand *Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::data0))
5270 Src->setIsKill(false);
5271
5272 auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, MBB&: *BB, InstInLoop: true);
5273
5274 MachineBasicBlock::iterator I = LoopBB->end();
5275
5276 const unsigned EncodedReg = AMDGPU::Hwreg::HwregEncoding::encode(
5277 Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: AMDGPU::Hwreg::OFFSET_MEM_VIOL, Values: 1);
5278
5279 // Clear TRAP_STS.MEM_VIOL
5280 BuildMI(BB&: *LoopBB, I: LoopBB->begin(), MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SETREG_IMM32_B32))
5281 .addImm(Val: 0)
5282 .addImm(Val: EncodedReg);
5283
5284 bundleInstWithWaitcnt(MI);
5285
5286 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5287
5288 // Load and check TRAP_STS.MEM_VIOL
5289 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: Reg)
5290 .addImm(Val: EncodedReg);
5291
5292 // FIXME: Do we need to use an isel pseudo that may clobber scc?
5293 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
5294 .addReg(RegNo: Reg, Flags: RegState::Kill)
5295 .addImm(Val: 0);
5296 // clang-format off
5297 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
5298 .addMBB(MBB: LoopBB);
5299 // clang-format on
5300
5301 return RemainderBB;
5302}
5303
5304// Do a v_movrels_b32 or v_movreld_b32 for each unique value of \p IdxReg in the
5305// wavefront. If the value is uniform and just happens to be in a VGPR, this
5306// will only do one iteration. In the worst case, this will loop 64 times.
5307//
5308// TODO: Just use v_readlane_b32 if we know the VGPR has a uniform value.
5309static MachineBasicBlock::iterator
5310emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI,
5311 MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
5312 const DebugLoc &DL, const MachineOperand &Idx,
5313 unsigned InitReg, unsigned ResultReg, unsigned PhiReg,
5314 unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode,
5315 Register &SGPRIdxReg) {
5316
5317 MachineFunction *MF = OrigBB.getParent();
5318 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5319 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5320 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
5321 MachineBasicBlock::iterator I = LoopBB.begin();
5322
5323 const TargetRegisterClass *BoolRC = TRI->getBoolRC();
5324 Register PhiExec = MRI.createVirtualRegister(RegClass: BoolRC);
5325 Register NewExec = MRI.createVirtualRegister(RegClass: BoolRC);
5326 Register CurrentIdxReg =
5327 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5328 Register CondReg = MRI.createVirtualRegister(RegClass: BoolRC);
5329
5330 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::PHI), DestReg: PhiReg)
5331 .addReg(RegNo: InitReg)
5332 .addMBB(MBB: &OrigBB)
5333 .addReg(RegNo: ResultReg)
5334 .addMBB(MBB: &LoopBB);
5335
5336 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::PHI), DestReg: PhiExec)
5337 .addReg(RegNo: InitSaveExecReg)
5338 .addMBB(MBB: &OrigBB)
5339 .addReg(RegNo: NewExec)
5340 .addMBB(MBB: &LoopBB);
5341
5342 // Read the next variant <- also loop target.
5343 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurrentIdxReg)
5344 .addReg(RegNo: Idx.getReg(), Flags: getUndefRegState(B: Idx.isUndef()));
5345
5346 // Compare the just read M0 value to all possible Idx values.
5347 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: CondReg)
5348 .addReg(RegNo: CurrentIdxReg)
5349 .addReg(RegNo: Idx.getReg(), Flags: {}, SubReg: Idx.getSubReg());
5350
5351 // Update EXEC, save the original EXEC value to VCC.
5352 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.AndSaveExecOpc), DestReg: NewExec)
5353 .addReg(RegNo: CondReg, Flags: RegState::Kill);
5354
5355 MRI.setSimpleHint(VReg: NewExec, PrefReg: CondReg);
5356
5357 if (UseGPRIdxMode) {
5358 if (Offset == 0) {
5359 SGPRIdxReg = CurrentIdxReg;
5360 } else {
5361 SGPRIdxReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
5362 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: SGPRIdxReg)
5363 .addReg(RegNo: CurrentIdxReg, Flags: RegState::Kill)
5364 .addImm(Val: Offset);
5365 }
5366 } else {
5367 // Move index from VCC into M0
5368 if (Offset == 0) {
5369 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
5370 .addReg(RegNo: CurrentIdxReg, Flags: RegState::Kill);
5371 } else {
5372 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: AMDGPU::M0)
5373 .addReg(RegNo: CurrentIdxReg, Flags: RegState::Kill)
5374 .addImm(Val: Offset);
5375 }
5376 }
5377
5378 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
5379 MachineInstr *InsertPt =
5380 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.XorTermOpc), DestReg: LMC.ExecReg)
5381 .addReg(RegNo: LMC.ExecReg)
5382 .addReg(RegNo: NewExec);
5383
5384 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
5385 // s_cbranch_scc0?
5386
5387 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
5388 // clang-format off
5389 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_EXECNZ))
5390 .addMBB(MBB: &LoopBB);
5391 // clang-format on
5392
5393 return InsertPt->getIterator();
5394}
5395
5396// This has slightly sub-optimal regalloc when the source vector is killed by
5397// the read. The register allocator does not understand that the kill is
5398// per-workitem, so is kept alive for the whole loop so we end up not re-using a
5399// subregister from it, using 1 more VGPR than necessary. This was saved when
5400// this was expanded after register allocation.
5401static MachineBasicBlock::iterator
5402loadM0FromVGPR(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI,
5403 unsigned InitResultReg, unsigned PhiReg, int Offset,
5404 bool UseGPRIdxMode, Register &SGPRIdxReg) {
5405 MachineFunction *MF = MBB.getParent();
5406 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5407 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5408 MachineRegisterInfo &MRI = MF->getRegInfo();
5409 const DebugLoc &DL = MI.getDebugLoc();
5410 MachineBasicBlock::iterator I(&MI);
5411
5412 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
5413 Register DstReg = MI.getOperand(i: 0).getReg();
5414 Register SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
5415 Register TmpExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
5416 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
5417
5418 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: TmpExec);
5419
5420 // Save the EXEC mask
5421 // clang-format off
5422 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: SaveExec)
5423 .addReg(RegNo: LMC.ExecReg);
5424 // clang-format on
5425
5426 auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, MBB, InstInLoop: false);
5427
5428 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5429
5430 auto InsPt = emitLoadM0FromVGPRLoop(TII, MRI, OrigBB&: MBB, LoopBB&: *LoopBB, DL, Idx: *Idx,
5431 InitReg: InitResultReg, ResultReg: DstReg, PhiReg, InitSaveExecReg: TmpExec,
5432 Offset, UseGPRIdxMode, SGPRIdxReg);
5433
5434 MachineBasicBlock *LandingPad = MF->CreateMachineBasicBlock();
5435 MachineFunction::iterator MBBI(LoopBB);
5436 ++MBBI;
5437 MF->insert(MBBI, MBB: LandingPad);
5438 LoopBB->removeSuccessor(Succ: RemainderBB);
5439 LandingPad->addSuccessor(Succ: RemainderBB);
5440 LoopBB->addSuccessor(Succ: LandingPad);
5441 MachineBasicBlock::iterator First = LandingPad->begin();
5442 // clang-format off
5443 BuildMI(BB&: *LandingPad, I: First, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
5444 .addReg(RegNo: SaveExec);
5445 // clang-format on
5446
5447 return InsPt;
5448}
5449
5450// Returns subreg index, offset
5451static std::pair<unsigned, int>
5452computeIndirectRegAndOffset(const SIRegisterInfo &TRI,
5453 const TargetRegisterClass *SuperRC, unsigned VecReg,
5454 int Offset) {
5455 int NumElts = TRI.getRegSizeInBits(RC: *SuperRC) / 32;
5456
5457 // Skip out of bounds offsets, or else we would end up using an undefined
5458 // register.
5459 if (Offset >= NumElts || Offset < 0)
5460 return std::pair(AMDGPU::sub0, Offset);
5461
5462 return std::pair(SIRegisterInfo::getSubRegFromChannel(Channel: Offset), 0);
5463}
5464
5465static void setM0ToIndexFromSGPR(const SIInstrInfo *TII,
5466 MachineRegisterInfo &MRI, MachineInstr &MI,
5467 int Offset) {
5468 MachineBasicBlock *MBB = MI.getParent();
5469 const DebugLoc &DL = MI.getDebugLoc();
5470 MachineBasicBlock::iterator I(&MI);
5471
5472 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5473
5474 assert(Idx->getReg() != AMDGPU::NoRegister);
5475
5476 if (Offset == 0) {
5477 // clang-format off
5478 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
5479 .add(MO: *Idx);
5480 // clang-format on
5481 } else {
5482 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: AMDGPU::M0)
5483 .add(MO: *Idx)
5484 .addImm(Val: Offset);
5485 }
5486}
5487
5488static Register getIndirectSGPRIdx(const SIInstrInfo *TII,
5489 MachineRegisterInfo &MRI, MachineInstr &MI,
5490 int Offset) {
5491 MachineBasicBlock *MBB = MI.getParent();
5492 const DebugLoc &DL = MI.getDebugLoc();
5493 MachineBasicBlock::iterator I(&MI);
5494
5495 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5496
5497 if (Offset == 0)
5498 return Idx->getReg();
5499
5500 Register Tmp = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5501 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: Tmp)
5502 .add(MO: *Idx)
5503 .addImm(Val: Offset);
5504 return Tmp;
5505}
5506
5507static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
5508 MachineBasicBlock &MBB,
5509 const GCNSubtarget &ST) {
5510 const SIInstrInfo *TII = ST.getInstrInfo();
5511 const SIRegisterInfo &TRI = TII->getRegisterInfo();
5512 MachineFunction *MF = MBB.getParent();
5513 MachineRegisterInfo &MRI = MF->getRegInfo();
5514
5515 Register Dst = MI.getOperand(i: 0).getReg();
5516 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5517 Register SrcReg = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src)->getReg();
5518 int Offset = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::offset)->getImm();
5519
5520 const TargetRegisterClass *VecRC = MRI.getRegClass(Reg: SrcReg);
5521 const TargetRegisterClass *IdxRC = MRI.getRegClass(Reg: Idx->getReg());
5522
5523 unsigned SubReg;
5524 std::tie(args&: SubReg, args&: Offset) =
5525 computeIndirectRegAndOffset(TRI, SuperRC: VecRC, VecReg: SrcReg, Offset);
5526
5527 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
5528
5529 // Check for a SGPR index.
5530 if (TII->getRegisterInfo().isSGPRClass(RC: IdxRC)) {
5531 MachineBasicBlock::iterator I(&MI);
5532 const DebugLoc &DL = MI.getDebugLoc();
5533
5534 if (UseGPRIdxMode) {
5535 // TODO: Look at the uses to avoid the copy. This may require rescheduling
5536 // to avoid interfering with other uses, so probably requires a new
5537 // optimization pass.
5538 Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
5539
5540 const MCInstrDesc &GPRIDXDesc =
5541 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: true);
5542 BuildMI(BB&: MBB, I, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5543 .addReg(RegNo: SrcReg)
5544 .addReg(RegNo: Idx)
5545 .addImm(Val: SubReg);
5546 } else {
5547 setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
5548
5549 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: Dst)
5550 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
5551 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5552 }
5553
5554 MI.eraseFromParent();
5555
5556 return &MBB;
5557 }
5558
5559 // Control flow needs to be inserted if indexing with a VGPR.
5560 const DebugLoc &DL = MI.getDebugLoc();
5561 MachineBasicBlock::iterator I(&MI);
5562
5563 Register PhiReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5564 Register InitReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5565
5566 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: InitReg);
5567
5568 Register SGPRIdxReg;
5569 auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitResultReg: InitReg, PhiReg, Offset,
5570 UseGPRIdxMode, SGPRIdxReg);
5571
5572 MachineBasicBlock *LoopBB = InsPt->getParent();
5573
5574 if (UseGPRIdxMode) {
5575 const MCInstrDesc &GPRIDXDesc =
5576 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: true);
5577
5578 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5579 .addReg(RegNo: SrcReg)
5580 .addReg(RegNo: SGPRIdxReg)
5581 .addImm(Val: SubReg);
5582 } else {
5583 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: Dst)
5584 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
5585 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5586 }
5587
5588 MI.eraseFromParent();
5589
5590 return LoopBB;
5591}
5592
5593static MachineBasicBlock *emitIndirectDst(MachineInstr &MI,
5594 MachineBasicBlock &MBB,
5595 const GCNSubtarget &ST) {
5596 const SIInstrInfo *TII = ST.getInstrInfo();
5597 const SIRegisterInfo &TRI = TII->getRegisterInfo();
5598 MachineFunction *MF = MBB.getParent();
5599 MachineRegisterInfo &MRI = MF->getRegInfo();
5600
5601 Register Dst = MI.getOperand(i: 0).getReg();
5602 const MachineOperand *SrcVec = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src);
5603 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5604 const MachineOperand *Val = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::val);
5605 int Offset = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::offset)->getImm();
5606 const TargetRegisterClass *VecRC = MRI.getRegClass(Reg: SrcVec->getReg());
5607 const TargetRegisterClass *IdxRC = MRI.getRegClass(Reg: Idx->getReg());
5608
5609 // This can be an immediate, but will be folded later.
5610 assert(Val->getReg());
5611
5612 unsigned SubReg;
5613 std::tie(args&: SubReg, args&: Offset) =
5614 computeIndirectRegAndOffset(TRI, SuperRC: VecRC, VecReg: SrcVec->getReg(), Offset);
5615 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
5616
5617 if (Idx->getReg() == AMDGPU::NoRegister) {
5618 MachineBasicBlock::iterator I(&MI);
5619 const DebugLoc &DL = MI.getDebugLoc();
5620
5621 assert(Offset == 0);
5622
5623 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: Dst)
5624 .add(MO: *SrcVec)
5625 .add(MO: *Val)
5626 .addImm(Val: SubReg);
5627
5628 MI.eraseFromParent();
5629 return &MBB;
5630 }
5631
5632 // Check for a SGPR index.
5633 if (TII->getRegisterInfo().isSGPRClass(RC: IdxRC)) {
5634 MachineBasicBlock::iterator I(&MI);
5635 const DebugLoc &DL = MI.getDebugLoc();
5636
5637 if (UseGPRIdxMode) {
5638 Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
5639
5640 const MCInstrDesc &GPRIDXDesc =
5641 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
5642 BuildMI(BB&: MBB, I, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5643 .addReg(RegNo: SrcVec->getReg())
5644 .add(MO: *Val)
5645 .addReg(RegNo: Idx)
5646 .addImm(Val: SubReg);
5647 } else {
5648 setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
5649
5650 const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
5651 VecSize: TRI.getRegSizeInBits(RC: *VecRC), EltSize: 32, IsSGPR: false);
5652 BuildMI(BB&: MBB, I, MIMD: DL, MCID: MovRelDesc, DestReg: Dst)
5653 .addReg(RegNo: SrcVec->getReg())
5654 .add(MO: *Val)
5655 .addImm(Val: SubReg);
5656 }
5657 MI.eraseFromParent();
5658 return &MBB;
5659 }
5660
5661 // Control flow needs to be inserted if indexing with a VGPR.
5662 if (Val->isReg())
5663 MRI.clearKillFlags(Reg: Val->getReg());
5664
5665 const DebugLoc &DL = MI.getDebugLoc();
5666
5667 Register PhiReg = MRI.createVirtualRegister(RegClass: VecRC);
5668
5669 Register SGPRIdxReg;
5670 auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitResultReg: SrcVec->getReg(), PhiReg, Offset,
5671 UseGPRIdxMode, SGPRIdxReg);
5672 MachineBasicBlock *LoopBB = InsPt->getParent();
5673
5674 if (UseGPRIdxMode) {
5675 const MCInstrDesc &GPRIDXDesc =
5676 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
5677
5678 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5679 .addReg(RegNo: PhiReg)
5680 .add(MO: *Val)
5681 .addReg(RegNo: SGPRIdxReg)
5682 .addImm(Val: SubReg);
5683 } else {
5684 const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
5685 VecSize: TRI.getRegSizeInBits(RC: *VecRC), EltSize: 32, IsSGPR: false);
5686 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: MovRelDesc, DestReg: Dst)
5687 .addReg(RegNo: PhiReg)
5688 .add(MO: *Val)
5689 .addImm(Val: SubReg);
5690 }
5691
5692 MI.eraseFromParent();
5693 return LoopBB;
5694}
5695
5696static MachineBasicBlock *expand64BitScalarArithmetic(MachineInstr &MI,
5697 MachineBasicBlock *BB) {
5698 // For targets older than GFX12, we emit a sequence of 32-bit operations.
5699 // For GFX12, we emit s_add_u64 and s_sub_u64.
5700 MachineFunction *MF = BB->getParent();
5701 const SIInstrInfo *TII = MF->getSubtarget<GCNSubtarget>().getInstrInfo();
5702 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5703 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
5704 const DebugLoc &DL = MI.getDebugLoc();
5705 MachineOperand &Dest = MI.getOperand(i: 0);
5706 MachineOperand &Src0 = MI.getOperand(i: 1);
5707 MachineOperand &Src1 = MI.getOperand(i: 2);
5708 bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
5709 if (ST.hasScalarAddSub64()) {
5710 unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64;
5711 // clang-format off
5712 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest.getReg())
5713 .add(MO: Src0)
5714 .add(MO: Src1);
5715 // clang-format on
5716 } else {
5717 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5718 const TargetRegisterClass *BoolRC = TRI->getBoolRC();
5719
5720 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5721 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5722
5723 MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
5724 MI, MRI, SuperReg: Src0, SuperRC: BoolRC, SubIdx: AMDGPU::sub0, SubRC: &AMDGPU::SReg_32RegClass);
5725 MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
5726 MI, MRI, SuperReg: Src0, SuperRC: BoolRC, SubIdx: AMDGPU::sub1, SubRC: &AMDGPU::SReg_32RegClass);
5727
5728 MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
5729 MI, MRI, SuperReg: Src1, SuperRC: BoolRC, SubIdx: AMDGPU::sub0, SubRC: &AMDGPU::SReg_32RegClass);
5730 MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
5731 MI, MRI, SuperReg: Src1, SuperRC: BoolRC, SubIdx: AMDGPU::sub1, SubRC: &AMDGPU::SReg_32RegClass);
5732
5733 unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
5734 unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
5735 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoOpc), DestReg: DestSub0).add(MO: Src0Sub0).add(MO: Src1Sub0);
5736 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: HiOpc), DestReg: DestSub1).add(MO: Src0Sub1).add(MO: Src1Sub1);
5737 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest.getReg())
5738 .addReg(RegNo: DestSub0)
5739 .addImm(Val: AMDGPU::sub0)
5740 .addReg(RegNo: DestSub1)
5741 .addImm(Val: AMDGPU::sub1);
5742 }
5743 MI.eraseFromParent();
5744 return BB;
5745}
5746
5747static void expand64BitV_CNDMASK(MachineInstr &MI, MachineBasicBlock *BB) {
5748 MachineFunction *MF = BB->getParent();
5749 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5750 const SIInstrInfo *TII = ST.getInstrInfo();
5751 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5752 MachineRegisterInfo &MRI = MF->getRegInfo();
5753 const DebugLoc &DL = MI.getDebugLoc();
5754 Register Dst = MI.getOperand(i: 0).getReg();
5755 const MachineOperand &Src0 = MI.getOperand(i: 1);
5756 const MachineOperand &Src1 = MI.getOperand(i: 2);
5757 Register SrcCond = MI.getOperand(i: 3).getReg();
5758
5759 Register DstLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5760 Register DstHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5761 const TargetRegisterClass *CondRC = TRI->getWaveMaskRegClass();
5762 Register SrcCondCopy = MRI.createVirtualRegister(RegClass: CondRC);
5763
5764 int Src0Idx =
5765 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
5766 int Src1Idx =
5767 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src1);
5768 const TargetRegisterClass *Src0RC =
5769 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: Src0Idx));
5770 const TargetRegisterClass *Src1RC =
5771 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: Src1Idx));
5772
5773 const TargetRegisterClass *Src0SubRC =
5774 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
5775 const TargetRegisterClass *Src1SubRC =
5776 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
5777
5778 MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
5779 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
5780 MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
5781 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
5782
5783 MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
5784 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
5785 MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
5786 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
5787
5788 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: SrcCondCopy).addReg(RegNo: SrcCond);
5789 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstLo)
5790 .addImm(Val: 0)
5791 .add(MO: Src0Sub0)
5792 .addImm(Val: 0)
5793 .add(MO: Src1Sub0)
5794 .addReg(RegNo: SrcCondCopy);
5795
5796 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstHi)
5797 .addImm(Val: 0)
5798 .add(MO: Src0Sub1)
5799 .addImm(Val: 0)
5800 .add(MO: Src1Sub1)
5801 .addReg(RegNo: SrcCondCopy);
5802
5803 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
5804 .addReg(RegNo: DstLo)
5805 .addImm(Val: AMDGPU::sub0)
5806 .addReg(RegNo: DstHi)
5807 .addImm(Val: AMDGPU::sub1);
5808 MI.eraseFromParent();
5809}
5810
5811static uint64_t getIdentityValueForWaveReduction(unsigned Opc) {
5812 switch (Opc) {
5813 case AMDGPU::S_MIN_U32:
5814 return std::numeric_limits<uint32_t>::max();
5815 case AMDGPU::S_MIN_I32:
5816 return std::numeric_limits<int32_t>::max();
5817 case AMDGPU::S_MAX_U32:
5818 return std::numeric_limits<uint32_t>::min();
5819 case AMDGPU::S_MAX_I32:
5820 return std::numeric_limits<int32_t>::min();
5821 case AMDGPU::V_ADD_F32_e64: // -0.0
5822 return 0x80000000;
5823 case AMDGPU::V_SUB_F32_e64: // +0.0
5824 return 0x0;
5825 case AMDGPU::S_ADD_I32:
5826 case AMDGPU::S_SUB_I32:
5827 case AMDGPU::S_OR_B32:
5828 case AMDGPU::S_XOR_B32:
5829 return std::numeric_limits<uint32_t>::min();
5830 case AMDGPU::S_AND_B32:
5831 return std::numeric_limits<uint32_t>::max();
5832 case AMDGPU::V_MIN_F32_e64:
5833 case AMDGPU::V_MAX_F32_e64:
5834 return 0x7fc00000; // qNAN
5835 case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
5836 return std::numeric_limits<uint64_t>::max();
5837 case AMDGPU::V_CMP_LT_I64_e64: // min.i64
5838 return std::numeric_limits<int64_t>::max();
5839 case AMDGPU::V_CMP_GT_U64_e64: // umax.u64
5840 return std::numeric_limits<uint64_t>::min();
5841 case AMDGPU::V_CMP_GT_I64_e64: // max.i64
5842 return std::numeric_limits<int64_t>::min();
5843 case AMDGPU::V_MIN_F64_e64:
5844 case AMDGPU::V_MAX_F64_e64:
5845 case AMDGPU::V_MIN_NUM_F64_e64:
5846 case AMDGPU::V_MAX_NUM_F64_e64:
5847 return 0x7FF8000000000000; // qNAN
5848 case AMDGPU::S_ADD_U64_PSEUDO:
5849 case AMDGPU::S_SUB_U64_PSEUDO:
5850 case AMDGPU::S_OR_B64:
5851 case AMDGPU::S_XOR_B64:
5852 return std::numeric_limits<uint64_t>::min();
5853 case AMDGPU::S_AND_B64:
5854 return std::numeric_limits<uint64_t>::max();
5855 case AMDGPU::V_ADD_F64_e64:
5856 case AMDGPU::V_ADD_F64_pseudo_e64:
5857 return 0x8000000000000000; // -0.0
5858 default:
5859 llvm_unreachable("Unexpected opcode in getIdentityValueForWaveReduction");
5860 }
5861}
5862
5863static bool is32bitWaveReduceOperation(unsigned Opc) {
5864 return Opc == AMDGPU::S_MIN_U32 || Opc == AMDGPU::S_MIN_I32 ||
5865 Opc == AMDGPU::S_MAX_U32 || Opc == AMDGPU::S_MAX_I32 ||
5866 Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32 ||
5867 Opc == AMDGPU::S_AND_B32 || Opc == AMDGPU::S_OR_B32 ||
5868 Opc == AMDGPU::S_XOR_B32 || Opc == AMDGPU::V_MIN_F32_e64 ||
5869 Opc == AMDGPU::V_MAX_F32_e64 || Opc == AMDGPU::V_ADD_F32_e64 ||
5870 Opc == AMDGPU::V_SUB_F32_e64;
5871}
5872
5873static bool isFloatingPointWaveReduceOperation(unsigned Opc) {
5874 return Opc == AMDGPU::V_MIN_F32_e64 || Opc == AMDGPU::V_MAX_F32_e64 ||
5875 Opc == AMDGPU::V_ADD_F32_e64 || Opc == AMDGPU::V_SUB_F32_e64 ||
5876 Opc == AMDGPU::V_MIN_F64_e64 || Opc == AMDGPU::V_MAX_F64_e64 ||
5877 Opc == AMDGPU::V_MIN_NUM_F64_e64 || Opc == AMDGPU::V_MAX_NUM_F64_e64 ||
5878 Opc == AMDGPU::V_ADD_F64_e64 || Opc == AMDGPU::V_ADD_F64_pseudo_e64;
5879}
5880
5881static std::tuple<unsigned, unsigned>
5882getDPPOpcForWaveReduction(unsigned Opc, const GCNSubtarget &ST) {
5883 unsigned DPPOpc;
5884 switch (Opc) {
5885 case AMDGPU::S_MIN_U32:
5886 DPPOpc = AMDGPU::V_MIN_U32_dpp;
5887 break;
5888 case AMDGPU::S_MIN_I32:
5889 DPPOpc = AMDGPU::V_MIN_I32_dpp;
5890 break;
5891 case AMDGPU::S_MAX_U32:
5892 DPPOpc = AMDGPU::V_MAX_U32_dpp;
5893 break;
5894 case AMDGPU::S_MAX_I32:
5895 DPPOpc = AMDGPU::V_MAX_I32_dpp;
5896 break;
5897 case AMDGPU::S_ADD_I32:
5898 case AMDGPU::S_SUB_I32:
5899 DPPOpc = ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_dpp
5900 : AMDGPU::V_ADD_CO_U32_dpp;
5901 break;
5902 case AMDGPU::S_AND_B32:
5903 DPPOpc = AMDGPU::V_AND_B32_dpp;
5904 break;
5905 case AMDGPU::S_OR_B32:
5906 DPPOpc = AMDGPU::V_OR_B32_dpp;
5907 break;
5908 case AMDGPU::S_XOR_B32:
5909 DPPOpc = AMDGPU::V_XOR_B32_dpp;
5910 break;
5911 case AMDGPU::V_ADD_F32_e64:
5912 case AMDGPU::V_SUB_F32_e64:
5913 DPPOpc = AMDGPU::V_ADD_F32_dpp;
5914 break;
5915 case AMDGPU::V_MIN_F32_e64:
5916 DPPOpc = AMDGPU::V_MIN_F32_dpp;
5917 break;
5918 case AMDGPU::V_MAX_F32_e64:
5919 DPPOpc = AMDGPU::V_MAX_F32_dpp;
5920 break;
5921 case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
5922 case AMDGPU::V_CMP_LT_I64_e64: // min.i64
5923 case AMDGPU::V_CMP_GT_U64_e64: // umax.u64
5924 case AMDGPU::V_CMP_GT_I64_e64: // max.i64
5925 case AMDGPU::S_ADD_U64_PSEUDO:
5926 case AMDGPU::S_SUB_U64_PSEUDO:
5927 case AMDGPU::S_AND_B64:
5928 case AMDGPU::S_OR_B64:
5929 case AMDGPU::S_XOR_B64:
5930 case AMDGPU::V_MIN_NUM_F64_e64:
5931 case AMDGPU::V_MIN_F64_e64:
5932 case AMDGPU::V_MAX_NUM_F64_e64:
5933 case AMDGPU::V_MAX_F64_e64:
5934 case AMDGPU::V_ADD_F64_pseudo_e64:
5935 case AMDGPU::V_ADD_F64_e64:
5936 DPPOpc = AMDGPU::V_MOV_B64_DPP_PSEUDO;
5937 break;
5938 default:
5939 llvm_unreachable("unhandled lane op");
5940 }
5941 unsigned ClampOpc = Opc;
5942 if (!ST.getInstrInfo()->isVALU(Opcode: Opc, /*AllowLDSDMA=*/true)) {
5943 if (Opc == AMDGPU::S_SUB_I32)
5944 ClampOpc = AMDGPU::S_ADD_I32;
5945 if (Opc == AMDGPU::S_ADD_U64_PSEUDO || Opc == AMDGPU::S_SUB_U64_PSEUDO)
5946 ClampOpc = AMDGPU::V_ADD_CO_U32_e64;
5947 else if (Opc == AMDGPU::S_AND_B64)
5948 ClampOpc = AMDGPU::V_AND_B32_e64;
5949 else if (Opc == AMDGPU::S_OR_B64)
5950 ClampOpc = AMDGPU::V_OR_B32_e64;
5951 else if (Opc == AMDGPU::S_XOR_B64)
5952 ClampOpc = AMDGPU::V_XOR_B32_e64;
5953 else
5954 ClampOpc = ST.getInstrInfo()->getVALUOp(Opc: ClampOpc);
5955 }
5956 return {DPPOpc, ClampOpc};
5957}
5958
5959static std::pair<Register, Register>
5960ExtractSubRegs(MachineInstr &MI, MachineOperand &Op,
5961 const TargetRegisterClass *SrcRC, const GCNSubtarget &ST,
5962 MachineRegisterInfo &MRI) {
5963 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5964 const SIInstrInfo *TII = ST.getInstrInfo();
5965 const TargetRegisterClass *SrcSubRC =
5966 TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
5967 Register Op1L =
5968 TII->buildExtractSubReg(MI, MRI, SuperReg: Op, SuperRC: SrcRC, SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
5969 Register Op1H =
5970 TII->buildExtractSubReg(MI, MRI, SuperReg: Op, SuperRC: SrcRC, SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
5971 return {Op1L, Op1H};
5972}
5973
5974static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
5975 MachineBasicBlock &BB,
5976 const GCNSubtarget &ST,
5977 unsigned Opc) {
5978 MachineRegisterInfo &MRI = BB.getParent()->getRegInfo();
5979 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5980 const DebugLoc &DL = MI.getDebugLoc();
5981 const SIInstrInfo *TII = ST.getInstrInfo();
5982
5983 // Reduction operations depend on whether the input operand is SGPR or VGPR.
5984 Register SrcReg = MI.getOperand(i: 1).getReg();
5985 bool isSGPR = TRI->isSGPRClass(RC: MRI.getRegClass(Reg: SrcReg));
5986 Register DstReg = MI.getOperand(i: 0).getReg();
5987 unsigned Stratergy = static_cast<unsigned>(MI.getOperand(i: 2).getImm());
5988 enum WAVE_REDUCE_STRATEGY : unsigned { DEFAULT = 0, ITERATIVE = 1, DPP = 2 };
5989 MachineBasicBlock *RetBB = nullptr;
5990 unsigned MIOpc = MI.getOpcode();
5991 auto BuildRegSequence = [&](MachineBasicBlock &BB,
5992 MachineBasicBlock::iterator MI, Register Dst,
5993 Register Src0, Register Src1) {
5994 auto RegSequence =
5995 BuildMI(BB, I: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dst)
5996 .addReg(RegNo: Src0)
5997 .addImm(Val: AMDGPU::sub0)
5998 .addReg(RegNo: Src1)
5999 .addImm(Val: AMDGPU::sub1);
6000 return RegSequence;
6001 };
6002 if (isSGPR) {
6003 switch (Opc) {
6004 case AMDGPU::S_MIN_U32:
6005 case AMDGPU::S_MIN_I32:
6006 case AMDGPU::V_MIN_F32_e64:
6007 case AMDGPU::S_MAX_U32:
6008 case AMDGPU::S_MAX_I32:
6009 case AMDGPU::V_MAX_F32_e64:
6010 case AMDGPU::S_AND_B32:
6011 case AMDGPU::S_OR_B32: {
6012 // Idempotent operations.
6013 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstReg).addReg(RegNo: SrcReg);
6014 RetBB = &BB;
6015 break;
6016 }
6017 case AMDGPU::V_CMP_LT_U64_e64: // umin
6018 case AMDGPU::V_CMP_LT_I64_e64: // min
6019 case AMDGPU::V_CMP_GT_U64_e64: // umax
6020 case AMDGPU::V_CMP_GT_I64_e64: // max
6021 case AMDGPU::V_MIN_F64_e64:
6022 case AMDGPU::V_MIN_NUM_F64_e64:
6023 case AMDGPU::V_MAX_F64_e64:
6024 case AMDGPU::V_MAX_NUM_F64_e64:
6025 case AMDGPU::S_AND_B64:
6026 case AMDGPU::S_OR_B64: {
6027 // Idempotent operations.
6028 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B64), DestReg: DstReg).addReg(RegNo: SrcReg);
6029 RetBB = &BB;
6030 break;
6031 }
6032 case AMDGPU::S_XOR_B32:
6033 case AMDGPU::S_XOR_B64:
6034 case AMDGPU::S_ADD_I32:
6035 case AMDGPU::S_ADD_U64_PSEUDO:
6036 case AMDGPU::V_ADD_F32_e64:
6037 case AMDGPU::V_ADD_F64_e64:
6038 case AMDGPU::V_ADD_F64_pseudo_e64:
6039 case AMDGPU::S_SUB_I32:
6040 case AMDGPU::S_SUB_U64_PSEUDO:
6041 case AMDGPU::V_SUB_F32_e64: {
6042 const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
6043 const TargetRegisterClass *DstRegClass = MRI.getRegClass(Reg: DstReg);
6044 Register ExecMask = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6045 Register NumActiveLanes =
6046 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6047
6048 bool IsWave32 = ST.isWave32();
6049 unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
6050 MCRegister ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6051 unsigned BitCountOpc =
6052 IsWave32 ? AMDGPU::S_BCNT1_I32_B32 : AMDGPU::S_BCNT1_I32_B64;
6053
6054 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MovOpc), DestReg: ExecMask).addReg(RegNo: ExecReg);
6055
6056 auto NewAccumulator =
6057 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BitCountOpc), DestReg: NumActiveLanes)
6058 .addReg(RegNo: ExecMask);
6059
6060 switch (Opc) {
6061 case AMDGPU::S_XOR_B32:
6062 case AMDGPU::S_XOR_B64: {
6063 // Performing an XOR operation on a uniform value
6064 // depends on the parity of the number of active lanes.
6065 // For even parity, the result will be 0, for odd
6066 // parity the result will be the same as the input value.
6067 Register ParityRegister =
6068 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6069
6070 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_AND_B32), DestReg: ParityRegister)
6071 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
6072 .addImm(Val: 1)
6073 .setOperandDead(3); // Dead scc
6074 if (Opc == AMDGPU::S_XOR_B32) {
6075 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
6076 .addReg(RegNo: SrcReg)
6077 .addReg(RegNo: ParityRegister);
6078 } else {
6079 Register DestSub0 =
6080 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6081 Register DestSub1 =
6082 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6083 auto [Op1L, Op1H] = ExtractSubRegs(MI, Op&: MI.getOperand(i: 1),
6084 SrcRC: MRI.getRegClass(Reg: SrcReg), ST, MRI);
6085 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub0)
6086 .addReg(RegNo: Op1L)
6087 .addReg(RegNo: ParityRegister);
6088 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub1)
6089 .addReg(RegNo: Op1H)
6090 .addReg(RegNo: ParityRegister);
6091 BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
6092 }
6093 break;
6094 }
6095 case AMDGPU::S_SUB_I32: {
6096 Register NegatedVal = MRI.createVirtualRegister(RegClass: DstRegClass);
6097
6098 // Take the negation of the source operand.
6099 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedVal)
6100 .addImm(Val: 0)
6101 .addReg(RegNo: SrcReg);
6102 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
6103 .addReg(RegNo: NegatedVal)
6104 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg());
6105 break;
6106 }
6107 case AMDGPU::S_ADD_I32: {
6108 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
6109 .addReg(RegNo: SrcReg)
6110 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg());
6111 break;
6112 }
6113 case AMDGPU::S_ADD_U64_PSEUDO:
6114 case AMDGPU::S_SUB_U64_PSEUDO: {
6115 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6116 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6117 Register Op1H_Op0L_Reg =
6118 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6119 Register Op1L_Op0H_Reg =
6120 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6121 Register CarryReg =
6122 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6123 Register AddReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6124 Register NegatedValLo =
6125 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6126 Register NegatedValHi =
6127 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6128 auto [Op1L, Op1H] = ExtractSubRegs(MI, Op&: MI.getOperand(i: 1),
6129 SrcRC: MRI.getRegClass(Reg: SrcReg), ST, MRI);
6130 if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6131 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedValLo)
6132 .addImm(Val: 0)
6133 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
6134 .setOperandDead(3); // Dead scc
6135 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ASHR_I32), DestReg: NegatedValHi)
6136 .addReg(RegNo: NegatedValLo)
6137 .addImm(Val: 31)
6138 .setOperandDead(3); // Dead scc
6139 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: Op1L_Op0H_Reg)
6140 .addReg(RegNo: Op1L)
6141 .addReg(RegNo: NegatedValHi);
6142 }
6143 Register LowOpcode = Opc == AMDGPU::S_SUB_U64_PSEUDO
6144 ? NegatedValLo
6145 : NewAccumulator->getOperand(i: 0).getReg();
6146 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub0)
6147 .addReg(RegNo: Op1L)
6148 .addReg(RegNo: LowOpcode);
6149 if (ST.hasScalarMulHiInsts()) {
6150 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_HI_U32), DestReg: CarryReg)
6151 .addReg(RegNo: Op1L)
6152 .addReg(RegNo: LowOpcode);
6153 } else {
6154 Register VCarryReg =
6155 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6156 Register LowOpVGPR =
6157 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6158 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: LowOpVGPR)
6159 .addReg(RegNo: LowOpcode);
6160 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MUL_HI_U32_e64), DestReg: VCarryReg)
6161 .addReg(RegNo: Op1L)
6162 .addReg(RegNo: LowOpVGPR);
6163 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CarryReg)
6164 .addReg(RegNo: VCarryReg);
6165 }
6166 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: Op1H_Op0L_Reg)
6167 .addReg(RegNo: Op1H)
6168 .addReg(RegNo: LowOpcode);
6169
6170 Register HiVal = Opc == AMDGPU::S_SUB_U64_PSEUDO ? AddReg : DestSub1;
6171 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: HiVal)
6172 .addReg(RegNo: CarryReg)
6173 .addReg(RegNo: Op1H_Op0L_Reg)
6174 .setOperandDead(3); // Dead scc
6175
6176 if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6177 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: DestSub1)
6178 .addReg(RegNo: HiVal)
6179 .addReg(RegNo: Op1L_Op0H_Reg)
6180 .setOperandDead(3); // Dead scc
6181 }
6182 BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
6183 break;
6184 }
6185 case AMDGPU::V_ADD_F32_e64:
6186 case AMDGPU::V_ADD_F64_e64:
6187 case AMDGPU::V_ADD_F64_pseudo_e64:
6188 case AMDGPU::V_SUB_F32_e64: {
6189 bool is32BitOpc = is32bitWaveReduceOperation(Opc);
6190 const TargetRegisterClass *VregRC = TII->getRegClass(MCID: TII->get(Opcode: Opc), OpNum: 0);
6191 Register ActiveLanesVreg = MRI.createVirtualRegister(RegClass: VregRC);
6192 Register DstVreg = MRI.createVirtualRegister(RegClass: VregRC);
6193 // Get number of active lanes as a float val.
6194 BuildMI(BB, I&: MI, MIMD: DL,
6195 MCID: TII->get(Opcode: is32BitOpc ? AMDGPU::V_CVT_F32_I32_e64
6196 : AMDGPU::V_CVT_F64_I32_e64),
6197 DestReg: ActiveLanesVreg)
6198 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
6199 .addImm(Val: 0) // clamp
6200 .addImm(Val: 0); // output-modifier
6201
6202 // Take negation of input for SUB reduction
6203 unsigned srcMod = (MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32 ||
6204 MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64)
6205 ? SISrcMods::NEG
6206 : SISrcMods::NONE;
6207 unsigned MulOpc = is32BitOpc ? AMDGPU::V_MUL_F32_e64
6208 : ST.getGeneration() >= AMDGPUSubtarget::GFX12
6209 ? AMDGPU::V_MUL_F64_pseudo_e64
6210 : AMDGPU::V_MUL_F64_e64;
6211 auto DestVregInst = BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MulOpc),
6212 DestReg: DstVreg)
6213 .addImm(Val: srcMod) // src0 modifier
6214 .addReg(RegNo: SrcReg)
6215 .addImm(Val: SISrcMods::NONE) // src1 modifier
6216 .addReg(RegNo: ActiveLanesVreg)
6217 .addImm(Val: SISrcMods::NONE) // clamp
6218 .addImm(Val: SISrcMods::NONE); // output-mod
6219 if (is32BitOpc) {
6220 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
6221 .addReg(RegNo: DstVreg);
6222 } else {
6223 Register LaneValueLoReg =
6224 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6225 Register LaneValueHiReg =
6226 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6227 auto [Op1L, Op1H] =
6228 ExtractSubRegs(MI, Op&: DestVregInst->getOperand(i: 0), SrcRC: VregRC, ST, MRI);
6229 // lane value input should be in an sgpr
6230 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
6231 DestReg: LaneValueLoReg)
6232 .addReg(RegNo: Op1L);
6233 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
6234 DestReg: LaneValueHiReg)
6235 .addReg(RegNo: Op1H);
6236 NewAccumulator =
6237 BuildRegSequence(BB, MI, DstReg, LaneValueLoReg, LaneValueHiReg);
6238 }
6239 }
6240 }
6241 RetBB = &BB;
6242 }
6243 }
6244 } else {
6245 MachineBasicBlock::iterator I = BB.end();
6246 Register SrcReg = MI.getOperand(i: 1).getReg();
6247 bool is32BitOpc = is32bitWaveReduceOperation(Opc);
6248 bool isFPOp = isFloatingPointWaveReduceOperation(Opc);
6249 bool NeedsMovDPP = !is32BitOpc;
6250 // Create virtual registers required for lowering.
6251 const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
6252 const TargetRegisterClass *DstRegClass = MRI.getRegClass(Reg: DstReg);
6253 const TargetRegisterClass *SrcRegClass = MRI.getRegClass(Reg: SrcReg);
6254 bool IsWave32 = ST.isWave32();
6255 unsigned MovOpcForExec = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
6256 unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6257 if (Stratergy == WAVE_REDUCE_STRATEGY::ITERATIVE ||
6258 !ST.hasDPP()) { // If target doesn't support DPP operations, default to
6259 // iterative stratergy
6260
6261 // To reduce the VGPR using iterative approach, we need to iterate
6262 // over all the active lanes. Lowering consists of ComputeLoop,
6263 // which iterate over only active lanes. We use copy of EXEC register
6264 // as induction variable and every active lane modifies it using bitset0
6265 // so that we will get the next active lane for next iteration.
6266
6267 // Create Control flow for loop
6268 // Split MI's Machine Basic block into For loop
6269 auto [ComputeLoop, ComputeEnd] = splitBlockForLoop(MI, MBB&: BB, InstInLoop: true);
6270
6271 Register LoopIterator = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6272 Register IdentityValReg = MRI.createVirtualRegister(RegClass: DstRegClass);
6273 Register AccumulatorReg = MRI.createVirtualRegister(RegClass: DstRegClass);
6274 Register ActiveBitsReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6275 Register NewActiveBitsReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6276 Register FF1Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6277 Register LaneValueReg = MRI.createVirtualRegister(RegClass: DstRegClass);
6278
6279 // Create initial values of induction variable from Exec, Accumulator and
6280 // insert branch instr to newly created ComputeBlock
6281 BuildMI(BB, I, MIMD: DL, MCID: TII->get(Opcode: MovOpcForExec), DestReg: LoopIterator).addReg(RegNo: ExecReg);
6282 uint64_t IdentityValue =
6283 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
6284 ? 0x0 // +0.0 for double sub reduction
6285 : getIdentityValueForWaveReduction(Opc);
6286 BuildMI(BB, I, MIMD: DL,
6287 MCID: TII->get(Opcode: is32BitOpc ? AMDGPU::S_MOV_B32
6288 : AMDGPU::S_MOV_B64_IMM_PSEUDO),
6289 DestReg: IdentityValReg)
6290 .addImm(Val: IdentityValue);
6291 // clang-format off
6292 BuildMI(BB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_BRANCH))
6293 .addMBB(MBB: ComputeLoop);
6294 // clang-format on
6295
6296 // Start constructing ComputeLoop
6297 I = ComputeLoop->begin();
6298 auto Accumulator =
6299 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::PHI), DestReg: AccumulatorReg)
6300 .addReg(RegNo: IdentityValReg)
6301 .addMBB(MBB: &BB);
6302 auto ActiveBits =
6303 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::PHI), DestReg: ActiveBitsReg)
6304 .addReg(RegNo: LoopIterator)
6305 .addMBB(MBB: &BB);
6306
6307 I = ComputeLoop->end();
6308 MachineInstr *NewAccumulator;
6309 // Perform the computations
6310 unsigned SFFOpc =
6311 IsWave32 ? AMDGPU::S_FF1_I32_B32 : AMDGPU::S_FF1_I32_B64;
6312 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: SFFOpc), DestReg: FF1Reg)
6313 .addReg(RegNo: ActiveBitsReg);
6314 if (is32BitOpc) {
6315 Register OpDstReg = DstReg;
6316 bool hasSrc0Modifier = AMDGPU::getNamedOperandIdx(
6317 Opcode: Opc, Name: AMDGPU::OpName::src0_modifiers) != -1;
6318 bool hasSrc1Modifier = AMDGPU::getNamedOperandIdx(
6319 Opcode: Opc, Name: AMDGPU::OpName::src1_modifiers) != -1;
6320 bool hasClamp =
6321 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::clamp) != -1;
6322 bool hasOpSel =
6323 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel) != -1;
6324 bool hasOMod =
6325 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::omod) != -1;
6326 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6327 DestReg: LaneValueReg)
6328 .addReg(RegNo: SrcReg)
6329 .addReg(RegNo: FF1Reg);
6330 if (ST.getInstrInfo()->isVALU(Opcode: Opc, /*AllowLDSDMA=*/true)) {
6331 // Get the Lane Value in VGPR to avoid the Constant Bus Restriction
6332 Register LaneValVgpr = MRI.createVirtualRegister(RegClass: SrcRegClass);
6333 Register VgprResultReg = MRI.createVirtualRegister(RegClass: SrcRegClass);
6334 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: LaneValVgpr)
6335 .addReg(RegNo: LaneValueReg);
6336 OpDstReg = VgprResultReg;
6337 LaneValueReg = LaneValVgpr;
6338 }
6339 auto OpInstr = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: OpDstReg);
6340 if (hasSrc0Modifier)
6341 OpInstr.addImm(Val: SISrcMods::NONE); // src0 modifier
6342 OpInstr.addReg(RegNo: AccumulatorReg); // src0
6343 if (hasSrc1Modifier)
6344 OpInstr.addImm(Val: SISrcMods::NONE); // src1 modifier
6345 OpInstr.addReg(RegNo: LaneValueReg); // src1
6346 if (hasClamp)
6347 OpInstr.addImm(Val: 0); // clamp
6348 if (hasOpSel)
6349 OpInstr.addImm(Val: 0); // opsel
6350 if (hasOMod)
6351 OpInstr.addImm(Val: 0); // omod
6352 if (ST.getInstrInfo()->isVALU(Opcode: Opc, /*AllowLDSDMA=*/true)) {
6353 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
6354 DestReg: DstReg)
6355 .addReg(RegNo: OpDstReg);
6356 }
6357 } else {
6358 Register LaneValueLoReg =
6359 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6360 Register LaneValueHiReg =
6361 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6362 Register LaneValReg =
6363 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6364 auto [Op1L, Op1H] = ExtractSubRegs(MI, Op&: MI.getOperand(i: 1),
6365 SrcRC: MRI.getRegClass(Reg: SrcReg), ST, MRI);
6366 // lane value input should be in an sgpr
6367 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6368 DestReg: LaneValueLoReg)
6369 .addReg(RegNo: Op1L)
6370 .addReg(RegNo: FF1Reg);
6371 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6372 DestReg: LaneValueHiReg)
6373 .addReg(RegNo: Op1H)
6374 .addReg(RegNo: FF1Reg);
6375 auto LaneValue = BuildRegSequence(*ComputeLoop, I, LaneValReg,
6376 LaneValueLoReg, LaneValueHiReg);
6377 switch (Opc) {
6378 case AMDGPU::S_OR_B64:
6379 case AMDGPU::S_AND_B64:
6380 case AMDGPU::S_XOR_B64: {
6381 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstReg)
6382 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg())
6383 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6384 .setOperandDead(3); // Dead scc
6385 break;
6386 }
6387 case AMDGPU::V_CMP_GT_I64_e64:
6388 case AMDGPU::V_CMP_GT_U64_e64:
6389 case AMDGPU::V_CMP_LT_I64_e64:
6390 case AMDGPU::V_CMP_LT_U64_e64: {
6391 Register LaneMaskReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6392 Register ComparisonResultReg =
6393 MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6394 int SrcIdx =
6395 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src);
6396 const TargetRegisterClass *VregClass =
6397 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: SrcIdx));
6398 Register AccumulatorVReg = MRI.createVirtualRegister(RegClass: VregClass);
6399 auto [SrcReg0Sub0, SrcReg0Sub1] = ExtractSubRegs(
6400 MI, Op&: Accumulator->getOperand(i: 0), SrcRC: VregClass, ST, MRI);
6401 BuildRegSequence(*ComputeLoop, I, AccumulatorVReg, SrcReg0Sub0,
6402 SrcReg0Sub1);
6403 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: LaneMaskReg)
6404 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6405 .addReg(RegNo: AccumulatorVReg);
6406
6407 unsigned AndOpc = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
6408 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AndOpc), DestReg: ComparisonResultReg)
6409 .addReg(RegNo: LaneMaskReg)
6410 .addReg(RegNo: ActiveBitsReg);
6411
6412 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6413 MCID: TII->get(Opcode: AMDGPU::S_CSELECT_B64), DestReg: DstReg)
6414 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6415 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg());
6416 break;
6417 }
6418 case AMDGPU::V_MIN_F64_e64:
6419 case AMDGPU::V_MIN_NUM_F64_e64:
6420 case AMDGPU::V_MAX_F64_e64:
6421 case AMDGPU::V_MAX_NUM_F64_e64:
6422 case AMDGPU::V_ADD_F64_e64:
6423 case AMDGPU::V_ADD_F64_pseudo_e64: {
6424 int SrcIdx =
6425 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src);
6426 const TargetRegisterClass *VregRC =
6427 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: SrcIdx));
6428 Register AccumulatorVReg = MRI.createVirtualRegister(RegClass: VregRC);
6429 Register DstVreg = MRI.createVirtualRegister(RegClass: VregRC);
6430 Register LaneValLo =
6431 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6432 Register LaneValHi =
6433 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6434 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AccumulatorVReg)
6435 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg());
6436 unsigned Modifier =
6437 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
6438 ? SISrcMods::NEG
6439 : SISrcMods::NONE;
6440 auto DstVregInst =
6441 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstVreg)
6442 .addImm(Val: Modifier) // src0 modifiers
6443 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6444 .addImm(Val: SISrcMods::NONE) // src1 modifiers
6445 .addReg(RegNo: AccumulatorVReg)
6446 .addImm(Val: SISrcMods::NONE) // clamp
6447 .addImm(Val: SISrcMods::NONE); // omod
6448 auto ReadLaneLo =
6449 BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6450 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: LaneValLo);
6451 auto ReadLaneHi =
6452 BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6453 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: LaneValHi);
6454 MachineBasicBlock::iterator Iters = *ReadLaneLo;
6455 auto [Op1L, Op1H] = ExtractSubRegs(MI&: *Iters, Op&: DstVregInst->getOperand(i: 0),
6456 SrcRC: VregRC, ST, MRI);
6457 ReadLaneLo.addReg(RegNo: Op1L);
6458 ReadLaneHi.addReg(RegNo: Op1H);
6459 NewAccumulator =
6460 BuildRegSequence(*ComputeLoop, I, DstReg, LaneValLo, LaneValHi);
6461 break;
6462 }
6463 case AMDGPU::S_ADD_U64_PSEUDO:
6464 case AMDGPU::S_SUB_U64_PSEUDO: {
6465 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstReg)
6466 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg())
6467 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg());
6468 ComputeLoop =
6469 expand64BitScalarArithmetic(MI&: *NewAccumulator, BB: ComputeLoop);
6470 break;
6471 }
6472 }
6473 }
6474 // Manipulate the iterator to get the next active lane
6475 unsigned BITSETOpc =
6476 IsWave32 ? AMDGPU::S_BITSET0_B32 : AMDGPU::S_BITSET0_B64;
6477 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: BITSETOpc), DestReg: NewActiveBitsReg)
6478 .addReg(RegNo: FF1Reg)
6479 .addReg(RegNo: ActiveBitsReg);
6480
6481 // Add phi nodes
6482 Accumulator.addReg(RegNo: DstReg).addMBB(MBB: ComputeLoop);
6483 ActiveBits.addReg(RegNo: NewActiveBitsReg).addMBB(MBB: ComputeLoop);
6484
6485 // Creating branching
6486 MachineInstrBuilder SetSCCInstr;
6487 if (!ST.hasScalarCompareEq64()) {
6488 // For targets <= gfx7, use an S_OR_B32/B64 instruction to set SCC.
6489 Register LaneMaskReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6490 unsigned CMPOpc = IsWave32 ? AMDGPU::S_OR_B32 : AMDGPU::S_OR_B64;
6491 SetSCCInstr =
6492 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: CMPOpc), DestReg: LaneMaskReg);
6493 } else {
6494 unsigned CMPOpc =
6495 IsWave32 ? AMDGPU::S_CMP_LG_U32 : AMDGPU::S_CMP_LG_U64;
6496 SetSCCInstr = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: CMPOpc));
6497 }
6498 SetSCCInstr.addReg(RegNo: NewActiveBitsReg);
6499 if (ST.hasScalarCompareEq64())
6500 SetSCCInstr.addImm(Val: 0);
6501 else
6502 SetSCCInstr.addReg(RegNo: NewActiveBitsReg);
6503 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
6504 .addMBB(MBB: ComputeLoop);
6505
6506 RetBB = ComputeEnd;
6507 } else {
6508 assert(ST.hasDPP() && "Sub Target does not support DPP Operations");
6509 MachineBasicBlock *CurrBB = &BB;
6510 Register SrcWithIdentity = MRI.createVirtualRegister(RegClass: SrcRegClass);
6511 Register IdentityVGPR = MRI.createVirtualRegister(RegClass: SrcRegClass);
6512 Register IdentitySGPR = MRI.createVirtualRegister(RegClass: DstRegClass);
6513 Register DPPRowShr1 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6514 Register DPPRowShr2 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6515 Register DPPRowShr4 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6516 Register DPPRowShr8 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6517 Register RowBcast15 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6518 Register ReducedValSGPR = MRI.createVirtualRegister(RegClass: DstRegClass);
6519 Register NegatedReducedVal = MRI.createVirtualRegister(RegClass: DstRegClass);
6520 Register RowBcast31 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6521 Register UndefExec = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6522 Register FinalDPPResult;
6523 MachineInstr *SrcWithIdentityInstr;
6524 MachineInstr *LastBcastInstr;
6525 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefExec);
6526
6527 uint64_t IdentityValue = getIdentityValueForWaveReduction(Opc);
6528 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL,
6529 MCID: TII->get(Opcode: is32BitOpc ? AMDGPU::S_MOV_B32
6530 : AMDGPU::S_MOV_B64_IMM_PSEUDO),
6531 DestReg: IdentitySGPR)
6532 .addImm(Val: IdentityValue);
6533 auto IdentityCopyInstr =
6534 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: IdentityVGPR)
6535 .addReg(RegNo: IdentitySGPR);
6536 auto DPPClampOpcPair = getDPPOpcForWaveReduction(Opc, ST);
6537 unsigned DPPOpc = std::get<0>(t&: DPPClampOpcPair);
6538 unsigned ClampOpc = std::get<1>(t&: DPPClampOpcPair);
6539 auto BuildSetInactiveInstr = [&](Register Dst, Register Src0,
6540 Register Src1) {
6541 return BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_SET_INACTIVE_B32),
6542 DestReg: Dst)
6543 .addImm(Val: 0) // src0 modifiers
6544 .addReg(RegNo: Src0) // src0
6545 .addImm(Val: 0) // src1 modifiers
6546 .addReg(RegNo: Src1) // identity value for inactive lanes
6547 .addReg(RegNo: UndefExec); // bool i1
6548 };
6549 auto BuildDPPMachineInstr = [&](Register Dst, Register Src,
6550 unsigned DPPCtrl) {
6551 auto DPPInstr =
6552 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: DPPOpc), DestReg: Dst).addReg(RegNo: Src); // old
6553 if (isFPOp && !NeedsMovDPP)
6554 DPPInstr.addImm(Val: SISrcMods::NONE); // src0 modifier
6555 DPPInstr.addReg(RegNo: Src); // src0
6556 if (isFPOp && !NeedsMovDPP)
6557 DPPInstr.addImm(Val: SISrcMods::NONE); // src1 modifier
6558 if (!NeedsMovDPP)
6559 DPPInstr.addReg(RegNo: Src); // src1
6560 if (AMDGPU::getNamedOperandIdx(Opcode: DPPOpc, Name: AMDGPU::OpName::clamp) >= 0)
6561 DPPInstr.addImm(Val: 0); // clamp
6562 DPPInstr
6563 .addImm(Val: DPPCtrl) // dpp-ctrl
6564 .addImm(Val: 0xf) // row-mask
6565 .addImm(Val: 0xf) // bank-mask
6566 .addImm(Val: 0); // bound-control
6567 };
6568 auto BuildClampInstr = [&](Register Dst, Register Src0, Register Src1,
6569 bool isAddSub = false,
6570 bool needsCarryIn = false,
6571 Register CarryIn = Register()) {
6572 unsigned InstrOpc = ClampOpc;
6573 Register CarryOutReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6574 if (needsCarryIn)
6575 InstrOpc = AMDGPU::V_ADDC_U32_e64;
6576 auto ClampInstr = BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: InstrOpc), DestReg: Dst);
6577 if (isFPOp)
6578 ClampInstr.addImm(Val: SISrcMods::NONE); // src0 mod
6579 if (isAddSub) {
6580 if (needsCarryIn)
6581 ClampInstr.addReg(RegNo: CarryOutReg,
6582 Flags: RegState::Define |
6583 RegState::Dead); // killed carry-out reg
6584 else
6585 ClampInstr.addReg(RegNo: CarryOutReg, Flags: RegState::Define); // carry-out reg
6586 }
6587 ClampInstr.addReg(RegNo: Src0); // src0
6588 if (isFPOp)
6589 ClampInstr.addImm(Val: SISrcMods::NONE); // src1 mod
6590 ClampInstr.addReg(RegNo: Src1); // src1
6591 if (needsCarryIn)
6592 ClampInstr.addReg(RegNo: CarryIn, Flags: RegState::Kill); // carry-in reg
6593 if (AMDGPU::getNamedOperandIdx(Opcode: InstrOpc, Name: AMDGPU::OpName::clamp) >= 0)
6594 ClampInstr.addImm(Val: 0); // clamp
6595 if (isFPOp)
6596 ClampInstr.addImm(Val: 0); // omod
6597 LastBcastInstr = ClampInstr;
6598 return CarryOutReg;
6599 };
6600 auto BuildPostDPPInstr = [&](Register Src0, Register Src1) {
6601 bool isAddSubOpc =
6602 Opc == AMDGPU::S_ADD_U64_PSEUDO || Opc == AMDGPU::S_SUB_U64_PSEUDO;
6603 bool isBitWiseOpc = Opc == AMDGPU::S_AND_B64 ||
6604 Opc == AMDGPU::S_OR_B64 || Opc == AMDGPU::S_XOR_B64;
6605 Register ReturnReg = MRI.createVirtualRegister(RegClass: SrcRegClass);
6606 if (isAddSubOpc || isBitWiseOpc) {
6607 Register ResLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6608 Register ResHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6609 MachineOperand Src0Operand =
6610 MachineOperand::CreateReg(Reg: Src0, /*isDef=*/false);
6611 MachineOperand Src1Operand =
6612 MachineOperand::CreateReg(Reg: Src1, /*isDef=*/false);
6613 auto [Src0Lo, Src0Hi] =
6614 ExtractSubRegs(MI, Op&: Src0Operand, SrcRC: SrcRegClass, ST, MRI);
6615 auto [Src1Lo, Src1Hi] =
6616 ExtractSubRegs(MI, Op&: Src1Operand, SrcRC: SrcRegClass, ST, MRI);
6617 Register CarryReg = BuildClampInstr(
6618 ResLo, Src0Lo, Src1Lo, isAddSubOpc, /*needsCarryIn*/ false);
6619 BuildClampInstr(ResHi, Src0Hi, Src1Hi, isAddSubOpc,
6620 /*needsCarryIn*/ isAddSubOpc, CarryReg);
6621 BuildRegSequence(*CurrBB, MI, ReturnReg, ResLo, ResHi);
6622 } else {
6623 if (isFPOp) {
6624 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: ReturnReg)
6625 .addImm(Val: SISrcMods::NONE) // src0 modifiers
6626 .addReg(RegNo: Src0)
6627 .addImm(Val: SISrcMods::NONE) // src1 modifiers
6628 .addReg(RegNo: Src1)
6629 .addImm(Val: SISrcMods::NONE) // clamp
6630 .addImm(Val: SISrcMods::NONE); // omod
6631 } else {
6632 Register CmpMaskReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6633 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: CmpMaskReg)
6634 .addReg(RegNo: Src0) // src0
6635 .addReg(RegNo: Src1); // src1
6636 LastBcastInstr =
6637 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B64_PSEUDO),
6638 DestReg: ReturnReg)
6639 .addReg(RegNo: Src1) // src0
6640 .addReg(RegNo: Src0) // src1
6641 .addReg(RegNo: CmpMaskReg); // src2
6642 expand64BitV_CNDMASK(MI&: *LastBcastInstr, BB: CurrBB);
6643 }
6644 }
6645 return ReturnReg;
6646 };
6647
6648 // Set inactive lanes to the identity value.
6649 if (is32BitOpc) {
6650 SrcWithIdentityInstr =
6651 BuildSetInactiveInstr(SrcWithIdentity, SrcReg, IdentityVGPR);
6652 } else {
6653 Register SrcWithIdentitylo =
6654 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6655 Register SrcWithIdentityhi =
6656 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6657 auto [Reg0Sub0, Reg0Sub1] = ExtractSubRegs(
6658 MI, Op&: IdentityCopyInstr->getOperand(i: 0), SrcRC: SrcRegClass, ST, MRI);
6659 auto [SrcReg0Sub0, SrcReg0Sub1] =
6660 ExtractSubRegs(MI, Op&: MI.getOperand(i: 1), SrcRC: SrcRegClass, ST, MRI);
6661 MachineInstr *SetInactiveLoInstr =
6662 BuildSetInactiveInstr(SrcWithIdentitylo, SrcReg0Sub0, Reg0Sub0);
6663 MachineInstr *SetInactiveHiInstr =
6664 BuildSetInactiveInstr(SrcWithIdentityhi, SrcReg0Sub1, Reg0Sub1);
6665 SrcWithIdentityInstr =
6666 BuildRegSequence(*CurrBB, MI, SrcWithIdentity,
6667 SetInactiveLoInstr->getOperand(i: 0).getReg(),
6668 SetInactiveHiInstr->getOperand(i: 0).getReg());
6669 }
6670 // DPP reduction
6671 Register SrcWithIdentityReg =
6672 SrcWithIdentityInstr->getOperand(i: 0).getReg();
6673 BuildDPPMachineInstr(DPPRowShr1, SrcWithIdentityReg,
6674 AMDGPU::DPP::ROW_SHR_FIRST);
6675 if (NeedsMovDPP)
6676 DPPRowShr1 = BuildPostDPPInstr(SrcWithIdentityReg, DPPRowShr1);
6677
6678 BuildDPPMachineInstr(DPPRowShr2, DPPRowShr1,
6679 (AMDGPU::DPP::ROW_SHR_FIRST + 1));
6680 if (NeedsMovDPP)
6681 DPPRowShr2 = BuildPostDPPInstr(DPPRowShr1, DPPRowShr2);
6682
6683 BuildDPPMachineInstr(DPPRowShr4, DPPRowShr2,
6684 (AMDGPU::DPP::ROW_SHR_FIRST + 3));
6685 if (NeedsMovDPP)
6686 DPPRowShr4 = BuildPostDPPInstr(DPPRowShr2, DPPRowShr4);
6687
6688 BuildDPPMachineInstr(DPPRowShr8, DPPRowShr4,
6689 (AMDGPU::DPP::ROW_SHR_FIRST + 7));
6690 if (NeedsMovDPP)
6691 DPPRowShr8 = BuildPostDPPInstr(DPPRowShr4, DPPRowShr8);
6692
6693 if (ST.hasDPPBroadcasts()) {
6694 BuildDPPMachineInstr(RowBcast15, DPPRowShr8, AMDGPU::DPP::BCAST15);
6695 if (NeedsMovDPP)
6696 RowBcast15 = BuildPostDPPInstr(DPPRowShr8, RowBcast15);
6697 } else {
6698 // magic constant: 0x1E0
6699 // To Set BIT_MODE : bit 15 = 0
6700 // XOR mask : bit [14:10] = 0
6701 // OR mask : bit [9:5] = 15
6702 // AND mask : bit [4:0] = 0
6703 if (is32BitOpc) {
6704 Register SwizzledValue =
6705 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6706 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_SWIZZLE_B32),
6707 DestReg: SwizzledValue)
6708 .addReg(RegNo: DPPRowShr8) // addr
6709 .addImm(Val: 0x1E0) // swizzle offset (i16)
6710 .addImm(Val: 0x0); // gds (i1)
6711 BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue);
6712 } else {
6713 Register SwizzledValuelo =
6714 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6715 Register SwizzledValuehi =
6716 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6717 Register SwizzledValue64 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6718 MachineOperand DPPRowShr8Op =
6719 MachineOperand::CreateReg(Reg: DPPRowShr8, /*isDef=*/false);
6720 auto [Op1L, Op1H] =
6721 ExtractSubRegs(MI, Op&: DPPRowShr8Op, SrcRC: SrcRegClass, ST, MRI);
6722 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_SWIZZLE_B32),
6723 DestReg: SwizzledValuelo)
6724 .addReg(RegNo: Op1L) // addr
6725 .addImm(Val: 0x1E0) // swizzle offset (i16)
6726 .addImm(Val: 0x0); // gds (i1)
6727 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_SWIZZLE_B32),
6728 DestReg: SwizzledValuehi)
6729 .addReg(RegNo: Op1H) // addr
6730 .addImm(Val: 0x1E0) // swizzle offset (i16)
6731 .addImm(Val: 0x0); // gds (i1)
6732 BuildRegSequence(*CurrBB, MI, SwizzledValue64, SwizzledValuelo,
6733 SwizzledValuehi);
6734 if (NeedsMovDPP)
6735 RowBcast15 = BuildPostDPPInstr(DPPRowShr8, SwizzledValue64);
6736 else
6737 BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue64);
6738 }
6739 }
6740 FinalDPPResult = RowBcast15;
6741 if (!IsWave32) {
6742 if (ST.hasDPPBroadcasts()) {
6743 BuildDPPMachineInstr(RowBcast31, RowBcast15, AMDGPU::DPP::BCAST31);
6744 if (NeedsMovDPP)
6745 RowBcast31 = BuildPostDPPInstr(RowBcast15, RowBcast31);
6746 } else {
6747 Register ShiftedThreadID =
6748 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6749 Register PermuteByteOffset =
6750 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6751 Register PermutedValue = MRI.createVirtualRegister(RegClass: SrcRegClass);
6752 Register Lane32Offset =
6753 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6754 Register WordSizeConst =
6755 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6756 Register ThreadIDRegLo =
6757 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6758 Register ThreadIDReg =
6759 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6760 // Get the thread ID.
6761 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MBCNT_LO_U32_B32_e64),
6762 DestReg: ThreadIDRegLo)
6763 .addImm(Val: -1)
6764 .addImm(Val: 0);
6765 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MBCNT_HI_U32_B32_e64),
6766 DestReg: ThreadIDReg)
6767 .addImm(Val: -1)
6768 .addReg(RegNo: ThreadIDRegLo);
6769 // shift each lane over by 32 positions, so value in 31st lane is
6770 // present in 63rd lane.
6771 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: Lane32Offset)
6772 .addImm(Val: 0x20);
6773 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ADD_U32_e64),
6774 DestReg: ShiftedThreadID)
6775 .addReg(RegNo: ThreadIDReg)
6776 .addReg(RegNo: Lane32Offset)
6777 .addImm(Val: 0); // clamp
6778 // multiply by reg size.
6779 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: WordSizeConst)
6780 .addImm(Val: 0x4);
6781 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MUL_LO_U32_e64),
6782 DestReg: PermuteByteOffset)
6783 .addReg(RegNo: WordSizeConst)
6784 .addReg(RegNo: ShiftedThreadID);
6785 // Permute the lanes
6786 if (is32BitOpc) {
6787 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_PERMUTE_B32),
6788 DestReg: PermutedValue)
6789 .addReg(RegNo: PermuteByteOffset) // addr
6790 .addReg(RegNo: RowBcast15) // data
6791 .addImm(Val: 0); // offset
6792 } else {
6793 Register PermutedValuelo =
6794 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6795 Register PermutedValuehi =
6796 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6797 MachineOperand RowBcast15Op =
6798 MachineOperand::CreateReg(Reg: RowBcast15, /*isDef=*/false);
6799 auto [RowBcast15Lo, RowBcast15Hi] =
6800 ExtractSubRegs(MI, Op&: RowBcast15Op, SrcRC: SrcRegClass, ST, MRI);
6801 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_PERMUTE_B32),
6802 DestReg: PermutedValuelo)
6803 .addReg(RegNo: PermuteByteOffset) // addr
6804 .addReg(RegNo: RowBcast15Lo) // swizzle offset (i16)
6805 .addImm(Val: 0x0); // gds (i1)
6806 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_PERMUTE_B32),
6807 DestReg: PermutedValuehi)
6808 .addReg(RegNo: PermuteByteOffset) // addr
6809 .addReg(RegNo: RowBcast15Hi) // swizzle offset (i16)
6810 .addImm(Val: 0x0); // gds (i1)
6811 BuildRegSequence(*CurrBB, MI, PermutedValue, PermutedValuelo,
6812 PermutedValuehi);
6813 }
6814 if (NeedsMovDPP)
6815 RowBcast31 = BuildPostDPPInstr(RowBcast15, PermutedValue);
6816 else
6817 BuildClampInstr(RowBcast31, RowBcast15, PermutedValue);
6818 }
6819 FinalDPPResult = RowBcast31;
6820 }
6821 if (MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32 ||
6822 MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64) {
6823 Register NegatedValVGPR = MRI.createVirtualRegister(RegClass: SrcRegClass);
6824 // Opc for f32 reduction is V_SUB_F32.
6825 // For f64, there is no equivalent V_SUB_F64 opcode, so use
6826 // V_ADD_F64/V_ADD_F64_pseudo, and negate the second operand.
6827 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc),
6828 DestReg: NegatedValVGPR)
6829 .addImm(Val: SISrcMods::NONE) // src0 mods
6830 .addReg(RegNo: IdentityVGPR) // src0
6831 .addImm(Val: is32BitOpc ? SISrcMods::NONE : SISrcMods::NEG) // src1 mods
6832 .addReg(RegNo: IsWave32 ? RowBcast15 : RowBcast31) // src1
6833 .addImm(Val: SISrcMods::NONE) // clamp
6834 .addImm(Val: SISrcMods::NONE); // omod
6835 FinalDPPResult = NegatedValVGPR;
6836 }
6837 // The final reduced value is in the last lane.
6838 if (is32BitOpc) {
6839 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6840 DestReg: ReducedValSGPR)
6841 .addReg(RegNo: FinalDPPResult)
6842 .addImm(Val: ST.getWavefrontSize() - 1);
6843 } else {
6844 Register LaneValueLoReg =
6845 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6846 Register LaneValueHiReg =
6847 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6848 const TargetRegisterClass *SrcRC = MRI.getRegClass(Reg: SrcReg);
6849 MachineOperand FinalDPPResultOperand =
6850 MachineOperand::CreateReg(Reg: FinalDPPResult, /*isDef=*/false);
6851 auto [Op1L, Op1H] =
6852 ExtractSubRegs(MI, Op&: FinalDPPResultOperand, SrcRC, ST, MRI);
6853 // lane value input should be in an sgpr
6854 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6855 DestReg: LaneValueLoReg)
6856 .addReg(RegNo: Op1L)
6857 .addImm(Val: ST.getWavefrontSize() - 1);
6858 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6859 DestReg: LaneValueHiReg)
6860 .addReg(RegNo: Op1H)
6861 .addImm(Val: ST.getWavefrontSize() - 1);
6862 BuildRegSequence(*CurrBB, MI, ReducedValSGPR, LaneValueLoReg,
6863 LaneValueHiReg);
6864 }
6865 if (Opc == AMDGPU::S_SUB_I32) {
6866 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedReducedVal)
6867 .addImm(Val: 0)
6868 .addReg(RegNo: ReducedValSGPR);
6869 } else if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6870 auto NegatedValInstr =
6871 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: NegatedReducedVal)
6872 .addImm(Val: 0)
6873 .addReg(RegNo: ReducedValSGPR);
6874 CurrBB = expand64BitScalarArithmetic(MI&: *NegatedValInstr, BB: CurrBB);
6875 }
6876 // Mark the final result as a whole-wave-mode calculation.
6877 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::STRICT_WWM), DestReg: DstReg)
6878 .addReg(RegNo: Opc == AMDGPU::S_SUB_I32 || Opc == AMDGPU::S_SUB_U64_PSEUDO
6879 ? NegatedReducedVal
6880 : ReducedValSGPR);
6881 RetBB = CurrBB;
6882 }
6883 }
6884 MI.eraseFromParent();
6885 return RetBB;
6886}
6887
6888MachineBasicBlock *
6889SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
6890 MachineBasicBlock *BB) const {
6891 MachineFunction *MF = BB->getParent();
6892 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
6893 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
6894 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
6895 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
6896 MachineRegisterInfo &MRI = MF->getRegInfo();
6897 const DebugLoc &DL = MI.getDebugLoc();
6898
6899 switch (MI.getOpcode()) {
6900 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U32:
6901 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MIN_U32);
6902 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U64:
6903 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_LT_U64_e64);
6904 case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I32:
6905 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MIN_I32);
6906 case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I64:
6907 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_LT_I64_e64);
6908 case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F32:
6909 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_MIN_F32_e64);
6910 case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F64:
6911 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6912 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6913 ? AMDGPU::V_MIN_NUM_F64_e64
6914 : AMDGPU::V_MIN_F64_e64);
6915 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U32:
6916 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MAX_U32);
6917 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U64:
6918 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_GT_U64_e64);
6919 case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I32:
6920 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MAX_I32);
6921 case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I64:
6922 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_GT_I64_e64);
6923 case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F32:
6924 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_MAX_F32_e64);
6925 case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F64:
6926 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6927 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6928 ? AMDGPU::V_MAX_NUM_F64_e64
6929 : AMDGPU::V_MAX_F64_e64);
6930 case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_I32:
6931 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_ADD_I32);
6932 case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_U64:
6933 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_ADD_U64_PSEUDO);
6934 case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F32:
6935 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_ADD_F32_e64);
6936 case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F64:
6937 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6938 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6939 ? AMDGPU::V_ADD_F64_pseudo_e64
6940 : AMDGPU::V_ADD_F64_e64);
6941 case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_I32:
6942 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_SUB_I32);
6943 case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_U64:
6944 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_SUB_U64_PSEUDO);
6945 case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32:
6946 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_SUB_F32_e64);
6947 case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64:
6948 // There is no S/V_SUB_F64 opcode. Double type subtraction is expanded as
6949 // fadd + neg, by setting the NEG bit in the instruction.
6950 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6951 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6952 ? AMDGPU::V_ADD_F64_pseudo_e64
6953 : AMDGPU::V_ADD_F64_e64);
6954 case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B32:
6955 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_AND_B32);
6956 case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B64:
6957 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_AND_B64);
6958 case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B32:
6959 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_OR_B32);
6960 case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B64:
6961 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_OR_B64);
6962 case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B32:
6963 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_XOR_B32);
6964 case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B64:
6965 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_XOR_B64);
6966 case AMDGPU::S_UADDO_PSEUDO:
6967 case AMDGPU::S_USUBO_PSEUDO: {
6968 MachineOperand &Dest0 = MI.getOperand(i: 0);
6969 MachineOperand &Dest1 = MI.getOperand(i: 1);
6970 MachineOperand &Src0 = MI.getOperand(i: 2);
6971 MachineOperand &Src1 = MI.getOperand(i: 3);
6972
6973 unsigned Opc = (MI.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
6974 ? AMDGPU::S_ADD_U32
6975 : AMDGPU::S_SUB_U32;
6976 // clang-format off
6977 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest0.getReg())
6978 .add(MO: Src0)
6979 .add(MO: Src1);
6980 // clang-format on
6981
6982 unsigned SelOpc =
6983 Subtarget->isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
6984 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SelOpc), DestReg: Dest1.getReg()).addImm(Val: -1).addImm(Val: 0);
6985
6986 MI.eraseFromParent();
6987 return BB;
6988 }
6989 case AMDGPU::S_ADD_U64_PSEUDO:
6990 case AMDGPU::S_SUB_U64_PSEUDO: {
6991 return expand64BitScalarArithmetic(MI, BB);
6992 }
6993 case AMDGPU::V_ADD_U64_PSEUDO:
6994 case AMDGPU::V_SUB_U64_PSEUDO: {
6995 bool IsAdd = (MI.getOpcode() == AMDGPU::V_ADD_U64_PSEUDO);
6996
6997 MachineOperand &Dest = MI.getOperand(i: 0);
6998 MachineOperand &Src0 = MI.getOperand(i: 1);
6999 MachineOperand &Src1 = MI.getOperand(i: 2);
7000
7001 if (ST.hasAddSubU64Insts()) {
7002 auto I = BuildMI(BB&: *BB, I&: MI, MIMD: DL,
7003 MCID: TII->get(Opcode: IsAdd ? AMDGPU::V_ADD_U64_e64
7004 : AMDGPU::V_SUB_U64_e64),
7005 DestReg: Dest.getReg())
7006 .add(MO: Src0)
7007 .add(MO: Src1)
7008 .addImm(Val: 0); // clamp
7009 TII->legalizeOperands(MI&: *I);
7010 MI.eraseFromParent();
7011 return BB;
7012 }
7013
7014 if (IsAdd && ST.hasLshlAddU64Inst()) {
7015 auto Add = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_LSHL_ADD_U64_e64),
7016 DestReg: Dest.getReg())
7017 .add(MO: Src0)
7018 .addImm(Val: 0)
7019 .add(MO: Src1);
7020 TII->legalizeOperands(MI&: *Add);
7021 MI.eraseFromParent();
7022 return BB;
7023 }
7024
7025 const auto *CarryRC = TRI->getWaveMaskRegClass();
7026
7027 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7028 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7029
7030 Register CarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
7031 Register DeadCarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
7032
7033 const TargetRegisterClass *Src0RC = Src0.isReg()
7034 ? MRI.getRegClass(Reg: Src0.getReg())
7035 : &AMDGPU::VReg_64RegClass;
7036 const TargetRegisterClass *Src1RC = Src1.isReg()
7037 ? MRI.getRegClass(Reg: Src1.getReg())
7038 : &AMDGPU::VReg_64RegClass;
7039
7040 const TargetRegisterClass *Src0SubRC =
7041 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
7042 const TargetRegisterClass *Src1SubRC =
7043 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
7044
7045 MachineOperand SrcReg0Sub0 = TII->buildExtractSubRegOrImm(
7046 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
7047 MachineOperand SrcReg1Sub0 = TII->buildExtractSubRegOrImm(
7048 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
7049
7050 MachineOperand SrcReg0Sub1 = TII->buildExtractSubRegOrImm(
7051 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
7052 MachineOperand SrcReg1Sub1 = TII->buildExtractSubRegOrImm(
7053 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
7054
7055 unsigned LoOpc =
7056 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
7057 MachineInstr *LoHalf = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoOpc), DestReg: DestSub0)
7058 .addReg(RegNo: CarryReg, Flags: RegState::Define)
7059 .add(MO: SrcReg0Sub0)
7060 .add(MO: SrcReg1Sub0)
7061 .addImm(Val: 0); // clamp bit
7062
7063 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
7064 MachineInstr *HiHalf =
7065 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: HiOpc), DestReg: DestSub1)
7066 .addReg(RegNo: DeadCarryReg, Flags: RegState::Define | RegState::Dead)
7067 .add(MO: SrcReg0Sub1)
7068 .add(MO: SrcReg1Sub1)
7069 .addReg(RegNo: CarryReg, Flags: RegState::Kill)
7070 .addImm(Val: 0); // clamp bit
7071
7072 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest.getReg())
7073 .addReg(RegNo: DestSub0)
7074 .addImm(Val: AMDGPU::sub0)
7075 .addReg(RegNo: DestSub1)
7076 .addImm(Val: AMDGPU::sub1);
7077 TII->legalizeOperands(MI&: *LoHalf);
7078 TII->legalizeOperands(MI&: *HiHalf);
7079 MI.eraseFromParent();
7080 return BB;
7081 }
7082 case AMDGPU::S_ADD_CO_PSEUDO:
7083 case AMDGPU::S_SUB_CO_PSEUDO: {
7084 // This pseudo has a chance to be selected
7085 // only from uniform add/subcarry node. All the VGPR operands
7086 // therefore assumed to be splat vectors.
7087 MachineBasicBlock::iterator MII = MI;
7088 MachineOperand &Dest = MI.getOperand(i: 0);
7089 MachineOperand &CarryDest = MI.getOperand(i: 1);
7090 MachineOperand &Src0 = MI.getOperand(i: 2);
7091 MachineOperand &Src1 = MI.getOperand(i: 3);
7092 MachineOperand &Src2 = MI.getOperand(i: 4);
7093 if (Src0.isReg() && TRI->isVectorRegister(MRI, Reg: Src0.getReg())) {
7094 Register RegOp0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7095 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp0)
7096 .addReg(RegNo: Src0.getReg());
7097 Src0.setReg(RegOp0);
7098 }
7099 if (Src1.isReg() && TRI->isVectorRegister(MRI, Reg: Src1.getReg())) {
7100 Register RegOp1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7101 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp1)
7102 .addReg(RegNo: Src1.getReg());
7103 Src1.setReg(RegOp1);
7104 }
7105 Register RegOp2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7106 if (TRI->isVectorRegister(MRI, Reg: Src2.getReg())) {
7107 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp2)
7108 .addReg(RegNo: Src2.getReg());
7109 Src2.setReg(RegOp2);
7110 }
7111
7112 if (ST.isWave64()) {
7113 if (ST.hasScalarCompareEq64()) {
7114 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U64))
7115 .addReg(RegNo: Src2.getReg())
7116 .addImm(Val: 0);
7117 } else {
7118 const TargetRegisterClass *Src2RC = MRI.getRegClass(Reg: Src2.getReg());
7119 const TargetRegisterClass *SubRC =
7120 TRI->getSubRegisterClass(Src2RC, AMDGPU::sub0);
7121 MachineOperand Src2Sub0 = TII->buildExtractSubRegOrImm(
7122 MI: MII, MRI, SuperReg: Src2, SuperRC: Src2RC, SubIdx: AMDGPU::sub0, SubRC);
7123 MachineOperand Src2Sub1 = TII->buildExtractSubRegOrImm(
7124 MI: MII, MRI, SuperReg: Src2, SuperRC: Src2RC, SubIdx: AMDGPU::sub1, SubRC);
7125 Register Src2_32 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7126
7127 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_OR_B32), DestReg: Src2_32)
7128 .add(MO: Src2Sub0)
7129 .add(MO: Src2Sub1);
7130
7131 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
7132 .addReg(RegNo: Src2_32, Flags: RegState::Kill)
7133 .addImm(Val: 0);
7134 }
7135 } else {
7136 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
7137 .addReg(RegNo: Src2.getReg())
7138 .addImm(Val: 0);
7139 }
7140
7141 unsigned Opc = MI.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO
7142 ? AMDGPU::S_ADDC_U32
7143 : AMDGPU::S_SUBB_U32;
7144
7145 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest.getReg()).add(MO: Src0).add(MO: Src1);
7146
7147 unsigned SelOpc =
7148 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
7149
7150 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: SelOpc), DestReg: CarryDest.getReg())
7151 .addImm(Val: -1)
7152 .addImm(Val: 0);
7153
7154 MI.eraseFromParent();
7155 return BB;
7156 }
7157 case AMDGPU::SI_INIT_M0: {
7158 MachineOperand &M0Init = MI.getOperand(i: 0);
7159 BuildMI(BB&: *BB, I: MI.getIterator(), MIMD: MI.getDebugLoc(),
7160 MCID: TII->get(Opcode: M0Init.isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32),
7161 DestReg: AMDGPU::M0)
7162 .add(MO: M0Init);
7163 MI.eraseFromParent();
7164 return BB;
7165 }
7166 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM: {
7167 // Set SCC to true, in case the barrier instruction gets converted to a NOP.
7168 BuildMI(BB&: *BB, I: MI.getIterator(), MIMD: MI.getDebugLoc(),
7169 MCID: TII->get(Opcode: AMDGPU::S_CMP_EQ_U32))
7170 .addImm(Val: 0)
7171 .addImm(Val: 0);
7172 return BB;
7173 }
7174 case AMDGPU::GET_GROUPSTATICSIZE: {
7175 assert(getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA ||
7176 getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL);
7177 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32))
7178 .add(MO: MI.getOperand(i: 0))
7179 .addImm(Val: MFI->getLDSSize());
7180 MI.eraseFromParent();
7181 return BB;
7182 }
7183 case AMDGPU::GET_SHADERCYCLESHILO: {
7184 assert(MF->getSubtarget<GCNSubtarget>().hasShaderCyclesHiLoRegisters());
7185 // The algorithm is:
7186 //
7187 // hi1 = getreg(SHADER_CYCLES_HI)
7188 // lo1 = getreg(SHADER_CYCLES_LO)
7189 // hi2 = getreg(SHADER_CYCLES_HI)
7190 //
7191 // If hi1 == hi2 then there was no overflow and the result is hi2:lo1.
7192 // Otherwise there was overflow and the result is hi2:0. In both cases the
7193 // result should represent the actual time at some point during the sequence
7194 // of three getregs.
7195 using namespace AMDGPU::Hwreg;
7196 Register RegHi1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7197 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegHi1)
7198 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES_HI, Values: 0, Values: 32));
7199 Register RegLo1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7200 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegLo1)
7201 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES, Values: 0, Values: 32));
7202 Register RegHi2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7203 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegHi2)
7204 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES_HI, Values: 0, Values: 32));
7205 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_EQ_U32))
7206 .addReg(RegNo: RegHi1)
7207 .addReg(RegNo: RegHi2);
7208 Register RegLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7209 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: RegLo)
7210 .addReg(RegNo: RegLo1)
7211 .addImm(Val: 0);
7212 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE))
7213 .add(MO: MI.getOperand(i: 0))
7214 .addReg(RegNo: RegLo)
7215 .addImm(Val: AMDGPU::sub0)
7216 .addReg(RegNo: RegHi2)
7217 .addImm(Val: AMDGPU::sub1);
7218 MI.eraseFromParent();
7219 return BB;
7220 }
7221 case AMDGPU::SI_INDIRECT_SRC_V1:
7222 case AMDGPU::SI_INDIRECT_SRC_V2:
7223 case AMDGPU::SI_INDIRECT_SRC_V3:
7224 case AMDGPU::SI_INDIRECT_SRC_V4:
7225 case AMDGPU::SI_INDIRECT_SRC_V5:
7226 case AMDGPU::SI_INDIRECT_SRC_V6:
7227 case AMDGPU::SI_INDIRECT_SRC_V7:
7228 case AMDGPU::SI_INDIRECT_SRC_V8:
7229 case AMDGPU::SI_INDIRECT_SRC_V9:
7230 case AMDGPU::SI_INDIRECT_SRC_V10:
7231 case AMDGPU::SI_INDIRECT_SRC_V11:
7232 case AMDGPU::SI_INDIRECT_SRC_V12:
7233 case AMDGPU::SI_INDIRECT_SRC_V16:
7234 case AMDGPU::SI_INDIRECT_SRC_V32:
7235 return emitIndirectSrc(MI, MBB&: *BB, ST: *getSubtarget());
7236 case AMDGPU::SI_INDIRECT_DST_V1:
7237 case AMDGPU::SI_INDIRECT_DST_V2:
7238 case AMDGPU::SI_INDIRECT_DST_V3:
7239 case AMDGPU::SI_INDIRECT_DST_V4:
7240 case AMDGPU::SI_INDIRECT_DST_V5:
7241 case AMDGPU::SI_INDIRECT_DST_V6:
7242 case AMDGPU::SI_INDIRECT_DST_V7:
7243 case AMDGPU::SI_INDIRECT_DST_V8:
7244 case AMDGPU::SI_INDIRECT_DST_V9:
7245 case AMDGPU::SI_INDIRECT_DST_V10:
7246 case AMDGPU::SI_INDIRECT_DST_V11:
7247 case AMDGPU::SI_INDIRECT_DST_V12:
7248 case AMDGPU::SI_INDIRECT_DST_V16:
7249 case AMDGPU::SI_INDIRECT_DST_V32:
7250 return emitIndirectDst(MI, MBB&: *BB, ST: *getSubtarget());
7251 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
7252 case AMDGPU::SI_KILL_I1_PSEUDO:
7253 return splitKillBlock(MI, BB);
7254 case AMDGPU::V_CNDMASK_B64_PSEUDO: {
7255 expand64BitV_CNDMASK(MI, BB);
7256 return BB;
7257 }
7258 case AMDGPU::SI_BR_UNDEF: {
7259 MachineInstr *Br = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
7260 .add(MO: MI.getOperand(i: 0));
7261 Br->getOperand(i: 1).setIsUndef(); // read undef SCC
7262 MI.eraseFromParent();
7263 return BB;
7264 }
7265 case AMDGPU::ADJCALLSTACKUP:
7266 case AMDGPU::ADJCALLSTACKDOWN: {
7267 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
7268 MachineInstrBuilder MIB(*MF, &MI);
7269 MIB.addReg(RegNo: Info->getStackPtrOffsetReg(), Flags: RegState::ImplicitDefine)
7270 .addReg(RegNo: Info->getStackPtrOffsetReg(), Flags: RegState::Implicit);
7271 return BB;
7272 }
7273 case AMDGPU::SI_CALL_ISEL: {
7274 unsigned ReturnAddrReg = TII->getRegisterInfo().getReturnAddressReg(MF: *MF);
7275
7276 MachineInstrBuilder MIB;
7277 MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_CALL), DestReg: ReturnAddrReg);
7278
7279 for (const MachineOperand &MO : MI.operands())
7280 MIB.add(MO);
7281
7282 MIB.cloneMemRefs(OtherMI: MI);
7283 MI.eraseFromParent();
7284 return BB;
7285 }
7286 case AMDGPU::V_ADD_CO_U32_e32:
7287 case AMDGPU::V_SUB_CO_U32_e32:
7288 case AMDGPU::V_SUBREV_CO_U32_e32: {
7289 // TODO: Define distinct V_*_I32_Pseudo instructions instead.
7290 unsigned Opc = MI.getOpcode();
7291
7292 bool NeedClampOperand = false;
7293 if (TII->pseudoToMCOpcode(Opcode: Opc) == -1) {
7294 Opc = AMDGPU::getVOPe64(Opcode: Opc);
7295 NeedClampOperand = true;
7296 }
7297
7298 auto I = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: MI.getOperand(i: 0).getReg());
7299 if (TII->isVOP3(MI: *I)) {
7300 I.addReg(RegNo: TRI->getVCC(), Flags: RegState::Define);
7301 }
7302 I.add(MO: MI.getOperand(i: 1)).add(MO: MI.getOperand(i: 2));
7303 if (NeedClampOperand)
7304 I.addImm(Val: 0); // clamp bit for e64 encoding
7305
7306 TII->legalizeOperands(MI&: *I);
7307
7308 MI.eraseFromParent();
7309 return BB;
7310 }
7311 case AMDGPU::V_ADDC_U32_e32:
7312 case AMDGPU::V_SUBB_U32_e32:
7313 case AMDGPU::V_SUBBREV_U32_e32:
7314 // These instructions have an implicit use of vcc which counts towards the
7315 // constant bus limit.
7316 TII->legalizeOperands(MI);
7317 return BB;
7318 case AMDGPU::DS_GWS_INIT:
7319 case AMDGPU::DS_GWS_SEMA_BR:
7320 case AMDGPU::DS_GWS_BARRIER:
7321 case AMDGPU::DS_GWS_SEMA_V:
7322 case AMDGPU::DS_GWS_SEMA_P:
7323 case AMDGPU::DS_GWS_SEMA_RELEASE_ALL:
7324 // A s_waitcnt 0 is required to be the instruction immediately following.
7325 if (getSubtarget()->hasGWSAutoReplay()) {
7326 bundleInstWithWaitcnt(MI);
7327 return BB;
7328 }
7329
7330 return emitGWSMemViolTestLoop(MI, BB);
7331 case AMDGPU::S_SETREG_B32: {
7332 // Try to optimize cases that only set the denormal mode or rounding mode.
7333 //
7334 // If the s_setreg_b32 fully sets all of the bits in the rounding mode or
7335 // denormal mode to a constant, we can use s_round_mode or s_denorm_mode
7336 // instead.
7337 //
7338 // FIXME: This could be predicates on the immediate, but tablegen doesn't
7339 // allow you to have a no side effect instruction in the output of a
7340 // sideeffecting pattern.
7341 auto [ID, Offset, Width] =
7342 AMDGPU::Hwreg::HwregEncoding::decode(Encoded: MI.getOperand(i: 1).getImm());
7343 if (ID != AMDGPU::Hwreg::ID_MODE)
7344 return BB;
7345
7346 const unsigned WidthMask = maskTrailingOnes<unsigned>(N: Width);
7347 const unsigned SetMask = WidthMask << Offset;
7348
7349 if (getSubtarget()->hasDenormModeInst()) {
7350 unsigned SetDenormOp = 0;
7351 unsigned SetRoundOp = 0;
7352
7353 // The dedicated instructions can only set the whole denorm or round mode
7354 // at once, not a subset of bits in either.
7355 if (SetMask ==
7356 (AMDGPU::Hwreg::FP_ROUND_MASK | AMDGPU::Hwreg::FP_DENORM_MASK)) {
7357 // If this fully sets both the round and denorm mode, emit the two
7358 // dedicated instructions for these.
7359 SetRoundOp = AMDGPU::S_ROUND_MODE;
7360 SetDenormOp = AMDGPU::S_DENORM_MODE;
7361 } else if (SetMask == AMDGPU::Hwreg::FP_ROUND_MASK) {
7362 SetRoundOp = AMDGPU::S_ROUND_MODE;
7363 } else if (SetMask == AMDGPU::Hwreg::FP_DENORM_MASK) {
7364 SetDenormOp = AMDGPU::S_DENORM_MODE;
7365 }
7366
7367 if (SetRoundOp || SetDenormOp) {
7368 MachineInstr *Def = MRI.getVRegDef(Reg: MI.getOperand(i: 0).getReg());
7369 if (Def && Def->isMoveImmediate() && Def->getOperand(i: 1).isImm()) {
7370 unsigned ImmVal = Def->getOperand(i: 1).getImm();
7371 if (SetRoundOp) {
7372 BuildMI(BB&: *BB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SetRoundOp))
7373 .addImm(Val: ImmVal & 0xf);
7374
7375 // If we also have the denorm mode, get just the denorm mode bits.
7376 ImmVal >>= 4;
7377 }
7378
7379 if (SetDenormOp) {
7380 BuildMI(BB&: *BB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SetDenormOp))
7381 .addImm(Val: ImmVal & 0xf);
7382 }
7383
7384 MI.eraseFromParent();
7385 return BB;
7386 }
7387 }
7388 }
7389
7390 // If only FP bits are touched, used the no side effects pseudo.
7391 if ((SetMask & (AMDGPU::Hwreg::FP_ROUND_MASK |
7392 AMDGPU::Hwreg::FP_DENORM_MASK)) == SetMask)
7393 MI.setDesc(TII->get(Opcode: AMDGPU::S_SETREG_B32_mode));
7394
7395 return BB;
7396 }
7397 case AMDGPU::S_INVERSE_BALLOT_U32:
7398 case AMDGPU::S_INVERSE_BALLOT_U64:
7399 // These opcodes only exist to let SIFixSGPRCopies insert a readfirstlane if
7400 // necessary. After that they are equivalent to a COPY.
7401 MI.setDesc(TII->get(Opcode: AMDGPU::COPY));
7402 return BB;
7403 case AMDGPU::ENDPGM_TRAP: {
7404 if (BB->succ_empty() && std::next(x: MI.getIterator()) == BB->end()) {
7405 MI.setDesc(TII->get(Opcode: AMDGPU::S_ENDPGM));
7406 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
7407 return BB;
7408 }
7409
7410 // We need a block split to make the real endpgm a terminator. We also don't
7411 // want to break phis in successor blocks, so we can't just delete to the
7412 // end of the block.
7413
7414 MachineBasicBlock *SplitBB = BB->splitAt(SplitInst&: MI, UpdateLiveIns: false /*UpdateLiveIns*/);
7415 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
7416 MF->push_back(MBB: TrapBB);
7417 // clang-format off
7418 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ENDPGM))
7419 .addImm(Val: 0);
7420 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_EXECNZ))
7421 .addMBB(MBB: TrapBB);
7422 // clang-format on
7423
7424 BB->addSuccessor(Succ: TrapBB);
7425 MI.eraseFromParent();
7426 return SplitBB;
7427 }
7428 case AMDGPU::SIMULATED_TRAP: {
7429 assert(Subtarget->hasPrivEnabledTrap2NopBug());
7430 MachineBasicBlock *SplitBB =
7431 TII->insertSimulatedTrap(MRI, MBB&: *BB, MI, DL: MI.getDebugLoc());
7432 MI.eraseFromParent();
7433 return SplitBB;
7434 }
7435 case AMDGPU::SI_TCRETURN_GFX_WholeWave:
7436 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN: {
7437 assert(MFI->isWholeWaveFunction());
7438
7439 // During ISel, it's difficult to propagate the original EXEC mask to use as
7440 // an input to SI_WHOLE_WAVE_FUNC_RETURN. Set it up here instead.
7441 MachineInstr *Setup = TII->getWholeWaveFunctionSetup(MF&: *BB->getParent());
7442 assert(Setup && "Couldn't find SI_SETUP_WHOLE_WAVE_FUNC");
7443 Register OriginalExec = Setup->getOperand(i: 0).getReg();
7444 MF->getRegInfo().clearKillFlags(Reg: OriginalExec);
7445 MI.getOperand(i: 0).setReg(OriginalExec);
7446 return BB;
7447 }
7448 case AMDGPU::V_DOT2_F32_F16:
7449 case AMDGPU::V_DOT2_F32_BF16: {
7450 // Hint RA to assign dst and src2 the same physical register.
7451 // For targets without VOP2, but with VOPD, variant of the instruction this
7452 // is one of the conditions to attempt converting VOP3P to VOPD.
7453 MRI.setSimpleHint(VReg: MI.getOperand(i: 0).getReg(), PrefReg: MI.getOperand(i: 6).getReg());
7454 return BB;
7455 }
7456 case AMDGPU::SCHED_BARRIER:
7457 case AMDGPU::SCHED_GROUP_BARRIER:
7458 MI.getOperand(i: 0).setImm(MI.getOperand(i: 0).getImm() &
7459 static_cast<unsigned>(AMDGPU::SchedGroupMask::ALL));
7460 return BB;
7461 default:
7462 if (TII->isImage(MI) || TII->isMUBUF(MI)) {
7463 if (!MI.mayStore())
7464 AddMemOpInit(MI);
7465 return BB;
7466 }
7467 return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, MBB: BB);
7468 }
7469}
7470
7471bool SITargetLowering::enableAggressiveFMAFusion(EVT VT) const {
7472 // This currently forces unfolding various combinations of fsub into fma with
7473 // free fneg'd operands. As long as we have fast FMA (controlled by
7474 // isFMAFasterThanFMulAndFAdd), we should perform these.
7475
7476 // When fma is quarter rate, for f64 where add / sub are at best half rate,
7477 // most of these combines appear to be cycle neutral but save on instruction
7478 // count / code size.
7479 return true;
7480}
7481
7482bool SITargetLowering::enableAggressiveFMAFusion(LLT Ty) const { return true; }
7483
7484EVT SITargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx,
7485 EVT VT) const {
7486 if (!VT.isVector()) {
7487 return MVT::i1;
7488 }
7489 return EVT::getVectorVT(Context&: Ctx, VT: MVT::i1, NumElements: VT.getVectorNumElements());
7490}
7491
7492MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT VT) const {
7493 // TODO: Should i16 be used always if legal? For now it would force VALU
7494 // shifts.
7495 return (VT == MVT::i16) ? MVT::i16 : MVT::i32;
7496}
7497
7498LLT SITargetLowering::getPreferredShiftAmountTy(LLT Ty) const {
7499 return (Ty.getScalarSizeInBits() <= 16 && Subtarget->has16BitInsts())
7500 ? Ty.changeElementSize(NewEltSize: 16)
7501 : Ty.changeElementSize(NewEltSize: 32);
7502}
7503
7504// Answering this is somewhat tricky and depends on the specific device which
7505// have different rates for fma or all f64 operations.
7506//
7507// v_fma_f64 and v_mul_f64 always take the same number of cycles as each other
7508// regardless of which device (although the number of cycles differs between
7509// devices), so it is always profitable for f64.
7510//
7511// v_fma_f32 takes 4 or 16 cycles depending on the device, so it is profitable
7512// only on full rate devices. Normally, we should prefer selecting v_mad_f32
7513// which we can always do even without fused FP ops since it returns the same
7514// result as the separate operations and since it is always full
7515// rate. Therefore, we lie and report that it is not faster for f32. v_mad_f32
7516// however does not support denormals, so we do report fma as faster if we have
7517// a fast fma device and require denormals.
7518//
7519bool SITargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT,
7520 DenormalFPEnv FPEnv) const {
7521 VT = VT.getScalarType();
7522 if (!VT.isSimple())
7523 return false;
7524
7525 switch (VT.getSimpleVT().SimpleTy) {
7526 case MVT::f32: {
7527 // If mad is not available this depends only on if f32 fma is full rate.
7528 if (!Subtarget->hasMadMacF32Insts())
7529 return Subtarget->hasFastFMAF32();
7530
7531 // Otherwise f32 mad is always full rate and returns the same result as
7532 // the separate operations so should be preferred over fma.
7533 // However does not support denormals.
7534 if (FPEnv.F32Mode != DenormalMode::getPreserveSign())
7535 return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts();
7536
7537 // If the subtarget has v_fmac_f32, that's just as good as v_mac_f32.
7538 return Subtarget->hasFastFMAF32() && Subtarget->hasDLInsts();
7539 }
7540 case MVT::f64:
7541 return true;
7542 case MVT::f16:
7543 case MVT::bf16:
7544 return Subtarget->has16BitInsts() &&
7545 FPEnv.DefaultMode != DenormalMode::getPreserveSign();
7546 default:
7547 break;
7548 }
7549
7550 return false;
7551}
7552
7553bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
7554 EVT VT) const {
7555 return isFMAFasterThanFMulAndFAdd(VT, FPEnv: getDenormalFPEnv(MF));
7556}
7557
7558bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
7559 Type *Ty) const {
7560 return isFMAFasterThanFMulAndFAdd(
7561 VT: getValueType(DL: F.getDataLayout(), Ty, /*AllowUnknown=*/true),
7562 FPEnv: F.getDenormalFPEnv());
7563}
7564
7565bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
7566 LLT Ty) const {
7567 switch (Ty.getScalarSizeInBits()) {
7568 case 16:
7569 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f16);
7570 case 32:
7571 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f32);
7572 case 64:
7573 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f64);
7574 default:
7575 break;
7576 }
7577
7578 return false;
7579}
7580
7581bool SITargetLowering::isFMADLegal(EVT VT, DenormalFPEnv FPEnv) const {
7582 // TODO: Check future ftz flag
7583 // v_mad_f32/v_mac_f32 do not support denormals.
7584 if (VT == MVT::f32)
7585 return Subtarget->hasMadMacF32Insts() &&
7586 FPEnv.F32Mode == DenormalMode::getPreserveSign();
7587 if (VT == MVT::f16)
7588 return Subtarget->hasMadF16() &&
7589 FPEnv.DefaultMode == DenormalMode::getPreserveSign();
7590
7591 return false;
7592}
7593
7594bool SITargetLowering::isFMADLegal(const MachineInstr &MI, LLT Ty) const {
7595 if (!Ty.isScalar())
7596 return false;
7597
7598 DenormalFPEnv FPEnv = getDenormalFPEnv(MF: *MI.getMF());
7599 if (Ty.getScalarSizeInBits() == 16)
7600 return isFMADLegal(VT: MVT::f16, FPEnv);
7601 if (Ty.getScalarSizeInBits() == 32)
7602 return isFMADLegal(VT: MVT::f32, FPEnv);
7603
7604 return false;
7605}
7606
7607bool SITargetLowering::isFMADLegal(const SelectionDAG &DAG,
7608 const SDNode *N) const {
7609 return isFMADLegal(VT: N->getValueType(ResNo: 0),
7610 FPEnv: getDenormalFPEnv(MF: DAG.getMachineFunction()));
7611}
7612
7613bool SITargetLowering::isFMADLegal(const Function &F, Type *Ty) const {
7614 return isFMADLegal(VT: getValueType(DL: F.getDataLayout(), Ty: Ty->getScalarType(),
7615 /*AllowUnknown=*/true),
7616 FPEnv: F.getDenormalFPEnv());
7617}
7618
7619//===----------------------------------------------------------------------===//
7620// Custom DAG Lowering Operations
7621//===----------------------------------------------------------------------===//
7622
7623// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
7624// wider vector type is legal.
7625SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op,
7626 SelectionDAG &DAG) const {
7627 unsigned Opc = Op.getOpcode();
7628 EVT VT = Op.getValueType();
7629 assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
7630
7631 auto [Lo, Hi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
7632 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
7633
7634 SDLoc SL(Op);
7635
7636 // Forward any trailing scalar operands unchanged to both halves.
7637 SmallVector<SDValue, 2> LoOps = {Lo};
7638 SmallVector<SDValue, 2> HiOps = {Hi};
7639 auto TrailingOps = drop_begin(RangeOrContainer: Op->ops());
7640 LoOps.append(in_start: TrailingOps.begin(), in_end: TrailingOps.end());
7641 HiOps.append(in_start: TrailingOps.begin(), in_end: TrailingOps.end());
7642
7643 SDValue OpLo = DAG.getNode(Opcode: Opc, DL: SL, VT: LoVT, Ops: LoOps, Flags: Op->getFlags());
7644 SDValue OpHi = DAG.getNode(Opcode: Opc, DL: SL, VT: HiVT, Ops: HiOps, Flags: Op->getFlags());
7645
7646 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7647}
7648
7649// Enable lowering of ROTR for vxi32 types. This is a workaround for a
7650// regression whereby extra unnecessary instructions were added to codegen
7651// for rotr operations, casued by legalising v2i32 or. This resulted in extra
7652// instructions to extract the result from the vector.
7653SDValue SITargetLowering::lowerROTR(SDValue Op, SelectionDAG &DAG) const {
7654 [[maybe_unused]] EVT VT = Op.getValueType();
7655
7656 assert((VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v8i32 ||
7657 VT == MVT::v16i32) &&
7658 "Unexpected ValueType.");
7659
7660 return DAG.UnrollVectorOp(N: Op.getNode());
7661}
7662
7663// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
7664// wider vector type is legal.
7665SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op,
7666 SelectionDAG &DAG) const {
7667 unsigned Opc = Op.getOpcode();
7668 EVT VT = Op.getValueType();
7669 assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
7670
7671 auto [Lo0, Hi0] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
7672 auto [Lo1, Hi1] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 1);
7673
7674 SDLoc SL(Op);
7675
7676 SDValue OpLo =
7677 DAG.getNode(Opcode: Opc, DL: SL, VT: Lo0.getValueType(), N1: Lo0, N2: Lo1, Flags: Op->getFlags());
7678 SDValue OpHi =
7679 DAG.getNode(Opcode: Opc, DL: SL, VT: Hi0.getValueType(), N1: Hi0, N2: Hi1, Flags: Op->getFlags());
7680
7681 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7682}
7683
7684SDValue SITargetLowering::splitTernaryVectorOp(SDValue Op,
7685 SelectionDAG &DAG) const {
7686 unsigned Opc = Op.getOpcode();
7687 EVT VT = Op.getValueType();
7688 assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
7689
7690 SDValue Op0 = Op.getOperand(i: 0);
7691 SDValue Lo0, Hi0;
7692 if (Op0.getValueType().isVector())
7693 std::tie(args&: Lo0, args&: Hi0) = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
7694 else
7695 Lo0 = Hi0 = DAG.getFreeze(V: Op0);
7696
7697 auto [Lo1, Hi1] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 1);
7698 auto [Lo2, Hi2] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 2);
7699
7700 SDLoc SL(Op);
7701 auto ResVT = DAG.GetSplitDestVTs(VT);
7702
7703 SDValue OpLo =
7704 DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT.first, N1: Lo0, N2: Lo1, N3: Lo2, Flags: Op->getFlags());
7705 SDValue OpHi =
7706 DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT.second, N1: Hi0, N2: Hi1, N3: Hi2, Flags: Op->getFlags());
7707
7708 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7709}
7710
7711SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
7712 switch (Op.getOpcode()) {
7713 default:
7714 return AMDGPUTargetLowering::LowerOperation(Op, DAG);
7715 case ISD::BRCOND:
7716 return LowerBRCOND(Op, DAG);
7717 case ISD::RETURNADDR:
7718 return LowerRETURNADDR(Op, DAG);
7719 case ISD::SPONENTRY:
7720 return LowerSPONENTRY(Op, DAG);
7721 case ISD::LOAD: {
7722 SDValue Result = LowerLOAD(Op, DAG);
7723 assert((!Result.getNode() || Result.getNode()->getNumValues() == 2) &&
7724 "Load should return a value and a chain");
7725 return Result;
7726 }
7727 case ISD::FSQRT: {
7728 EVT VT = Op.getValueType();
7729 if (VT == MVT::f32)
7730 return lowerFSQRTF32(Op, DAG);
7731 if (VT == MVT::f64)
7732 return lowerFSQRTF64(Op, DAG);
7733 return SDValue();
7734 }
7735 case ISD::FSIN:
7736 case ISD::FCOS:
7737 return LowerTrig(Op, DAG);
7738 case ISD::SELECT:
7739 return LowerSELECT(Op, DAG);
7740 case ISD::FDIV:
7741 return LowerFDIV(Op, DAG);
7742 case ISD::FFREXP:
7743 return LowerFFREXP(Op, DAG);
7744 case ISD::ATOMIC_CMP_SWAP:
7745 return LowerATOMIC_CMP_SWAP(Op, DAG);
7746 case ISD::STORE:
7747 return LowerSTORE(Op, DAG);
7748 case ISD::GlobalAddress: {
7749 MachineFunction &MF = DAG.getMachineFunction();
7750 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
7751 return LowerGlobalAddress(MFI, Op, DAG);
7752 }
7753 case ISD::BlockAddress:
7754 return LowerBlockAddress(Op, DAG);
7755 case ISD::ExternalSymbol:
7756 return LowerExternalSymbol(Op, DAG);
7757 case ISD::INTRINSIC_WO_CHAIN:
7758 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
7759 case ISD::CONVERT_FROM_ARBITRARY_FP:
7760 return LowerCONVERT_FROM_ARBITRARY_FP(Op, DAG);
7761 case ISD::CONVERT_TO_ARBITRARY_FP:
7762 return LowerCONVERT_TO_ARBITRARY_FP(Op, DAG);
7763 case ISD::INTRINSIC_W_CHAIN:
7764 return LowerINTRINSIC_W_CHAIN(Op, DAG);
7765 case ISD::INTRINSIC_VOID:
7766 return LowerINTRINSIC_VOID(Op, DAG);
7767 case ISD::ADDRSPACECAST:
7768 return lowerADDRSPACECAST(Op, DAG);
7769 case ISD::INSERT_SUBVECTOR:
7770 return lowerINSERT_SUBVECTOR(Op, DAG);
7771 case ISD::INSERT_VECTOR_ELT:
7772 return lowerINSERT_VECTOR_ELT(Op, DAG);
7773 case ISD::EXTRACT_VECTOR_ELT:
7774 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
7775 case ISD::VECTOR_SHUFFLE:
7776 return lowerVECTOR_SHUFFLE(Op, DAG);
7777 case ISD::SCALAR_TO_VECTOR:
7778 return lowerSCALAR_TO_VECTOR(Op, DAG);
7779 case ISD::BUILD_VECTOR:
7780 return lowerBUILD_VECTOR(Op, DAG);
7781 case ISD::FP_ROUND:
7782 case ISD::STRICT_FP_ROUND:
7783 return lowerFP_ROUND(Op, DAG);
7784 case ISD::TRAP:
7785 return lowerTRAP(Op, DAG);
7786 case ISD::DEBUGTRAP:
7787 return lowerDEBUGTRAP(Op, DAG);
7788 case ISD::ABS:
7789 case ISD::FABS:
7790 case ISD::FNEG:
7791 case ISD::FCANONICALIZE:
7792 case ISD::BSWAP:
7793 return splitUnaryVectorOp(Op, DAG);
7794 case ISD::FP_TO_SINT_SAT:
7795 case ISD::FP_TO_UINT_SAT:
7796 if (Op.getValueType().isVector() && Op.getValueType() != MVT::v2i16 &&
7797 Op.getOperand(i: 0).getValueType().getScalarType() == MVT::f32)
7798 return splitUnaryVectorOp(Op, DAG);
7799 return LowerFP_TO_INT_SAT(Op, DAG);
7800 case ISD::FSUB:
7801 if (Op.getValueType() == MVT::bf16) {
7802 // Custom expansion:
7803 // fsub bf16 %a, %b -> fadd v2bf16(widen %a), fneg v2bf16(widen %b)
7804 // Then extract back to bf16.
7805 //
7806 // We create fneg on v2bf16 (not bf16) so the instruction selector can
7807 // fold the negation into the packed add's neg_lo/neg_hi modifiers,
7808 // generating a single v_pk_add_bf16 instruction. If we negate bf16 first,
7809 // it becomes a separate v_xor instruction before widening.
7810 SDLoc DL(Op);
7811 SDValue Op0 = Op.getOperand(i: 0);
7812 SDValue Op1 = Op.getOperand(i: 1);
7813
7814 // Widen both operands to v2bf16
7815 SDValue Vec0 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2bf16, Operand: Op0);
7816 SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2bf16, Operand: Op1);
7817
7818 // Create FNEG v2bf16 for the second operand
7819 SDValue NegVec1 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::v2bf16, Operand: Vec1);
7820
7821 // Perform FADD v2bf16
7822 SDValue Result = DAG.getNode(Opcode: ISD::FADD, DL, VT: MVT::v2bf16, N1: Vec0, N2: NegVec1);
7823
7824 // Extract element 0 back to bf16
7825 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::bf16, N1: Result,
7826 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
7827 }
7828 return SDValue();
7829 case ISD::FMINNUM:
7830 case ISD::FMAXNUM:
7831 return lowerFMINNUM_FMAXNUM(Op, DAG);
7832 case ISD::FMINIMUMNUM:
7833 case ISD::FMAXIMUMNUM:
7834 return lowerFMINIMUMNUM_FMAXIMUMNUM(Op, DAG);
7835 case ISD::FLDEXP:
7836 case ISD::STRICT_FLDEXP:
7837 return lowerFLDEXP(Op, DAG);
7838 case ISD::FMA:
7839 return splitTernaryVectorOp(Op, DAG);
7840 case ISD::FP_TO_SINT:
7841 case ISD::FP_TO_UINT:
7842 if (Subtarget->hasVCvtPkIU16F32() && Op.getValueType() == MVT::i16 &&
7843 Op.getOperand(i: 0).getValueType() == MVT::f32) {
7844 // Make f32->i16 legal so we can select V_CVT_PK_[IU]16_F32.
7845 return Op;
7846 }
7847 return LowerFP_TO_INT(Op, DAG);
7848 case ISD::SHL:
7849 case ISD::SRA:
7850 case ISD::SRL:
7851 case ISD::ADD:
7852 case ISD::SUB:
7853 case ISD::SMIN:
7854 case ISD::SMAX:
7855 case ISD::UMIN:
7856 case ISD::UMAX:
7857 case ISD::FMINNUM_IEEE:
7858 case ISD::FMAXNUM_IEEE:
7859 case ISD::FMINIMUM:
7860 case ISD::FMAXIMUM:
7861 case ISD::UADDSAT:
7862 case ISD::USUBSAT:
7863 case ISD::SADDSAT:
7864 case ISD::SSUBSAT:
7865 case ISD::FADD:
7866 case ISD::FMUL:
7867 return splitBinaryVectorOp(Op, DAG);
7868 case ISD::FCOPYSIGN:
7869 return lowerFCOPYSIGN(Op, DAG);
7870 case ISD::MUL:
7871 return lowerMUL(Op, DAG);
7872 case ISD::SMULO:
7873 case ISD::UMULO:
7874 return lowerXMULO(Op, DAG);
7875 case ISD::SMUL_LOHI:
7876 case ISD::UMUL_LOHI:
7877 return lowerXMUL_LOHI(Op, DAG);
7878 case ISD::DYNAMIC_STACKALLOC:
7879 return LowerDYNAMIC_STACKALLOC(Op, DAG);
7880 case ISD::STACKSAVE:
7881 return LowerSTACKSAVE(Op, DAG);
7882 case ISD::GET_ROUNDING:
7883 return lowerGET_ROUNDING(Op, DAG);
7884 case ISD::SET_ROUNDING:
7885 return lowerSET_ROUNDING(Op, DAG);
7886 case ISD::PREFETCH:
7887 return lowerPREFETCH(Op, DAG);
7888 case ISD::FP_EXTEND:
7889 case ISD::STRICT_FP_EXTEND:
7890 return lowerFP_EXTEND(Op, DAG);
7891 case ISD::GET_FPENV:
7892 return lowerGET_FPENV(Op, DAG);
7893 case ISD::SET_FPENV:
7894 return lowerSET_FPENV(Op, DAG);
7895 case ISD::ROTR:
7896 return lowerROTR(Op, DAG);
7897 case ISD::INLINEASM:
7898 return LowerINLINEASM(Op, DAG);
7899 }
7900 return SDValue();
7901}
7902
7903// TFE results are dword granular: value dwords followed by one status dword.
7904static std::pair<SDValue, SDValue>
7905splitTFEValueAndStatus(SDValue Op, EVT VT, const SDLoc &DL, SelectionDAG &DAG) {
7906 LLVMContext &C = *DAG.getContext();
7907 unsigned NumValueDWords = divideCeil(Numerator: VT.getSizeInBits(), Denominator: 32);
7908 SDValue Status = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
7909 N2: DAG.getVectorIdxConstant(Val: NumValueDWords, DL));
7910 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL);
7911 SDValue ValueDWords =
7912 NumValueDWords == 1
7913 ? DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op, N2: ZeroIdx)
7914 : DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL,
7915 VT: EVT::getVectorVT(Context&: C, VT: MVT::i32, NumElements: NumValueDWords), N1: Op,
7916 N2: ZeroIdx);
7917 if (!VT.isVector() && VT.getSizeInBits() < 32)
7918 ValueDWords =
7919 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VT.changeTypeToInteger(), Operand: ValueDWords);
7920 return {DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: ValueDWords), Status};
7921}
7922
7923// Used for D16: Casts the result of an instruction into the right vector,
7924// packs values if loads return unpacked values.
7925static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
7926 const SDLoc &DL, SelectionDAG &DAG,
7927 bool Unpacked) {
7928 if (!LoadVT.isVector())
7929 return Result;
7930
7931 // Cast back to the original packed type or to a larger type that is a
7932 // multiple of 32 bit for D16. Widening the return type is a required for
7933 // legalization.
7934 EVT FittingLoadVT = LoadVT;
7935 if ((LoadVT.getVectorNumElements() % 2) == 1) {
7936 FittingLoadVT =
7937 EVT::getVectorVT(Context&: *DAG.getContext(), VT: LoadVT.getVectorElementType(),
7938 NumElements: LoadVT.getVectorNumElements() + 1);
7939 }
7940
7941 if (Unpacked) { // From v2i32/v4i32 back to v2f16/v4f16.
7942 // Truncate to v2i16/v4i16.
7943 EVT IntLoadVT = FittingLoadVT.changeTypeToInteger();
7944
7945 // Workaround legalizer not scalarizing truncate after vector op
7946 // legalization but not creating intermediate vector trunc.
7947 SmallVector<SDValue, 4> Elts;
7948 DAG.ExtractVectorElements(Op: Result, Args&: Elts);
7949 for (SDValue &Elt : Elts)
7950 Elt = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Elt);
7951
7952 // Pad illegal v1i16/v3fi6 to v4i16
7953 if ((LoadVT.getVectorNumElements() % 2) == 1)
7954 Elts.push_back(Elt: DAG.getPOISON(VT: MVT::i16));
7955
7956 Result = DAG.getBuildVector(VT: IntLoadVT, DL, Ops: Elts);
7957
7958 // Bitcast to original type (v2f16/v4f16).
7959 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FittingLoadVT, Operand: Result);
7960 }
7961
7962 // Cast back to the original packed type.
7963 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FittingLoadVT, Operand: Result);
7964}
7965
7966SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, MemSDNode *M,
7967 SelectionDAG &DAG,
7968 ArrayRef<SDValue> Ops,
7969 bool IsIntrinsic) const {
7970 SDLoc DL(M);
7971
7972 bool IsTFE = M->getNumValues() == 3;
7973 bool Unpacked = Subtarget->hasUnpackedD16VMem();
7974 EVT LoadVT = M->getValueType(ResNo: 0);
7975
7976 EVT EquivLoadVT = LoadVT;
7977 if (LoadVT.isVector()) {
7978 if (Unpacked) {
7979 EquivLoadVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
7980 NumElements: LoadVT.getVectorNumElements());
7981 } else if ((LoadVT.getVectorNumElements() % 2) == 1) {
7982 // Widen v3f16 to legal type
7983 EquivLoadVT =
7984 EVT::getVectorVT(Context&: *DAG.getContext(), VT: LoadVT.getVectorElementType(),
7985 NumElements: LoadVT.getVectorNumElements() + 1);
7986 }
7987 }
7988
7989 if (IsTFE) {
7990 unsigned NumValueDWords = divideCeil(Numerator: EquivLoadVT.getSizeInBits(), Denominator: 32);
7991 EVT LoadDWordsVT =
7992 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumValueDWords + 1);
7993 SDVTList VTList = DAG.getVTList(VT1: LoadDWordsVT, VT2: MVT::Other);
7994 SDValue Load = DAG.getMemIntrinsicNode(
7995 Opcode, dl: DL, VTList, Ops, MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
7996 auto [Value, Status] = splitTFEValueAndStatus(Op: Load, VT: EquivLoadVT, DL, DAG);
7997 SDValue Adjusted =
7998 adjustLoadValueTypeImpl(Result: Value, LoadVT, DL, DAG, Unpacked);
7999 return DAG.getMergeValues(Ops: {Adjusted, Status, Load.getValue(R: 1)}, dl: DL);
8000 }
8001
8002 // Change from v4f16/v2f16 to EquivLoadVT.
8003 SDVTList VTList = DAG.getVTList(VT1: EquivLoadVT, VT2: MVT::Other);
8004
8005 SDValue Load = DAG.getMemIntrinsicNode(
8006 Opcode: IsIntrinsic ? (unsigned)ISD::INTRINSIC_W_CHAIN : Opcode, dl: DL, VTList, Ops,
8007 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
8008
8009 SDValue Adjusted = adjustLoadValueTypeImpl(Result: Load, LoadVT, DL, DAG, Unpacked);
8010
8011 return DAG.getMergeValues(Ops: {Adjusted, Load.getValue(R: 1)}, dl: DL);
8012}
8013
8014SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
8015 SelectionDAG &DAG,
8016 ArrayRef<SDValue> Ops) const {
8017 SDLoc DL(M);
8018 EVT LoadVT = M->getValueType(ResNo: 0);
8019 EVT EltType = LoadVT.getScalarType();
8020 EVT IntVT = LoadVT.changeTypeToInteger();
8021
8022 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
8023
8024 if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
8025 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
8026 DAG.getMachineFunction().getFunction(),
8027 "unsupported sub-dword format buffer load", DL.getDebugLoc()));
8028 return DAG.getMergeValues(Ops: {DAG.getPOISON(VT: LoadVT), M->getOperand(Num: 0)}, dl: DL);
8029 }
8030
8031 assert(M->getNumValues() == 2 || M->getNumValues() == 3);
8032 bool IsTFE = M->getNumValues() == 3;
8033
8034 if (IsD16 && IsTFE && !Subtarget->hasBufferTFEFormatD16()) {
8035 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
8036 DAG.getMachineFunction().getFunction(),
8037 "TFE D16 format buffer load is not supported on this GPU",
8038 DL.getDebugLoc()));
8039 return DAG.getErrorMergeValues(ResultTypes: {M->value_begin(), M->value_end()},
8040 Chain: M->getOperand(Num: 0), dl: DL);
8041 }
8042
8043 unsigned Opc = IsD16 ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_D16_TFE
8044 : AMDGPUISD::BUFFER_LOAD_FORMAT_D16)
8045 : IsFormat ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_TFE
8046 : AMDGPUISD::BUFFER_LOAD_FORMAT)
8047 : IsTFE ? AMDGPUISD::BUFFER_LOAD_TFE
8048 : AMDGPUISD::BUFFER_LOAD;
8049
8050 if (IsD16)
8051 return adjustLoadValueType(Opcode: Opc, M, DAG, Ops);
8052
8053 // Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics
8054 if (!IsD16 && !LoadVT.isVector() && EltType.getSizeInBits() < 32)
8055 return handleByteShortBufferLoads(DAG, LoadVT, DL, Ops, MMO: M->getMemOperand(),
8056 IsTFE);
8057
8058 if (isTypeLegal(VT: LoadVT)) {
8059 return getMemIntrinsicNode(Opcode: Opc, DL, VTList: M->getVTList(), Ops, MemVT: IntVT,
8060 MMO: M->getMemOperand(), DAG);
8061 }
8062
8063 EVT CastVT = getEquivalentMemType(Context&: *DAG.getContext(), VT: LoadVT);
8064 SDVTList VTList = IsTFE ? DAG.getVTList(VT1: CastVT, VT2: MVT::i32, VT3: MVT::Other)
8065 : DAG.getVTList(VT1: CastVT, VT2: MVT::Other);
8066 SDValue MemNode = getMemIntrinsicNode(Opcode: Opc, DL, VTList, Ops, MemVT: CastVT,
8067 MMO: M->getMemOperand(), DAG);
8068 SDValue Data = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: MemNode);
8069 if (IsTFE)
8070 return DAG.getMergeValues(Ops: {Data, MemNode.getValue(R: 1), MemNode.getValue(R: 2)},
8071 dl: DL);
8072 return DAG.getMergeValues(Ops: {Data, MemNode.getValue(R: 1)}, dl: DL);
8073}
8074
8075static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N,
8076 SelectionDAG &DAG) {
8077 EVT VT = N->getValueType(ResNo: 0);
8078 SDValue Src = N->getOperand(Num: 1);
8079 SDLoc SL(N);
8080
8081 if (Src.getOpcode() == ISD::SETCC) {
8082 SDValue Op0 = Src.getOperand(i: 0);
8083 SDValue Op1 = Src.getOperand(i: 1);
8084 // Need to expand bfloat to float for comparison (setcc).
8085 if (Op0.getValueType() == MVT::bf16) {
8086 Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op0);
8087 Op1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op1);
8088 }
8089 // (ballot (ISD::SETCC ...)) -> (AMDGPUISD::SETCC ...)
8090 return DAG.getNode(Opcode: AMDGPUISD::SETCC, DL: SL, VT, N1: Op0, N2: Op1, N3: Src.getOperand(i: 2));
8091 }
8092 if (const ConstantSDNode *Arg = dyn_cast<ConstantSDNode>(Val&: Src)) {
8093 // (ballot 0) -> 0
8094 if (Arg->isZero())
8095 return DAG.getConstant(Val: 0, DL: SL, VT);
8096
8097 // (ballot 1) -> EXEC/EXEC_LO
8098 if (Arg->isOne()) {
8099 Register Exec;
8100 if (VT.getScalarSizeInBits() == 32)
8101 Exec = AMDGPU::EXEC_LO;
8102 else if (VT.getScalarSizeInBits() == 64)
8103 Exec = AMDGPU::EXEC;
8104 else
8105 return SDValue();
8106
8107 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: Exec, VT);
8108 }
8109 }
8110
8111 // (ballot (i1 $src)) -> (AMDGPUISD::SETCC (i32 (zext $src)) (i32 0)
8112 // ISD::SETNE)
8113 return DAG.getNode(
8114 Opcode: AMDGPUISD::SETCC, DL: SL, VT, N1: DAG.getZExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32),
8115 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), N3: DAG.getCondCode(Cond: ISD::SETNE));
8116}
8117
8118static SDValue lowerBFEIntrinsic(SDValue Op, SelectionDAG &DAG,
8119 Intrinsic::ID IntrinsicID) {
8120 bool Signed = IntrinsicID == Intrinsic::amdgcn_sbfe;
8121 SDLoc DL(Op);
8122 EVT VT = Op.getValueType();
8123 SDValue Src = Op.getOperand(i: 1);
8124 SDValue Offset = Op.getOperand(i: 2);
8125 SDValue Width = Op.getOperand(i: 3);
8126
8127 if (VT != MVT::i32) {
8128 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
8129 DAG.getMachineFunction().getFunction(),
8130 Twine(Intrinsic::getBaseName(id: IntrinsicID)) + " only supports i32",
8131 DL.getDebugLoc()));
8132 return DAG.getPOISON(VT);
8133 }
8134
8135 return DAG.getNode(Opcode: Signed ? AMDGPUISD::BFE_I32 : AMDGPUISD::BFE_U32, DL, VT,
8136 N1: Src, N2: Offset, N3: Width);
8137}
8138
8139static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
8140 EVT VT);
8141
8142static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N,
8143 SelectionDAG &DAG) {
8144 EVT VT = N->getValueType(ResNo: 0);
8145 unsigned ValSize = VT.getSizeInBits();
8146 unsigned IID = N->getConstantOperandVal(Num: 0);
8147 bool IsPermLane16 = IID == Intrinsic::amdgcn_permlane16 ||
8148 IID == Intrinsic::amdgcn_permlanex16;
8149 bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
8150 IID == Intrinsic::amdgcn_set_inactive_chain_arg;
8151 bool IsPermlaneShuffle = IID == Intrinsic::amdgcn_permlane_bcast ||
8152 IID == Intrinsic::amdgcn_permlane_up ||
8153 IID == Intrinsic::amdgcn_permlane_down ||
8154 IID == Intrinsic::amdgcn_permlane_xor;
8155 SDLoc SL(N);
8156 MVT IntVT = MVT::getIntegerVT(BitWidth: ValSize);
8157 const GCNSubtarget *ST = TLI.getSubtarget();
8158
8159 unsigned SplitSize = 32;
8160 if (IID == Intrinsic::amdgcn_update_dpp && (ValSize % 64 == 0) &&
8161 ST->hasDPALU_DPP() &&
8162 AMDGPU::isLegalDPALU_DPPControl(ST: *ST, DC: N->getConstantOperandVal(Num: 3)))
8163 SplitSize = 64;
8164
8165 auto createLaneOp = [&DAG, &SL, N, IID](SDValue Src0, SDValue Src1,
8166 SDValue Src2, MVT ValT) -> SDValue {
8167 SmallVector<SDValue, 8> Operands;
8168 switch (IID) {
8169 case Intrinsic::amdgcn_permlane16:
8170 case Intrinsic::amdgcn_permlanex16:
8171 case Intrinsic::amdgcn_update_dpp:
8172 Operands.push_back(Elt: N->getOperand(Num: 6));
8173 Operands.push_back(Elt: N->getOperand(Num: 5));
8174 Operands.push_back(Elt: N->getOperand(Num: 4));
8175 [[fallthrough]];
8176 case Intrinsic::amdgcn_writelane:
8177 case Intrinsic::amdgcn_permlane_bcast:
8178 case Intrinsic::amdgcn_permlane_up:
8179 case Intrinsic::amdgcn_permlane_down:
8180 case Intrinsic::amdgcn_permlane_xor:
8181 Operands.push_back(Elt: Src2);
8182 [[fallthrough]];
8183 case Intrinsic::amdgcn_readlane:
8184 case Intrinsic::amdgcn_set_inactive:
8185 case Intrinsic::amdgcn_set_inactive_chain_arg:
8186 case Intrinsic::amdgcn_mov_dpp8:
8187 Operands.push_back(Elt: Src1);
8188 [[fallthrough]];
8189 case Intrinsic::amdgcn_readfirstlane:
8190 case Intrinsic::amdgcn_permlane64:
8191 Operands.push_back(Elt: Src0);
8192 break;
8193 default:
8194 llvm_unreachable("unhandled lane op");
8195 }
8196
8197 Operands.push_back(Elt: DAG.getTargetConstant(Val: IID, DL: SL, VT: MVT::i32));
8198 std::reverse(first: Operands.begin(), last: Operands.end());
8199
8200 if (SDNode *GL = N->getGluedNode()) {
8201 assert(GL->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
8202 GL = GL->getOperand(Num: 0).getNode();
8203 Operands.push_back(Elt: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL: SL, VT: MVT::Glue,
8204 Operand: SDValue(GL, 0)));
8205 }
8206
8207 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: ValT, Ops: Operands);
8208 };
8209
8210 SDValue Src0 = N->getOperand(Num: 1);
8211 SDValue Src1, Src2;
8212 if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
8213 IID == Intrinsic::amdgcn_mov_dpp8 ||
8214 IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16 ||
8215 IsPermlaneShuffle) {
8216 Src1 = N->getOperand(Num: 2);
8217 if (IID == Intrinsic::amdgcn_writelane ||
8218 IID == Intrinsic::amdgcn_update_dpp || IsPermLane16 ||
8219 IsPermlaneShuffle)
8220 Src2 = N->getOperand(Num: 3);
8221 }
8222
8223 if (ValSize == SplitSize) {
8224 // Already legal
8225 return SDValue();
8226 }
8227
8228 if (ValSize < 32) {
8229 bool IsFloat = VT.isFloatingPoint();
8230 Src0 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src0) : Src0,
8231 DL: SL, VT: MVT::i32);
8232
8233 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) {
8234 Src1 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src1) : Src1,
8235 DL: SL, VT: MVT::i32);
8236 }
8237
8238 if (IID == Intrinsic::amdgcn_writelane) {
8239 Src2 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src2) : Src2,
8240 DL: SL, VT: MVT::i32);
8241 }
8242
8243 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, MVT::i32);
8244 SDValue Trunc = DAG.getAnyExtOrTrunc(Op: LaneOp, DL: SL, VT: IntVT);
8245 return IsFloat ? DAG.getBitcast(VT, V: Trunc) : Trunc;
8246 }
8247
8248 if (ValSize % SplitSize != 0)
8249 return SDValue();
8250
8251 auto unrollLaneOp = [&DAG, &SL](SDNode *N) -> SDValue {
8252 EVT VT = N->getValueType(ResNo: 0);
8253 unsigned NE = VT.getVectorNumElements();
8254 EVT EltVT = VT.getVectorElementType();
8255 SmallVector<SDValue, 8> Scalars;
8256 unsigned NumOperands = N->getNumOperands();
8257 SmallVector<SDValue, 4> Operands(NumOperands);
8258 SDNode *GL = N->getGluedNode();
8259
8260 // only handle convergencectrl_glue
8261 assert(!GL || GL->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
8262
8263 for (unsigned i = 0; i != NE; ++i) {
8264 for (unsigned j = 0, e = GL ? NumOperands - 1 : NumOperands; j != e;
8265 ++j) {
8266 SDValue Operand = N->getOperand(Num: j);
8267 EVT OperandVT = Operand.getValueType();
8268 if (OperandVT.isVector()) {
8269 // A vector operand; extract a single element.
8270 EVT OperandEltVT = OperandVT.getVectorElementType();
8271 Operands[j] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: OperandEltVT,
8272 N1: Operand, N2: DAG.getVectorIdxConstant(Val: i, DL: SL));
8273 } else {
8274 // A scalar operand; just use it as is.
8275 Operands[j] = Operand;
8276 }
8277 }
8278
8279 if (GL)
8280 Operands[NumOperands - 1] =
8281 DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL: SL, VT: MVT::Glue,
8282 Operand: SDValue(GL->getOperand(Num: 0).getNode(), 0));
8283
8284 Scalars.push_back(Elt: DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: EltVT, Ops: Operands));
8285 }
8286
8287 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NE);
8288 return DAG.getBuildVector(VT: VecVT, DL: SL, Ops: Scalars);
8289 };
8290
8291 if (VT.isVector()) {
8292 switch (MVT::SimpleValueType EltTy =
8293 VT.getVectorElementType().getSimpleVT().SimpleTy) {
8294 case MVT::i32:
8295 case MVT::f32:
8296 if (SplitSize == 32) {
8297 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VT.getSimpleVT());
8298 return unrollLaneOp(LaneOp.getNode());
8299 }
8300 [[fallthrough]];
8301 case MVT::i16:
8302 case MVT::f16:
8303 case MVT::bf16: {
8304 unsigned SubVecNumElt =
8305 SplitSize / VT.getVectorElementType().getSizeInBits();
8306 MVT SubVecVT = MVT::getVectorVT(VT: EltTy, NumElements: SubVecNumElt);
8307 SmallVector<SDValue, 4> Pieces;
8308 SDValue Src0SubVec, Src1SubVec, Src2SubVec;
8309 for (unsigned i = 0, EltIdx = 0; i < ValSize / SplitSize; i++) {
8310 Src0SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src0,
8311 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
8312
8313 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive ||
8314 IsPermLane16) {
8315 Src1SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src1,
8316 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
8317
8318 Pieces.push_back(
8319 Elt: createLaneOp(Src0SubVec, Src1SubVec, Src2, SubVecVT));
8320 } else if (IID == Intrinsic::amdgcn_writelane) {
8321 Src2SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src2,
8322 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
8323 Pieces.push_back(
8324 Elt: createLaneOp(Src0SubVec, Src1, Src2SubVec, SubVecVT));
8325 } else {
8326 Pieces.push_back(Elt: createLaneOp(Src0SubVec, Src1, Src2, SubVecVT));
8327 }
8328
8329 EltIdx += SubVecNumElt;
8330 }
8331 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT, Ops: Pieces);
8332 }
8333 default:
8334 // Handle all other cases by bitcasting to i32 vectors
8335 break;
8336 }
8337 }
8338
8339 MVT VecVT =
8340 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplitSize), NumElements: ValSize / SplitSize);
8341 Src0 = DAG.getBitcast(VT: VecVT, V: Src0);
8342
8343 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
8344 Src1 = DAG.getBitcast(VT: VecVT, V: Src1);
8345
8346 if (IID == Intrinsic::amdgcn_writelane)
8347 Src2 = DAG.getBitcast(VT: VecVT, V: Src2);
8348
8349 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VecVT);
8350 SDValue UnrolledLaneOp = unrollLaneOp(LaneOp.getNode());
8351 return DAG.getBitcast(VT, V: UnrolledLaneOp);
8352}
8353
8354static SDValue lowerWaveShuffle(const SITargetLowering &TLI, SDNode *N,
8355 SelectionDAG &DAG) {
8356 EVT VT = N->getValueType(ResNo: 0);
8357
8358 if (VT.getSizeInBits() != 32)
8359 return SDValue();
8360
8361 SDLoc SL(N);
8362
8363 SDValue Value = N->getOperand(Num: 1);
8364 SDValue Index = N->getOperand(Num: 2);
8365
8366 // ds_bpermute requires index to be multiplied by 4
8367 SDValue ShiftAmount = DAG.getShiftAmountConstant(Val: 2, VT: MVT::i32, DL: SL);
8368 SDValue ShiftedIndex =
8369 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: Index.getValueType(), N1: Index, N2: ShiftAmount);
8370
8371 // Intrinsics will require i32 to operate on
8372 SDValue ValueI32 = DAG.getBitcast(VT: MVT::i32, V: Value);
8373
8374 auto MakeIntrinsic = [&DAG, &SL](unsigned IID, MVT RetVT,
8375 SmallVector<SDValue> IntrinArgs) -> SDValue {
8376 SmallVector<SDValue> Operands(1);
8377 Operands[0] = DAG.getTargetConstant(Val: IID, DL: SL, VT: MVT::i32);
8378 Operands.append(RHS: IntrinArgs);
8379 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: RetVT, Ops: Operands);
8380 };
8381
8382 // If we can bpermute across the whole wave, then just do that
8383 if (TLI.getSubtarget()->supportsWaveWideBPermute()) {
8384 SDValue BPermute = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
8385 {ShiftedIndex, ValueI32});
8386 return DAG.getBitcast(VT, V: BPermute);
8387 }
8388
8389 assert(TLI.getSubtarget()->isWave64());
8390
8391 // Otherwise, we need to make use of whole wave mode
8392 SDValue PoisonVal = DAG.getPOISON(VT: ValueI32->getValueType(ResNo: 0));
8393
8394 // Set inactive lanes to poison
8395 SDValue WWMValue = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
8396 {ValueI32, PoisonVal});
8397 SDValue WWMIndex = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
8398 {ShiftedIndex, PoisonVal});
8399
8400 SDValue Swapped =
8401 MakeIntrinsic(Intrinsic::amdgcn_permlane64, MVT::i32, {WWMValue});
8402
8403 // Get permutation of each half, then we'll select which one to use
8404 SDValue BPermSameHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
8405 {WWMIndex, WWMValue});
8406 SDValue BPermOtherHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute,
8407 MVT::i32, {WWMIndex, Swapped});
8408 SDValue BPermOtherHalfWWM =
8409 MakeIntrinsic(Intrinsic::amdgcn_wwm, MVT::i32, {BPermOtherHalf});
8410
8411 // Select which side to take the permute from
8412 SDValue ThreadIDMask = DAG.getAllOnesConstant(DL: SL, VT: MVT::i32);
8413 // We can get away with only using mbcnt_lo here since we're only
8414 // trying to detect which side of 32 each lane is on, and mbcnt_lo
8415 // returns 32 for lanes 32-63.
8416 SDValue ThreadID =
8417 MakeIntrinsic(Intrinsic::amdgcn_mbcnt_lo, MVT::i32,
8418 {ThreadIDMask, DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32)});
8419
8420 SDValue SameOrOtherHalf =
8421 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32,
8422 N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: ThreadID, N2: Index),
8423 N2: DAG.getTargetConstant(Val: 32, DL: SL, VT: MVT::i32));
8424 SDValue UseSameHalf =
8425 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: SameOrOtherHalf,
8426 RHS: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond: ISD::SETEQ);
8427 SDValue Result = DAG.getSelect(DL: SL, VT: MVT::i32, Cond: UseSameHalf, LHS: BPermSameHalf,
8428 RHS: BPermOtherHalfWWM);
8429 return DAG.getBitcast(VT, V: Result);
8430}
8431
8432void SITargetLowering::ReplaceNodeResults(SDNode *N,
8433 SmallVectorImpl<SDValue> &Results,
8434 SelectionDAG &DAG) const {
8435 switch (N->getOpcode()) {
8436 case ISD::INSERT_VECTOR_ELT: {
8437 if (SDValue Res = lowerINSERT_VECTOR_ELT(Op: SDValue(N, 0), DAG))
8438 Results.push_back(Elt: Res);
8439 return;
8440 }
8441 case ISD::EXTRACT_VECTOR_ELT: {
8442 if (SDValue Res = lowerEXTRACT_VECTOR_ELT(Op: SDValue(N, 0), DAG))
8443 Results.push_back(Elt: Res);
8444 return;
8445 }
8446 case ISD::CONVERT_TO_ARBITRARY_FP: {
8447 if (SDValue Res = LowerCONVERT_TO_ARBITRARY_FP(Op: SDValue(N, 0), DAG))
8448 Results.push_back(Elt: Res);
8449 return;
8450 }
8451 case ISD::INTRINSIC_WO_CHAIN: {
8452 unsigned IID = N->getConstantOperandVal(Num: 0);
8453 switch (IID) {
8454 case Intrinsic::amdgcn_wave_reduce_min:
8455 case Intrinsic::amdgcn_wave_reduce_umin:
8456 case Intrinsic::amdgcn_wave_reduce_max:
8457 case Intrinsic::amdgcn_wave_reduce_umax:
8458 case Intrinsic::amdgcn_wave_reduce_add:
8459 case Intrinsic::amdgcn_wave_reduce_sub:
8460 case Intrinsic::amdgcn_wave_reduce_and:
8461 case Intrinsic::amdgcn_wave_reduce_or:
8462 case Intrinsic::amdgcn_wave_reduce_xor: {
8463 EVT VT = N->getValueType(ResNo: 0);
8464 if (isTypeLegal(VT))
8465 return;
8466 SDLoc SL(N);
8467 bool NeedsSignExt = IID == Intrinsic::amdgcn_wave_reduce_min ||
8468 IID == Intrinsic::amdgcn_wave_reduce_max ||
8469 IID == Intrinsic::amdgcn_wave_reduce_add ||
8470 IID == Intrinsic::amdgcn_wave_reduce_sub;
8471 unsigned ExtOpc = NeedsSignExt ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
8472 SDValue ExtSrc = DAG.getNode(Opcode: ExtOpc, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 1));
8473 SDValue Result = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
8474 N1: N->getOperand(Num: 0), N2: ExtSrc, N3: N->getOperand(Num: 2));
8475 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Result));
8476 return;
8477 }
8478 case Intrinsic::amdgcn_make_buffer_rsrc:
8479 Results.push_back(Elt: lowerPointerAsRsrcIntrin(Op: N, DAG));
8480 return;
8481 case Intrinsic::amdgcn_cvt_pkrtz: {
8482 SDValue Src0 = N->getOperand(Num: 1);
8483 SDValue Src1 = N->getOperand(Num: 2);
8484 SDLoc SL(N);
8485 SDValue Cvt =
8486 DAG.getNode(Opcode: AMDGPUISD::CVT_PKRTZ_F16_F32, DL: SL, VT: MVT::i32, N1: Src0, N2: Src1);
8487 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Cvt));
8488 return;
8489 }
8490 case Intrinsic::amdgcn_cvt_pknorm_i16:
8491 case Intrinsic::amdgcn_cvt_pknorm_u16:
8492 case Intrinsic::amdgcn_cvt_pk_i16:
8493 case Intrinsic::amdgcn_cvt_pk_u16: {
8494 SDValue Src0 = N->getOperand(Num: 1);
8495 SDValue Src1 = N->getOperand(Num: 2);
8496 SDLoc SL(N);
8497 unsigned Opcode;
8498
8499 if (IID == Intrinsic::amdgcn_cvt_pknorm_i16)
8500 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
8501 else if (IID == Intrinsic::amdgcn_cvt_pknorm_u16)
8502 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
8503 else if (IID == Intrinsic::amdgcn_cvt_pk_i16)
8504 Opcode = AMDGPUISD::CVT_PK_I16_I32;
8505 else
8506 Opcode = AMDGPUISD::CVT_PK_U16_U32;
8507
8508 EVT VT = N->getValueType(ResNo: 0);
8509 if (isTypeLegal(VT))
8510 Results.push_back(Elt: DAG.getNode(Opcode, DL: SL, VT, N1: Src0, N2: Src1));
8511 else {
8512 SDValue Cvt = DAG.getNode(Opcode, DL: SL, VT: MVT::i32, N1: Src0, N2: Src1);
8513 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: Cvt));
8514 }
8515 return;
8516 }
8517 case Intrinsic::amdgcn_s_buffer_load: {
8518 SDValue Op = SDValue(N, 0);
8519 EVT VT = Op.getValueType();
8520 Results.push_back(Elt: lowerSBuffer(VT, MemVT: VT, DL: SDLoc(Op), Chain: DAG.getEntryNode(),
8521 Rsrc: Op.getOperand(i: 1), Offset: Op.getOperand(i: 2),
8522 CachePolicy: Op.getOperand(i: 3), DAG));
8523 return;
8524 }
8525 case Intrinsic::amdgcn_dead: {
8526 for (unsigned I = 0, E = N->getNumValues(); I < E; ++I)
8527 Results.push_back(Elt: DAG.getPOISON(VT: N->getValueType(ResNo: I)));
8528 return;
8529 }
8530 }
8531 break;
8532 }
8533 case ISD::INTRINSIC_W_CHAIN: {
8534 if (N->getConstantOperandVal(Num: 1) != Intrinsic::amdgcn_ptr_s_buffer_load &&
8535 N->getValueType(ResNo: 0).isSimple() &&
8536 SBufferLoadDiagnosticVTs[N->getSimpleValueType(ResNo: 0).SimpleTy])
8537 break;
8538 if (SDValue Res = LowerINTRINSIC_W_CHAIN(Op: SDValue(N, 0), DAG)) {
8539 if (Res.getOpcode() == ISD::MERGE_VALUES) {
8540 // FIXME: Hacky
8541 for (unsigned I = 0; I < Res.getNumOperands(); I++) {
8542 Results.push_back(Elt: Res.getOperand(i: I));
8543 }
8544 } else {
8545 for (unsigned I = 0; I < N->getNumValues(); ++I)
8546 Results.push_back(Elt: Res.getValue(R: I));
8547 }
8548 return;
8549 }
8550
8551 break;
8552 }
8553 case ISD::SELECT: {
8554 SDLoc SL(N);
8555 EVT VT = N->getValueType(ResNo: 0);
8556 EVT NewVT = getEquivalentMemType(Context&: *DAG.getContext(), VT);
8557 SDValue LHS = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: N->getOperand(Num: 1));
8558 SDValue RHS = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: N->getOperand(Num: 2));
8559
8560 EVT SelectVT = NewVT;
8561 if (NewVT.bitsLT(VT: MVT::i32)) {
8562 LHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: LHS);
8563 RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: RHS);
8564 SelectVT = MVT::i32;
8565 }
8566
8567 SDValue NewSelect =
8568 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: SelectVT, N1: N->getOperand(Num: 0), N2: LHS, N3: RHS);
8569
8570 if (NewVT != SelectVT)
8571 NewSelect = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: NewVT, Operand: NewSelect);
8572 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: NewSelect));
8573 return;
8574 }
8575 case ISD::FNEG: {
8576 if (N->getValueType(ResNo: 0) != MVT::v2f16)
8577 break;
8578
8579 SDLoc SL(N);
8580 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 0));
8581
8582 SDValue Op = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: BC,
8583 N2: DAG.getConstant(Val: 0x80008000, DL: SL, VT: MVT::i32));
8584 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Op));
8585 return;
8586 }
8587 case ISD::FABS: {
8588 if (N->getValueType(ResNo: 0) != MVT::v2f16)
8589 break;
8590
8591 SDLoc SL(N);
8592 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 0));
8593
8594 SDValue Op = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: BC,
8595 N2: DAG.getConstant(Val: 0x7fff7fff, DL: SL, VT: MVT::i32));
8596 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Op));
8597 return;
8598 }
8599 case ISD::FSQRT: {
8600 if (N->getValueType(ResNo: 0) != MVT::f16)
8601 break;
8602 Results.push_back(Elt: lowerFSQRTF16(Op: SDValue(N, 0), DAG));
8603 break;
8604 }
8605 default:
8606 AMDGPUTargetLowering::ReplaceNodeResults(N, Results, DAG);
8607 break;
8608 }
8609}
8610
8611/// Helper function for LowerBRCOND
8612static SDNode *findUser(SDValue Value, unsigned Opcode) {
8613
8614 for (SDUse &U : Value->uses()) {
8615 if (U.get() != Value)
8616 continue;
8617
8618 if (U.getUser()->getOpcode() == Opcode)
8619 return U.getUser();
8620 }
8621 return nullptr;
8622}
8623
8624unsigned SITargetLowering::isCFIntrinsic(const SDNode *Intr) const {
8625 if (Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
8626 switch (Intr->getConstantOperandVal(Num: 1)) {
8627 case Intrinsic::amdgcn_if:
8628 return AMDGPUISD::IF;
8629 case Intrinsic::amdgcn_else:
8630 return AMDGPUISD::ELSE;
8631 case Intrinsic::amdgcn_loop:
8632 return AMDGPUISD::LOOP;
8633 case Intrinsic::amdgcn_end_cf:
8634 llvm_unreachable("should not occur");
8635 default:
8636 return 0;
8637 }
8638 }
8639
8640 // break, if_break, else_break are all only used as inputs to loop, not
8641 // directly as branch conditions.
8642 return 0;
8643}
8644
8645bool SITargetLowering::shouldEmitFixup(const GlobalValue *GV) const {
8646 const Triple &TT = getTargetMachine().getTargetTriple();
8647 return (GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
8648 GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
8649 AMDGPU::shouldEmitConstantsToTextSection(TT);
8650}
8651
8652bool SITargetLowering::shouldEmitGOTReloc(const GlobalValue *GV) const {
8653 if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS())
8654 return false;
8655
8656 // FIXME: Either avoid relying on address space here or change the default
8657 // address space for functions to avoid the explicit check.
8658 return (GV->getValueType()->isFunctionTy() ||
8659 !isNonGlobalAddrSpace(AS: GV->getAddressSpace())) &&
8660 !shouldEmitFixup(GV) && !getTargetMachine().shouldAssumeDSOLocal(GV);
8661}
8662
8663bool SITargetLowering::shouldEmitPCReloc(const GlobalValue *GV) const {
8664 return !shouldEmitFixup(GV) && !shouldEmitGOTReloc(GV);
8665}
8666
8667bool SITargetLowering::shouldUseLDSConstAddress(const GlobalValue *GV) const {
8668 if (!GV->hasExternalLinkage())
8669 return true;
8670
8671 // With object linking, external LDS declarations need relocations so the
8672 // linker can assign their offsets.
8673 if (AMDGPUTargetMachine::EnableObjectLinking) {
8674 if (const auto *GVar = dyn_cast<GlobalVariable>(Val: GV)) {
8675 if (GVar->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
8676 GVar->getAddressSpace() == AMDGPUAS::BARRIER) {
8677 assert(GVar->isDeclaration() &&
8678 "AS 3 & 13 GVs should be declaration here "
8679 "when object linking is enabled");
8680 return false;
8681 }
8682 }
8683 }
8684
8685 const auto OS = getTargetMachine().getTargetTriple().getOS();
8686 return OS == Triple::AMDHSA || OS == Triple::AMDPAL;
8687}
8688
8689/// This transforms the control flow intrinsics to get the branch destination as
8690/// last parameter, also switches branch target with BR if the need arise
8691SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND, SelectionDAG &DAG) const {
8692 SDLoc DL(BRCOND);
8693
8694 SDNode *Intr = BRCOND.getOperand(i: 1).getNode();
8695 SDValue Target = BRCOND.getOperand(i: 2);
8696 SDNode *BR = nullptr;
8697 SDNode *SetCC = nullptr;
8698
8699 switch (Intr->getOpcode()) {
8700 case ISD::SETCC: {
8701 // As long as we negate the condition everything is fine
8702 SetCC = Intr;
8703 Intr = SetCC->getOperand(Num: 0).getNode();
8704 break;
8705 }
8706 case ISD::XOR: {
8707 // Similar to SETCC, if we have (xor c, -1), we will be fine.
8708 SDValue LHS = Intr->getOperand(Num: 0);
8709 SDValue RHS = Intr->getOperand(Num: 1);
8710 if (auto *C = dyn_cast<ConstantSDNode>(Val&: RHS); C && C->getZExtValue()) {
8711 Intr = LHS.getNode();
8712 break;
8713 }
8714 [[fallthrough]];
8715 }
8716 default: {
8717 // Get the target from BR if we don't negate the condition
8718 BR = findUser(Value: BRCOND, Opcode: ISD::BR);
8719 assert(BR && "brcond missing unconditional branch user");
8720 Target = BR->getOperand(Num: 1);
8721 }
8722 }
8723
8724 unsigned CFNode = isCFIntrinsic(Intr);
8725 if (CFNode == 0) {
8726 // This is a uniform branch so we don't need to legalize.
8727 return BRCOND;
8728 }
8729
8730 bool HaveChain = Intr->getOpcode() == ISD::INTRINSIC_VOID ||
8731 Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN;
8732
8733 assert(!SetCC ||
8734 (SetCC->getConstantOperandVal(1) == 1 &&
8735 cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() ==
8736 ISD::SETNE));
8737
8738 // operands of the new intrinsic call
8739 SmallVector<SDValue, 4> Ops;
8740 if (HaveChain)
8741 Ops.push_back(Elt: BRCOND.getOperand(i: 0));
8742
8743 Ops.append(in_start: Intr->op_begin() + (HaveChain ? 2 : 1), in_end: Intr->op_end());
8744 Ops.push_back(Elt: Target);
8745
8746 ArrayRef<EVT> Res(Intr->value_begin() + 1, Intr->value_end());
8747
8748 // build the new intrinsic call
8749 SDNode *Result = DAG.getNode(Opcode: CFNode, DL, VTList: DAG.getVTList(VTs: Res), Ops).getNode();
8750
8751 if (!HaveChain) {
8752 SDValue Ops[] = {SDValue(Result, 0), BRCOND.getOperand(i: 0)};
8753
8754 Result = DAG.getMergeValues(Ops, dl: DL).getNode();
8755 }
8756
8757 if (BR) {
8758 // Give the branch instruction our target
8759 SDValue Ops[] = {BR->getOperand(Num: 0), BRCOND.getOperand(i: 2)};
8760 SDValue NewBR = DAG.getNode(Opcode: ISD::BR, DL, VTList: BR->getVTList(), Ops);
8761 DAG.ReplaceAllUsesWith(From: BR, To: NewBR.getNode());
8762 }
8763
8764 SDValue Chain = SDValue(Result, Result->getNumValues() - 1);
8765
8766 // Copy the intrinsic results to registers
8767 for (unsigned i = 1, e = Intr->getNumValues() - 1; i != e; ++i) {
8768 SDNode *CopyToReg = findUser(Value: SDValue(Intr, i), Opcode: ISD::CopyToReg);
8769 if (!CopyToReg)
8770 continue;
8771
8772 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: CopyToReg->getOperand(Num: 1),
8773 N: SDValue(Result, i - 1), Glue: SDValue());
8774
8775 DAG.ReplaceAllUsesWith(From: SDValue(CopyToReg, 0), To: CopyToReg->getOperand(Num: 0));
8776 }
8777
8778 // Remove the old intrinsic from the chain
8779 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Intr, Intr->getNumValues() - 1),
8780 To: Intr->getOperand(Num: 0));
8781
8782 return Chain;
8783}
8784
8785SDValue SITargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
8786 MVT VT = Op.getSimpleValueType();
8787 SDLoc DL(Op);
8788 // Checking the depth
8789 if (Op.getConstantOperandVal(i: 0) != 0)
8790 return DAG.getConstant(Val: 0, DL, VT);
8791
8792 MachineFunction &MF = DAG.getMachineFunction();
8793 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8794 // Check for kernel and shader functions
8795 if (Info->isEntryFunction())
8796 return DAG.getConstant(Val: 0, DL, VT);
8797
8798 MachineFrameInfo &MFI = MF.getFrameInfo();
8799 // There is a call to @llvm.returnaddress in this function
8800 MFI.setReturnAddressIsTaken(true);
8801
8802 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
8803 // Get the return address reg and mark it as an implicit live-in
8804 Register Reg = MF.addLiveIn(PReg: TRI->getReturnAddressReg(MF),
8805 RC: getRegClassFor(VT, isDivergent: Op.getNode()->isDivergent()));
8806
8807 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg, VT);
8808}
8809
8810SDValue SITargetLowering::LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const {
8811 MachineFunction &MF = DAG.getMachineFunction();
8812 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
8813
8814 // For functions that set up their own stack, select the GET_STACK_BASE
8815 // pseudo.
8816 if (MFI->isBottomOfStack())
8817 return Op;
8818
8819 // For everything else, create a dummy stack object.
8820 int FI = MF.getFrameInfo().CreateFixedObject(Size: 1, SPOffset: 0, /*IsImmutable=*/false);
8821 return DAG.getFrameIndex(FI, VT: Op.getValueType());
8822}
8823
8824SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG, SDValue Op,
8825 const SDLoc &DL, EVT VT) const {
8826 return Op.getValueType().bitsLE(VT)
8827 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
8828 : DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
8829 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
8830}
8831
8832SDValue SITargetLowering::splitFP_ROUNDVectorOp(SDValue Op,
8833 SelectionDAG &DAG) const {
8834 EVT DstVT = Op.getValueType();
8835 unsigned NumElts = DstVT.getVectorNumElements();
8836 assert(NumElts > 2 && isPowerOf2_32(NumElts));
8837
8838 auto [Lo, Hi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
8839
8840 SDLoc DL(Op);
8841 unsigned Opc = Op.getOpcode();
8842 SDValue Flags = Op.getOperand(i: 1);
8843 EVT HalfDstVT =
8844 EVT::getVectorVT(Context&: *DAG.getContext(), VT: DstVT.getScalarType(), NumElements: NumElts / 2);
8845 SDValue OpLo = DAG.getNode(Opcode: Opc, DL, VT: HalfDstVT, N1: Lo, N2: Flags);
8846 SDValue OpHi = DAG.getNode(Opcode: Opc, DL, VT: HalfDstVT, N1: Hi, N2: Flags);
8847
8848 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: DstVT, N1: OpLo, N2: OpHi);
8849}
8850
8851SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
8852 bool IsStrict = Op->isStrictFPOpcode();
8853 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8854 EVT SrcVT = Src.getValueType();
8855 EVT DstVT = Op.getValueType();
8856
8857 if (DstVT.isVectorOf(EltVT: MVT::f16)) {
8858 assert(Subtarget->hasCvtPkF16F32Inst() && "support v_cvt_pk_f16_f32");
8859 if (SrcVT.getScalarType() != MVT::f32)
8860 return SDValue();
8861 return SrcVT == MVT::v2f32 ? Op : splitFP_ROUNDVectorOp(Op, DAG);
8862 }
8863
8864 if (SrcVT.getScalarType() != MVT::f64)
8865 return Op;
8866
8867 SDLoc DL(Op);
8868 if (DstVT == MVT::f16) {
8869 // TODO: Handle strictfp
8870 if (Op.getOpcode() != ISD::FP_ROUND)
8871 return Op;
8872
8873 if (!Subtarget->has16BitInsts()) {
8874 SDValue FpToFp16 = DAG.getNode(Opcode: ISD::FP_TO_FP16, DL, VT: MVT::i32, Operand: Src);
8875 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToFp16);
8876 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f16, Operand: Trunc);
8877 }
8878 if (Op->getFlags().hasApproximateFuncs()) {
8879 SDValue Flags = Op.getOperand(i: 1);
8880 SDValue Src32 = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: MVT::f32, N1: Src, N2: Flags);
8881 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: MVT::f16, N1: Src32, N2: Flags);
8882 }
8883 SDValue FpToFp16 = LowerF64ToF16Safe(Src, DL, DAG);
8884 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToFp16);
8885 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f16, Operand: Trunc);
8886 }
8887
8888 assert(DstVT.getScalarType() == MVT::bf16 &&
8889 "custom lower FP_ROUND for f16 or bf16");
8890 assert(Subtarget->hasBF16ConversionInsts() && "f32 -> bf16 is legal");
8891
8892 // Round-inexact-to-odd f64 to f32, then do the final rounding using the
8893 // hardware f32 -> bf16 instruction.
8894 EVT F32VT = SrcVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
8895 SDValue Rod = expandRoundInexactToOdd(ResultVT: F32VT, Op: Src, DL, DAG);
8896 if (IsStrict) {
8897 return DAG.getNode(
8898 Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {DstVT, MVT::Other},
8899 Ops: {Op.getOperand(i: 0), Rod, DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32)});
8900 }
8901 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: DstVT, N1: Rod,
8902 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
8903}
8904
8905SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
8906 SelectionDAG &DAG) const {
8907 EVT VT = Op.getValueType();
8908 const MachineFunction &MF = DAG.getMachineFunction();
8909 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8910 bool IsIEEEMode = Info->getMode().IEEE;
8911
8912 // FIXME: Assert during selection that this is only selected for
8913 // ieee_mode. Currently a combine can produce the ieee version for non-ieee
8914 // mode functions, but this happens to be OK since it's only done in cases
8915 // where there is known no sNaN.
8916 if (IsIEEEMode && !Subtarget->hasIEEEMinimumMaximumInsts())
8917 return expandFMINNUM_FMAXNUM(N: Op.getNode(), DAG);
8918
8919 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
8920 VT == MVT::v32f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16 ||
8921 VT == MVT::v16bf16 || VT == MVT::v32bf16 || VT == MVT::v4f64 ||
8922 VT == MVT::v8f64 || VT == MVT::v16f64 || VT == MVT::v32f64)
8923 return splitBinaryVectorOp(Op, DAG);
8924 return Op;
8925}
8926
8927SDValue
8928SITargetLowering::lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op,
8929 SelectionDAG &DAG) const {
8930 EVT VT = Op.getValueType();
8931 const MachineFunction &MF = DAG.getMachineFunction();
8932 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8933 bool IsIEEEMode = Info->getMode().IEEE;
8934
8935 if (IsIEEEMode && !Subtarget->hasIEEEMinimumMaximumInsts())
8936 return expandFMINIMUMNUM_FMAXIMUMNUM(N: Op.getNode(), DAG);
8937
8938 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
8939 VT == MVT::v32f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16 ||
8940 VT == MVT::v16bf16 || VT == MVT::v32bf16 || VT == MVT::v4f64 ||
8941 VT == MVT::v8f64 || VT == MVT::v16f64 || VT == MVT::v32f64)
8942 return splitBinaryVectorOp(Op, DAG);
8943 return Op;
8944}
8945
8946SDValue SITargetLowering::lowerFLDEXP(SDValue Op, SelectionDAG &DAG) const {
8947 bool IsStrict = Op.getOpcode() == ISD::STRICT_FLDEXP;
8948 EVT VT = Op.getValueType();
8949 assert(VT == MVT::f16);
8950
8951 SDValue Exp = Op.getOperand(i: IsStrict ? 2 : 1);
8952 EVT ExpVT = Exp.getValueType();
8953 if (ExpVT == MVT::i16)
8954 return Op;
8955
8956 SDLoc DL(Op);
8957
8958 // Correct the exponent type for f16 to i16.
8959 // Clamp the range of the exponent to the instruction's range.
8960
8961 // TODO: This should be a generic narrowing legalization, and can easily be
8962 // for GlobalISel.
8963
8964 SDValue MinExp = DAG.getSignedConstant(Val: minIntN(N: 16), DL, VT: ExpVT);
8965 SDValue ClampMin = DAG.getNode(Opcode: ISD::SMAX, DL, VT: ExpVT, N1: Exp, N2: MinExp);
8966
8967 SDValue MaxExp = DAG.getSignedConstant(Val: maxIntN(N: 16), DL, VT: ExpVT);
8968 SDValue Clamp = DAG.getNode(Opcode: ISD::SMIN, DL, VT: ExpVT, N1: ClampMin, N2: MaxExp);
8969
8970 SDValue TruncExp = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Clamp);
8971
8972 if (IsStrict) {
8973 return DAG.getNode(Opcode: ISD::STRICT_FLDEXP, DL, ResultTys: {VT, MVT::Other},
8974 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), TruncExp});
8975 }
8976
8977 return DAG.getNode(Opcode: ISD::FLDEXP, DL, VT, N1: Op.getOperand(i: 0), N2: TruncExp);
8978}
8979
8980static unsigned getExtOpcodeForPromotedOp(SDValue Op) {
8981 switch (Op->getOpcode()) {
8982 case ISD::ABS:
8983 case ISD::SRA:
8984 case ISD::SMIN:
8985 case ISD::SMAX:
8986 return ISD::SIGN_EXTEND;
8987 case ISD::SRL:
8988 case ISD::UMIN:
8989 case ISD::UMAX:
8990 case ISD::USUBSAT:
8991 case ISD::UADDSAT:
8992 return ISD::ZERO_EXTEND;
8993 case ISD::ADD:
8994 case ISD::SUB:
8995 case ISD::AND:
8996 case ISD::OR:
8997 case ISD::XOR:
8998 case ISD::SHL:
8999 case ISD::SELECT:
9000 case ISD::MUL:
9001 // operation result won't be influenced by garbage high bits.
9002 // TODO: are all of those cases correct, and are there more?
9003 return ISD::ANY_EXTEND;
9004 case ISD::SETCC: {
9005 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
9006 return ISD::isSignedIntSetCC(Code: CC) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
9007 }
9008 default:
9009 llvm_unreachable("unexpected opcode!");
9010 }
9011}
9012
9013SDValue
9014SITargetLowering::promoteUniformUnaryOpToI32(SDValue Op,
9015 DAGCombinerInfo &DCI) const {
9016 EVT OpTy = Op.getValueType();
9017 SelectionDAG &DAG = DCI.DAG;
9018 EVT ExtTy = OpTy.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
9019
9020 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: ExtTy, DestVT: OpTy))
9021 return SDValue();
9022
9023 SDLoc DL(Op);
9024 SDValue Input = Op.getOperand(i: 0);
9025 const unsigned ExtOp = getExtOpcodeForPromotedOp(Op);
9026 Input = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: Input);
9027
9028 SDValue NewVal = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: ExtTy, Operand: Input);
9029
9030 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OpTy, Operand: NewVal);
9031}
9032
9033SDValue SITargetLowering::promoteUniformOpToI32(SDValue Op,
9034 DAGCombinerInfo &DCI) const {
9035 const unsigned Opc = Op.getOpcode();
9036 assert(Opc == ISD::ADD || Opc == ISD::SUB || Opc == ISD::SHL ||
9037 Opc == ISD::SRL || Opc == ISD::SRA || Opc == ISD::AND ||
9038 Opc == ISD::OR || Opc == ISD::XOR || Opc == ISD::MUL ||
9039 Opc == ISD::SETCC || Opc == ISD::SELECT || Opc == ISD::SMIN ||
9040 Opc == ISD::SMAX || Opc == ISD::UMIN || Opc == ISD::UMAX ||
9041 Opc == ISD::USUBSAT || Opc == ISD::UADDSAT);
9042
9043 EVT OpTy = (Opc != ISD::SETCC) ? Op.getValueType()
9044 : Op->getOperand(Num: 0).getValueType();
9045 auto &DAG = DCI.DAG;
9046 auto ExtTy = OpTy.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
9047
9048 if (DCI.isBeforeLegalizeOps() ||
9049 isNarrowingProfitable(N: Op.getNode(), SrcVT: ExtTy, DestVT: OpTy))
9050 return SDValue();
9051
9052 SDLoc DL(Op);
9053 SDValue LHS;
9054 SDValue RHS;
9055 if (Opc == ISD::SELECT) {
9056 LHS = Op->getOperand(Num: 1);
9057 RHS = Op->getOperand(Num: 2);
9058 } else {
9059 LHS = Op->getOperand(Num: 0);
9060 RHS = Op->getOperand(Num: 1);
9061 }
9062
9063 const unsigned ExtOp = getExtOpcodeForPromotedOp(Op);
9064 LHS = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: {LHS});
9065
9066 // Special case: for shifts, the RHS always needs a zext.
9067 if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA)
9068 RHS = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtTy, Operand: {RHS});
9069 else
9070 RHS = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: {RHS});
9071
9072 // setcc always return i1/i1 vec so no need to truncate after.
9073 if (Opc == ISD::SETCC) {
9074 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
9075 return DAG.getSetCC(DL, VT: Op.getValueType(), LHS, RHS, Cond: CC);
9076 }
9077
9078 // For other ops, we extend the operation's return type as well so we need to
9079 // truncate back to the original type.
9080 SDValue NewVal;
9081 if (Opc == ISD::SELECT)
9082 NewVal = DAG.getNode(Opcode: ISD::SELECT, DL, VT: ExtTy, Ops: {Op->getOperand(Num: 0), LHS, RHS});
9083 else if (Opc == ISD::UADDSAT) {
9084 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL, VT: ExtTy, N1: LHS, N2: RHS);
9085 SDValue MaxVal = DAG.getConstant(
9086 Val: APInt::getMaxValue(numBits: OpTy.getScalarSizeInBits()).zext(width: 32), DL, VT: ExtTy);
9087 NewVal = DAG.getNode(Opcode: ISD::UMIN, DL, VT: ExtTy, N1: Sum, N2: MaxVal);
9088 } else
9089 NewVal = DAG.getNode(Opcode: Opc, DL, VT: ExtTy, Ops: {LHS, RHS});
9090
9091 return DAG.getZExtOrTrunc(Op: NewVal, DL, VT: OpTy);
9092}
9093
9094SDValue SITargetLowering::lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
9095 SDValue Mag = Op.getOperand(i: 0);
9096 EVT MagVT = Mag.getValueType();
9097
9098 if (MagVT.getVectorNumElements() > 2)
9099 return splitBinaryVectorOp(Op, DAG);
9100
9101 SDValue Sign = Op.getOperand(i: 1);
9102 EVT SignVT = Sign.getValueType();
9103
9104 if (MagVT == SignVT)
9105 return Op;
9106
9107 // fcopysign v2f16:mag, v2f32:sign ->
9108 // fcopysign v2f16:mag,
9109 // bitcast (trunc (srl (bitcast sign to v2i32), 16) to v2i16)
9110
9111 SDLoc SL(Op);
9112 SDValue SignAsInt32 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Sign);
9113 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: 16, VT: MVT::v2i32, DL: SL);
9114 SDValue SignShifted =
9115 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::v2i32, N1: SignAsInt32, N2: ShiftAmt);
9116 SDValue SignAsInt16 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::v2i16, Operand: SignShifted);
9117
9118 SDValue SignAsHalf16 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MagVT, Operand: SignAsInt16);
9119
9120 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT: MagVT, N1: Mag, N2: SignAsHalf16);
9121}
9122
9123// Custom lowering for vector multiplications and s_mul_u64.
9124SDValue SITargetLowering::lowerMUL(SDValue Op, SelectionDAG &DAG) const {
9125 EVT VT = Op.getValueType();
9126
9127 // Split vector operands.
9128 if (VT.isVector())
9129 return splitBinaryVectorOp(Op, DAG);
9130
9131 assert(VT == MVT::i64 && "The following code is a special for s_mul_u64");
9132
9133 // There are four ways to lower s_mul_u64:
9134 //
9135 // 1. If all the operands are uniform, then we lower it as it is.
9136 //
9137 // 2. If the operands are divergent, then we have to split s_mul_u64 in 32-bit
9138 // multiplications because there is not a vector equivalent of s_mul_u64.
9139 //
9140 // 3. If the cost model decides that it is more efficient to use vector
9141 // registers, then we have to split s_mul_u64 in 32-bit multiplications.
9142 // This happens in splitScalarSMULU64() in SIInstrInfo.cpp .
9143 //
9144 // 4. If the cost model decides to use vector registers and both of the
9145 // operands are zero-extended/sign-extended from 32-bits, then we split the
9146 // s_mul_u64 in two 32-bit multiplications. The problem is that it is not
9147 // possible to check if the operands are zero-extended or sign-extended in
9148 // SIInstrInfo.cpp. For this reason, here, we replace s_mul_u64 with
9149 // s_mul_u64_u32_pseudo if both operands are zero-extended and we replace
9150 // s_mul_u64 with s_mul_i64_i32_pseudo if both operands are sign-extended.
9151 // If the cost model decides that we have to use vector registers, then
9152 // splitScalarSMulPseudo() (in SIInstrInfo.cpp) split s_mul_u64_u32/
9153 // s_mul_i64_i32_pseudo in two vector multiplications. If the cost model
9154 // decides that we should use scalar registers, then s_mul_u64_u32_pseudo/
9155 // s_mul_i64_i32_pseudo is lowered as s_mul_u64 in expandPostRAPseudo() in
9156 // SIInstrInfo.cpp .
9157
9158 if (Op->isDivergent())
9159 return SDValue();
9160
9161 SDValue Op0 = Op.getOperand(i: 0);
9162 SDValue Op1 = Op.getOperand(i: 1);
9163 // If all the operands are zero-enteted to 32-bits, then we replace s_mul_u64
9164 // with s_mul_u64_u32_pseudo. If all the operands are sign-extended to
9165 // 32-bits, then we replace s_mul_u64 with s_mul_i64_i32_pseudo.
9166 KnownBits Op0KnownBits = DAG.computeKnownBits(Op: Op0);
9167 unsigned Op0LeadingZeros = Op0KnownBits.countMinLeadingZeros();
9168 KnownBits Op1KnownBits = DAG.computeKnownBits(Op: Op1);
9169 unsigned Op1LeadingZeros = Op1KnownBits.countMinLeadingZeros();
9170 SDLoc SL(Op);
9171 if (Op0LeadingZeros >= 32 && Op1LeadingZeros >= 32)
9172 return SDValue(
9173 DAG.getMachineNode(Opcode: AMDGPU::S_MUL_U64_U32_PSEUDO, dl: SL, VT, Op1: Op0, Op2: Op1), 0);
9174 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op: Op0);
9175 unsigned Op1SignBits = DAG.ComputeNumSignBits(Op: Op1);
9176 if (Op0SignBits >= 33 && Op1SignBits >= 33)
9177 return SDValue(
9178 DAG.getMachineNode(Opcode: AMDGPU::S_MUL_I64_I32_PSEUDO, dl: SL, VT, Op1: Op0, Op2: Op1), 0);
9179 // If all the operands are uniform, then we lower s_mul_u64 as it is.
9180 return Op;
9181}
9182
9183SDValue SITargetLowering::lowerXMULO(SDValue Op, SelectionDAG &DAG) const {
9184 EVT VT = Op.getValueType();
9185 SDLoc SL(Op);
9186 SDValue LHS = Op.getOperand(i: 0);
9187 SDValue RHS = Op.getOperand(i: 1);
9188 bool isSigned = Op.getOpcode() == ISD::SMULO;
9189
9190 if (ConstantSDNode *RHSC = isConstOrConstSplat(N: RHS)) {
9191 const APInt &C = RHSC->getAPIntValue();
9192 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
9193 if (C.isPowerOf2()) {
9194 // smulo(x, signed_min) is same as umulo(x, signed_min).
9195 bool UseArithShift = isSigned && !C.isMinSignedValue();
9196 SDValue ShiftAmt = DAG.getConstant(Val: C.logBase2(), DL: SL, VT: MVT::i32);
9197 SDValue Result = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT, N1: LHS, N2: ShiftAmt);
9198 SDValue Overflow =
9199 DAG.getSetCC(DL: SL, VT: MVT::i1,
9200 LHS: DAG.getNode(Opcode: UseArithShift ? ISD::SRA : ISD::SRL, DL: SL, VT,
9201 N1: Result, N2: ShiftAmt),
9202 RHS: LHS, Cond: ISD::SETNE);
9203 return DAG.getMergeValues(Ops: {Result, Overflow}, dl: SL);
9204 }
9205 }
9206
9207 SDValue Result = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT, N1: LHS, N2: RHS);
9208 SDValue Top =
9209 DAG.getNode(Opcode: isSigned ? ISD::MULHS : ISD::MULHU, DL: SL, VT, N1: LHS, N2: RHS);
9210
9211 SDValue Sign = isSigned
9212 ? DAG.getNode(Opcode: ISD::SRA, DL: SL, VT, N1: Result,
9213 N2: DAG.getConstant(Val: VT.getScalarSizeInBits() - 1,
9214 DL: SL, VT: MVT::i32))
9215 : DAG.getConstant(Val: 0, DL: SL, VT);
9216 SDValue Overflow = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Top, RHS: Sign, Cond: ISD::SETNE);
9217
9218 return DAG.getMergeValues(Ops: {Result, Overflow}, dl: SL);
9219}
9220
9221SDValue SITargetLowering::lowerXMUL_LOHI(SDValue Op, SelectionDAG &DAG) const {
9222 if (Op->isDivergent()) {
9223 // Select to V_MAD_[IU]64_[IU]32.
9224 return Op;
9225 }
9226 if (Subtarget->hasSMulHi()) {
9227 // Expand to S_MUL_I32 + S_MUL_HI_[IU]32.
9228 return SDValue();
9229 }
9230 // The multiply is uniform but we would have to use V_MUL_HI_[IU]32 to
9231 // calculate the high part, so we might as well do the whole thing with
9232 // V_MAD_[IU]64_[IU]32.
9233 return Op;
9234}
9235
9236SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const {
9237 if (!Subtarget->hasTrapHandler() ||
9238 Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
9239 return lowerTrapEndpgm(Op, DAG);
9240
9241 return Subtarget->supportsGetDoorbellID() ? lowerTrapHsa(Op, DAG)
9242 : lowerTrapHsaQueuePtr(Op, DAG);
9243}
9244
9245SDValue SITargetLowering::lowerTrapEndpgm(SDValue Op, SelectionDAG &DAG) const {
9246 SDLoc SL(Op);
9247 SDValue Chain = Op.getOperand(i: 0);
9248 return DAG.getNode(Opcode: AMDGPUISD::ENDPGM_TRAP, DL: SL, VT: MVT::Other, Operand: Chain);
9249}
9250
9251SDValue
9252SITargetLowering::loadImplicitKernelArgument(SelectionDAG &DAG, MVT VT,
9253 const SDLoc &DL, Align Alignment,
9254 ImplicitParameter Param) const {
9255 MachineFunction &MF = DAG.getMachineFunction();
9256 uint64_t Offset = getImplicitParameterOffset(MF, Param);
9257 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL: DL, Chain: DAG.getEntryNode(), Offset);
9258 MachinePointerInfo PtrInfo =
9259 getKernargSegmentPtrInfo(MF&: DAG.getMachineFunction());
9260 return DAG.getLoad(
9261 VT, dl: DL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: PtrInfo.getWithOffset(O: Offset), Alignment,
9262 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
9263}
9264
9265SDValue SITargetLowering::lowerTrapHsaQueuePtr(SDValue Op,
9266 SelectionDAG &DAG) const {
9267 SDLoc SL(Op);
9268 SDValue Chain = Op.getOperand(i: 0);
9269
9270 SDValue QueuePtr;
9271 // For code object version 5, QueuePtr is passed through implicit kernarg.
9272 const Module *M = DAG.getMachineFunction().getFunction().getParent();
9273 if (AMDGPU::getAMDHSACodeObjectVersion(M: *M) >= AMDGPU::AMDHSA_COV5) {
9274 QueuePtr =
9275 loadImplicitKernelArgument(DAG, VT: MVT::i64, DL: SL, Alignment: Align(8), Param: QUEUE_PTR);
9276 } else {
9277 MachineFunction &MF = DAG.getMachineFunction();
9278 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
9279 Register UserSGPR = Info->getQueuePtrUserSGPR();
9280
9281 if (UserSGPR == AMDGPU::NoRegister) {
9282 // We probably are in a function incorrectly marked with
9283 // amdgpu-no-queue-ptr. This is undefined. We don't want to delete the
9284 // trap, so just use a null pointer.
9285 QueuePtr = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i64);
9286 } else {
9287 QueuePtr = CreateLiveInRegister(DAG, RC: &AMDGPU::SReg_64RegClass, Reg: UserSGPR,
9288 VT: MVT::i64);
9289 }
9290 }
9291
9292 SDValue SGPR01 = DAG.getRegister(Reg: AMDGPU::SGPR0_SGPR1, VT: MVT::i64);
9293 SDValue ToReg = DAG.getCopyToReg(Chain, dl: SL, Reg: SGPR01, N: QueuePtr, Glue: SDValue());
9294
9295 uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap);
9296 SDValue Ops[] = {ToReg, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16), SGPR01,
9297 ToReg.getValue(R: 1)};
9298 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
9299}
9300
9301SDValue SITargetLowering::lowerTrapHsa(SDValue Op, SelectionDAG &DAG) const {
9302 SDLoc SL(Op);
9303 SDValue Chain = Op.getOperand(i: 0);
9304
9305 // We need to simulate the 's_trap 2' instruction on targets that run in
9306 // PRIV=1 (where it is treated as a nop).
9307 if (Subtarget->hasPrivEnabledTrap2NopBug())
9308 return DAG.getNode(Opcode: AMDGPUISD::SIMULATED_TRAP, DL: SL, VT: MVT::Other, Operand: Chain);
9309
9310 uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap);
9311 SDValue Ops[] = {Chain, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16)};
9312 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
9313}
9314
9315SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const {
9316 SDLoc SL(Op);
9317 SDValue Chain = Op.getOperand(i: 0);
9318 MachineFunction &MF = DAG.getMachineFunction();
9319
9320 if (!Subtarget->hasTrapHandler() ||
9321 Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) {
9322 LLVMContext &Ctx = MF.getFunction().getContext();
9323 Ctx.diagnose(DI: DiagnosticInfoUnsupported(MF.getFunction(),
9324 "debugtrap handler not supported",
9325 Op.getDebugLoc(), DS_Warning));
9326 return Chain;
9327 }
9328
9329 uint64_t TrapID =
9330 static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap);
9331 SDValue Ops[] = {Chain, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16)};
9332 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
9333}
9334
9335/// When a divergent value (in VGPR) is passed to an inline asm with an SGPR
9336/// constraint ('s'), we need to insert v_readfirstlane to move the value from
9337/// VGPR to SGPR. This is done by modifying the CopyToReg nodes in the glue
9338/// chain that feed into the INLINEASM node.
9339SDValue SITargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
9340 unsigned NumOps = Op.getNumOperands();
9341
9342 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
9343 SmallSet<Register, 8> SGPRInputRegs;
9344
9345 unsigned NumVals = 0;
9346 for (unsigned I = InlineAsm::Op_FirstOperand; I < NumOps - 1;
9347 I += 1 + NumVals) {
9348 const InlineAsm::Flag Flags(Op.getConstantOperandVal(i: I));
9349 NumVals = Flags.getNumOperandRegisters();
9350
9351 unsigned RCID;
9352 bool IsSGPRInput = Flags.getKind() == InlineAsm::Kind::RegUse &&
9353 NumVals > 0 && Flags.hasRegClassConstraint(RC&: RCID) &&
9354 TRI->isSGPRClass(RC: TRI->getRegClass(i: RCID));
9355
9356 for (unsigned J = 0; J < NumVals; ++J) {
9357 SDValue Val = Op.getOperand(i: I + 1 + J);
9358 if (const RegisterSDNode *RegNode =
9359 dyn_cast<RegisterSDNode>(Val: Val.getNode())) {
9360 Register Reg = RegNode->getReg();
9361 if (IsSGPRInput || (Reg.isPhysical() && TRI->isSGPRPhysReg(Reg)))
9362 SGPRInputRegs.insert(V: Reg);
9363 }
9364 }
9365 }
9366
9367 if (SGPRInputRegs.empty())
9368 return Op;
9369
9370 // Walk the glue chain and insert readfirstlane for divergent SGPR inputs.
9371 SDLoc DL(Op);
9372 SDNode *N = Op.getOperand(i: NumOps - 1).getNode();
9373
9374 while (N && N->getOpcode() == ISD::CopyToReg) {
9375 Register Reg = cast<RegisterSDNode>(Val: N->getOperand(Num: 1))->getReg();
9376 SDValue SrcVal = N->getOperand(Num: 2);
9377
9378 // Insert readfirstlane if copying a divergent value to an SGPR input.
9379 if (SrcVal->isDivergent() && SGPRInputRegs.count(V: Reg)) {
9380 SDValue ReadFirstLaneID =
9381 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
9382 SDValue ReadFirstLane =
9383 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: SrcVal.getValueType(),
9384 N1: ReadFirstLaneID, N2: SrcVal);
9385
9386 SmallVector<SDValue, 4> Ops = {N->getOperand(Num: 0), N->getOperand(Num: 1),
9387 ReadFirstLane};
9388 if (N->getNumOperands() > 3)
9389 Ops.push_back(Elt: N->getOperand(Num: 3)); // Glue input
9390
9391 DAG.UpdateNodeOperands(N, Ops);
9392 }
9393
9394 // Follow glue chain to next CopyToReg.
9395 SDNode *Next = nullptr;
9396 for (unsigned I = 0, E = N->getNumOperands(); I != E; ++I) {
9397 if (N->getOperand(Num: I).getValueType() == MVT::Glue) {
9398 Next = N->getOperand(Num: I).getNode();
9399 break;
9400 }
9401 }
9402 N = Next;
9403 }
9404
9405 return Op;
9406}
9407
9408SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL,
9409 SelectionDAG &DAG) const {
9410 unsigned BaseAS = AS;
9411 unsigned SANum = AMDGPU::getSyntheticApertureNumber(AS);
9412 if (SANum != AMDGPU::SyntheticAperture::None)
9413 BaseAS = AMDGPUAS::LOCAL_ADDRESS;
9414
9415 SDValue Aperture = getBaseSegmentAperture(AS: BaseAS, DL, DAG);
9416
9417 if (SANum != AMDGPU::SyntheticAperture::None) {
9418 SDValue Tag = DAG.getConstant(Val: SANum, DL, VT: MVT::i32);
9419 return DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Aperture, N2: Tag);
9420 }
9421
9422 return Aperture;
9423}
9424
9425SDValue SITargetLowering::getBaseSegmentAperture(unsigned AS, const SDLoc &DL,
9426 SelectionDAG &DAG) const {
9427 const bool IsLDS = (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::BARRIER);
9428
9429 if (Subtarget->hasApertureRegs()) {
9430 const unsigned ApertureRegNo =
9431 IsLDS ? AMDGPU::SRC_SHARED_BASE : AMDGPU::SRC_PRIVATE_BASE;
9432 assert((ApertureRegNo != AMDGPU::SRC_PRIVATE_BASE ||
9433 !Subtarget->hasGloballyAddressableScratch()) &&
9434 "Cannot use src_private_base with globally addressable scratch!");
9435 // Note: this feature (register) is broken. When used as a 32-bit operand,
9436 // it returns a wrong value (all zeroes?). The real value is in the upper 32
9437 // bits.
9438 //
9439 // To work around the issue, emit a 64 bit copy from this register
9440 // then extract the high bits. Note that this shouldn't even result in a
9441 // shift being emitted and simply become a pair of registers (e.g.):
9442 // s_mov_b64 s[6:7], src_shared_base
9443 // v_mov_b32_e32 v1, s7
9444 SDValue Copy =
9445 DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: ApertureRegNo, VT: MVT::v2i32);
9446 return DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: Copy, Idx: 1);
9447 }
9448
9449 // For code object version 5, private_base and shared_base are passed through
9450 // implicit kernargs.
9451 const Module *M = DAG.getMachineFunction().getFunction().getParent();
9452 if (AMDGPU::getAMDHSACodeObjectVersion(M: *M) >= AMDGPU::AMDHSA_COV5) {
9453 ImplicitParameter Param = IsLDS ? SHARED_BASE : PRIVATE_BASE;
9454 return loadImplicitKernelArgument(DAG, VT: MVT::i32, DL, Alignment: Align(4), Param);
9455 }
9456
9457 MachineFunction &MF = DAG.getMachineFunction();
9458 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
9459 Register UserSGPR = Info->getQueuePtrUserSGPR();
9460 if (UserSGPR == AMDGPU::NoRegister) {
9461 // We probably are in a function incorrectly marked with
9462 // amdgpu-no-queue-ptr. This is undefined.
9463 return DAG.getPOISON(VT: MVT::i32);
9464 }
9465
9466 SDValue QueuePtr =
9467 CreateLiveInRegister(DAG, RC: &AMDGPU::SReg_64RegClass, Reg: UserSGPR, VT: MVT::i64);
9468
9469 // Offset into amd_queue_t for group_segment_aperture_base_hi /
9470 // private_segment_aperture_base_hi.
9471 uint32_t StructOffset = IsLDS ? 0x40 : 0x44;
9472
9473 SDValue Ptr =
9474 DAG.getObjectPtrOffset(SL: DL, Ptr: QueuePtr, Offset: TypeSize::getFixed(ExactSize: StructOffset));
9475
9476 // TODO: Use custom target PseudoSourceValue.
9477 // TODO: We should use the value from the IR intrinsic call, but it might not
9478 // be available and how do we get it?
9479 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
9480 return DAG.getLoad(VT: MVT::i32, dl: DL, Chain: QueuePtr.getValue(R: 1), Ptr, PtrInfo,
9481 Alignment: commonAlignment(A: Align(64), Offset: StructOffset),
9482 MMOFlags: MachineMemOperand::MODereferenceable |
9483 MachineMemOperand::MOInvariant);
9484}
9485
9486/// Return true if the value is a known valid address, such that a null check is
9487/// not necessary.
9488static bool isKnownNonNull(SDValue Val, SelectionDAG &DAG,
9489 const AMDGPUTargetMachine &TM, unsigned AddrSpace) {
9490 if (isa<FrameIndexSDNode, GlobalAddressSDNode, BasicBlockSDNode>(Val))
9491 return true;
9492
9493 if (auto *ConstVal = dyn_cast<ConstantSDNode>(Val))
9494 return ConstVal->getSExtValue() != AMDGPU::getNullPointerValue(AS: AddrSpace);
9495
9496 // TODO: Search through arithmetic, handle arguments and loads
9497 // marked nonnull.
9498 return false;
9499}
9500
9501SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op,
9502 SelectionDAG &DAG) const {
9503 SDLoc SL(Op);
9504
9505 const AMDGPUTargetMachine &TM =
9506 static_cast<const AMDGPUTargetMachine &>(getTargetMachine());
9507
9508 const auto *ASC = cast<AddrSpaceCastSDNode>(Val&: Op);
9509 unsigned SrcAS = ASC->getSrcAddressSpace();
9510 SDValue Src = ASC->getOperand(Num: 0);
9511 unsigned DestAS = ASC->getDestAddressSpace();
9512 bool IsNonNull = ASC->getFlags().hasNonNull();
9513
9514 SDValue FlatNullPtr = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i64);
9515
9516 // flat -> local/private/barrier
9517 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
9518 if (DestAS == AMDGPUAS::LOCAL_ADDRESS ||
9519 DestAS == AMDGPUAS::PRIVATE_ADDRESS || DestAS == AMDGPUAS::BARRIER) {
9520 SDValue Ptr = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Src);
9521
9522 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
9523 Subtarget->hasGloballyAddressableScratch()) {
9524 // flat -> private with globally addressable scratch: subtract
9525 // src_flat_scratch_base_lo.
9526 SDValue FlatScratchBaseLo(
9527 DAG.getMachineNode(
9528 Opcode: AMDGPU::S_MOV_B32, dl: SL, VT: MVT::i32,
9529 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO, VT: MVT::i32)),
9530 0);
9531 Ptr = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: Ptr, N2: FlatScratchBaseLo);
9532 }
9533
9534 if (IsNonNull || isKnownNonNull(Val: Op, DAG, TM, AddrSpace: SrcAS))
9535 return Ptr;
9536
9537 unsigned NullVal = AMDGPU::getNullPointerValue(AS: DestAS);
9538 SDValue SegmentNullPtr = DAG.getConstant(Val: NullVal, DL: SL, VT: MVT::i32);
9539 SDValue NonNull = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Src, RHS: FlatNullPtr, Cond: ISD::SETNE);
9540
9541 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i32, N1: NonNull, N2: Ptr,
9542 N3: SegmentNullPtr);
9543 }
9544 }
9545
9546 // local/private/barrier -> flat
9547 if (DestAS == AMDGPUAS::FLAT_ADDRESS) {
9548 if (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
9549 SrcAS == AMDGPUAS::PRIVATE_ADDRESS || SrcAS == AMDGPUAS::BARRIER) {
9550 SDValue CvtPtr;
9551 if (SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
9552 Subtarget->hasGloballyAddressableScratch()) {
9553 // For wave32: Addr = (TID[4:0] << 52) + FLAT_SCRATCH_BASE + privateAddr
9554 // For wave64: Addr = (TID[5:0] << 51) + FLAT_SCRATCH_BASE + privateAddr
9555 SDValue AllOnes = DAG.getSignedTargetConstant(Val: -1, DL: SL, VT: MVT::i32);
9556 SDValue ThreadID = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
9557 ThreadID = DAG.getNode(
9558 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
9559 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_mbcnt_lo, DL: SL, VT: MVT::i32),
9560 N2: AllOnes, N3: ThreadID);
9561 if (Subtarget->isWave64())
9562 ThreadID = DAG.getNode(
9563 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
9564 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_mbcnt_hi, DL: SL, VT: MVT::i32),
9565 N2: AllOnes, N3: ThreadID);
9566 SDValue ShAmt = DAG.getShiftAmountConstant(
9567 Val: 57 - 32 - Subtarget->getWavefrontSizeLog2(), VT: MVT::i32, DL: SL);
9568 SDValue SrcHi = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: ThreadID, N2: ShAmt);
9569 CvtPtr = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: SrcHi);
9570 CvtPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
9571 // Accessing src_flat_scratch_base_lo as a 64-bit operand gives the full
9572 // 64-bit hi:lo value.
9573 SDValue FlatScratchBase = {
9574 DAG.getMachineNode(
9575 Opcode: AMDGPU::S_MOV_B64, dl: SL, VT: MVT::i64,
9576 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE, VT: MVT::i64)),
9577 0};
9578 CvtPtr = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i64, N1: CvtPtr, N2: FlatScratchBase);
9579 } else {
9580 SDValue Aperture = getSegmentAperture(AS: SrcAS, DL: SL, DAG);
9581
9582 CvtPtr = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: Aperture);
9583 CvtPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
9584 }
9585
9586 if (IsNonNull || isKnownNonNull(Val: Op, DAG, TM, AddrSpace: SrcAS))
9587 return CvtPtr;
9588
9589 unsigned NullVal = AMDGPU::getNullPointerValue(AS: SrcAS);
9590 SDValue SegmentNullPtr = DAG.getConstant(Val: NullVal, DL: SL, VT: MVT::i32);
9591
9592 SDValue NonNull =
9593 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Src, RHS: SegmentNullPtr, Cond: ISD::SETNE);
9594
9595 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: NonNull, N2: CvtPtr,
9596 N3: FlatNullPtr);
9597 }
9598 }
9599
9600 if (SrcAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
9601 Op.getValueType() == MVT::i64) {
9602 const SIMachineFunctionInfo *Info =
9603 DAG.getMachineFunction().getInfo<SIMachineFunctionInfo>();
9604 if (Info->get32BitAddressHighBits() == 0)
9605 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i64, Operand: Src);
9606
9607 SDValue Hi = DAG.getConstant(Val: Info->get32BitAddressHighBits(), DL: SL, VT: MVT::i32);
9608 SDValue Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: Hi);
9609 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
9610 }
9611
9612 if (DestAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
9613 Src.getValueType() == MVT::i64)
9614 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Src);
9615
9616 // global <-> flat are no-ops and never emitted.
9617
9618 // Invalid casts are poison.
9619 return DAG.getPOISON(VT: Op->getValueType(ResNo: 0));
9620}
9621
9622// This lowers an INSERT_SUBVECTOR by extracting the individual elements from
9623// the small vector and inserting them into the big vector. That is better than
9624// the default expansion of doing it via a stack slot. Even though the use of
9625// the stack slot would be optimized away afterwards, the stack slot itself
9626// remains.
9627SDValue SITargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
9628 SelectionDAG &DAG) const {
9629 SDValue Vec = Op.getOperand(i: 0);
9630 SDValue Ins = Op.getOperand(i: 1);
9631 SDValue Idx = Op.getOperand(i: 2);
9632 EVT VecVT = Vec.getValueType();
9633 EVT InsVT = Ins.getValueType();
9634 EVT EltVT = VecVT.getVectorElementType();
9635 unsigned InsNumElts = InsVT.getVectorNumElements();
9636 unsigned IdxVal = Idx->getAsZExtVal();
9637 SDLoc SL(Op);
9638
9639 if (EltVT.getScalarSizeInBits() == 16 && IdxVal % 2 == 0) {
9640 // Insert 32-bit registers at a time.
9641 assert(InsNumElts % 2 == 0 && "expect legal vector types");
9642
9643 unsigned VecNumElts = VecVT.getVectorNumElements();
9644 EVT NewVecVT =
9645 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: VecNumElts / 2);
9646 EVT NewInsVT = InsNumElts == 2 ? MVT::i32
9647 : EVT::getVectorVT(Context&: *DAG.getContext(),
9648 VT: MVT::i32, NumElements: InsNumElts / 2);
9649
9650 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVecVT, Operand: Vec);
9651 Ins = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewInsVT, Operand: Ins);
9652
9653 for (unsigned I = 0; I != InsNumElts / 2; ++I) {
9654 SDValue Elt;
9655 if (InsNumElts == 2) {
9656 Elt = Ins;
9657 } else {
9658 Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Ins,
9659 N2: DAG.getConstant(Val: I, DL: SL, VT: MVT::i32));
9660 }
9661 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NewVecVT, N1: Vec, N2: Elt,
9662 N3: DAG.getConstant(Val: IdxVal / 2 + I, DL: SL, VT: MVT::i32));
9663 }
9664
9665 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: Vec);
9666 }
9667
9668 for (unsigned I = 0; I != InsNumElts; ++I) {
9669 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Ins,
9670 N2: DAG.getConstant(Val: I, DL: SL, VT: MVT::i32));
9671 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: VecVT, N1: Vec, N2: Elt,
9672 N3: DAG.getConstant(Val: IdxVal + I, DL: SL, VT: MVT::i32));
9673 }
9674 return Vec;
9675}
9676
9677SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
9678 SelectionDAG &DAG) const {
9679 SDValue Vec = Op.getOperand(i: 0);
9680 SDValue InsVal = Op.getOperand(i: 1);
9681 SDValue Idx = Op.getOperand(i: 2);
9682 EVT VecVT = Vec.getValueType();
9683 EVT EltVT = VecVT.getVectorElementType();
9684 unsigned VecSize = VecVT.getSizeInBits();
9685 unsigned EltSize = EltVT.getSizeInBits();
9686 SDLoc SL(Op);
9687
9688 // Specially handle the case of v4i16 with static indexing.
9689 unsigned NumElts = VecVT.getVectorNumElements();
9690 auto *KIdx = dyn_cast<ConstantSDNode>(Val&: Idx);
9691 if (NumElts == 4 && EltSize == 16 && KIdx) {
9692 SDValue BCVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Vec);
9693
9694 SDValue LoHalf = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: BCVec,
9695 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
9696 SDValue HiHalf = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: BCVec,
9697 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
9698
9699 SDValue LoVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: LoHalf);
9700 SDValue HiVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: HiHalf);
9701
9702 unsigned Idx = KIdx->getZExtValue();
9703 bool InsertLo = Idx < 2;
9704 SDValue InsHalf = DAG.getNode(
9705 Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: MVT::v2i16, N1: InsertLo ? LoVec : HiVec,
9706 N2: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: InsVal),
9707 N3: DAG.getConstant(Val: InsertLo ? Idx : (Idx - 2), DL: SL, VT: MVT::i32));
9708
9709 InsHalf = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: InsHalf);
9710
9711 SDValue Concat =
9712 InsertLo ? DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {InsHalf, HiHalf})
9713 : DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {LoHalf, InsHalf});
9714
9715 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: Concat);
9716 }
9717
9718 // Static indexing does not lower to stack access, and hence there is no need
9719 // for special custom lowering to avoid stack access.
9720 if (isa<ConstantSDNode>(Val: Idx))
9721 return SDValue();
9722
9723 // Avoid stack access for dynamic indexing by custom lowering to
9724 // v_bfi_b32 (v_bfm_b32 16, (shl idx, 16)), val, vec
9725
9726 assert(VecSize <= 64 && "Expected target vector size to be <= 64 bits");
9727
9728 MVT IntVT = MVT::getIntegerVT(BitWidth: VecSize);
9729
9730 // Convert vector index to bit-index and get the required bit mask.
9731 assert(isPowerOf2_32(EltSize));
9732 const auto EltMask = maskTrailingOnes<uint64_t>(N: EltSize);
9733 SDValue ScaleFactor = DAG.getConstant(Val: Log2_32(Value: EltSize), DL: SL, VT: MVT::i32);
9734 SDValue ScaledIdx = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Idx, N2: ScaleFactor);
9735 SDValue BFM = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: IntVT,
9736 N1: DAG.getConstant(Val: EltMask, DL: SL, VT: IntVT), N2: ScaledIdx);
9737
9738 // 1. Create a congruent vector with the target value in each element.
9739 SDValue ExtVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT,
9740 Operand: DAG.getSplatBuildVector(VT: VecVT, DL: SL, Op: InsVal));
9741
9742 // 2. Mask off all other indices except the required index within (1).
9743 SDValue LHS = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IntVT, N1: BFM, N2: ExtVal);
9744
9745 // 3. Mask off the required index within the target vector.
9746 SDValue BCVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT, Operand: Vec);
9747 SDValue RHS =
9748 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IntVT, N1: DAG.getNOT(DL: SL, Val: BFM, VT: IntVT), N2: BCVec);
9749
9750 // 4. Get (2) and (3) ORed into the target vector.
9751 SDValue BFI =
9752 DAG.getNode(Opcode: ISD::OR, DL: SL, VT: IntVT, N1: LHS, N2: RHS, Flags: SDNodeFlags::Disjoint);
9753
9754 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: BFI);
9755}
9756
9757SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
9758 SelectionDAG &DAG) const {
9759 SDLoc SL(Op);
9760
9761 EVT ResultVT = Op.getValueType();
9762 SDValue Vec = Op.getOperand(i: 0);
9763 SDValue Idx = Op.getOperand(i: 1);
9764 EVT VecVT = Vec.getValueType();
9765 unsigned VecSize = VecVT.getSizeInBits();
9766 EVT EltVT = VecVT.getVectorElementType();
9767
9768 DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr);
9769
9770 // Make sure we do any optimizations that will make it easier to fold
9771 // source modifiers before obscuring it with bit operations.
9772
9773 // XXX - Why doesn't this get called when vector_shuffle is expanded?
9774 if (SDValue Combined = performExtractVectorEltCombine(N: Op.getNode(), DCI))
9775 return Combined;
9776
9777 if (VecSize == 128 || VecSize == 256 || VecSize == 512) {
9778 SDValue Lo, Hi;
9779 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
9780
9781 if (VecSize == 128) {
9782 SDValue V2 = DAG.getBitcast(VT: MVT::v2i64, V: Vec);
9783 Lo = DAG.getBitcast(VT: LoVT,
9784 V: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9785 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32)));
9786 Hi = DAG.getBitcast(VT: HiVT,
9787 V: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9788 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32)));
9789 } else if (VecSize == 256) {
9790 SDValue V2 = DAG.getBitcast(VT: MVT::v4i64, V: Vec);
9791 SDValue Parts[4];
9792 for (unsigned P = 0; P < 4; ++P) {
9793 Parts[P] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9794 N2: DAG.getConstant(Val: P, DL: SL, VT: MVT::i32));
9795 }
9796
9797 Lo = DAG.getBitcast(VT: LoVT, V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i64,
9798 N1: Parts[0], N2: Parts[1]));
9799 Hi = DAG.getBitcast(VT: HiVT, V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i64,
9800 N1: Parts[2], N2: Parts[3]));
9801 } else {
9802 assert(VecSize == 512);
9803
9804 SDValue V2 = DAG.getBitcast(VT: MVT::v8i64, V: Vec);
9805 SDValue Parts[8];
9806 for (unsigned P = 0; P < 8; ++P) {
9807 Parts[P] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9808 N2: DAG.getConstant(Val: P, DL: SL, VT: MVT::i32));
9809 }
9810
9811 Lo = DAG.getBitcast(VT: LoVT,
9812 V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v4i64,
9813 N1: Parts[0], N2: Parts[1], N3: Parts[2], N4: Parts[3]));
9814 Hi = DAG.getBitcast(VT: HiVT,
9815 V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v4i64,
9816 N1: Parts[4], N2: Parts[5], N3: Parts[6], N4: Parts[7]));
9817 }
9818
9819 EVT IdxVT = Idx.getValueType();
9820 unsigned NElem = VecVT.getVectorNumElements();
9821 assert(isPowerOf2_32(NElem));
9822 SDValue IdxMask = DAG.getConstant(Val: NElem / 2 - 1, DL: SL, VT: IdxVT);
9823 SDValue NewIdx = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IdxVT, N1: Idx, N2: IdxMask);
9824 SDValue Half = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IdxMask, True: Hi, False: Lo, Cond: ISD::SETUGT);
9825 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Half, N2: NewIdx);
9826 }
9827
9828 assert(VecSize <= 64);
9829
9830 MVT IntVT = MVT::getIntegerVT(BitWidth: VecSize);
9831
9832 // If Vec is just a SCALAR_TO_VECTOR, then use the scalar integer directly.
9833 SDValue VecBC = peekThroughBitcasts(V: Vec);
9834 if (VecBC.getOpcode() == ISD::SCALAR_TO_VECTOR) {
9835 SDValue Src = VecBC.getOperand(i: 0);
9836 Src = DAG.getBitcast(VT: Src.getValueType().changeTypeToInteger(), V: Src);
9837 Vec = DAG.getAnyExtOrTrunc(Op: Src, DL: SL, VT: IntVT);
9838 }
9839
9840 unsigned EltSize = EltVT.getSizeInBits();
9841 assert(isPowerOf2_32(EltSize));
9842
9843 SDValue ScaleFactor = DAG.getConstant(Val: Log2_32(Value: EltSize), DL: SL, VT: MVT::i32);
9844
9845 // Convert vector index to bit-index (* EltSize)
9846 SDValue ScaledIdx = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Idx, N2: ScaleFactor);
9847
9848 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT, Operand: Vec);
9849 SDValue Elt = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: IntVT, N1: BC, N2: ScaledIdx);
9850
9851 if (ResultVT == MVT::f16 || ResultVT == MVT::bf16) {
9852 SDValue Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i16, Operand: Elt);
9853 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ResultVT, Operand: Result);
9854 }
9855
9856 return DAG.getAnyExtOrTrunc(Op: Elt, DL: SL, VT: ResultVT);
9857}
9858
9859static bool elementPairIsContiguous(ArrayRef<int> Mask, int Elt) {
9860 assert(Elt % 2 == 0);
9861 return Mask[Elt + 1] == Mask[Elt] + 1 && (Mask[Elt] % 2 == 0);
9862}
9863
9864static bool elementPairIsOddToEven(ArrayRef<int> Mask, int Elt) {
9865 assert(Elt % 2 == 0);
9866 return Mask[Elt] >= 0 && Mask[Elt + 1] >= 0 && (Mask[Elt] & 1) &&
9867 !(Mask[Elt + 1] & 1);
9868}
9869
9870SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
9871 SelectionDAG &DAG) const {
9872 SDLoc SL(Op);
9873 EVT ResultVT = Op.getValueType();
9874 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
9875 MVT EltVT = ResultVT.getVectorElementType().getSimpleVT();
9876 const int NewSrcNumElts = 2;
9877 MVT PackVT = MVT::getVectorVT(VT: EltVT, NumElements: NewSrcNumElts);
9878 int SrcNumElts = Op.getOperand(i: 0).getValueType().getVectorNumElements();
9879
9880 // Break up the shuffle into registers sized pieces.
9881 //
9882 // We're trying to form sub-shuffles that the register allocation pipeline
9883 // won't be able to figure out, like how to use v_pk_mov_b32 to do a register
9884 // blend or 16-bit op_sel. It should be able to figure out how to reassemble a
9885 // pair of copies into a consecutive register copy, so use the ordinary
9886 // extract_vector_elt lowering unless we can use the shuffle.
9887 //
9888 // TODO: This is a bit of hack, and we should probably always use
9889 // extract_subvector for the largest possible subvector we can (or at least
9890 // use it for PackVT aligned pieces). However we have worse support for
9891 // combines on them don't directly treat extract_subvector / insert_subvector
9892 // as legal. The DAG scheduler also ends up doing a worse job with the
9893 // extract_subvectors.
9894 const bool ShouldUseConsecutiveExtract = EltVT.getSizeInBits() == 16;
9895
9896 // vector_shuffle <0,1,6,7> lhs, rhs
9897 // -> concat_vectors (extract_subvector lhs, 0), (extract_subvector rhs, 2)
9898 //
9899 // vector_shuffle <6,7,2,3> lhs, rhs
9900 // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 2)
9901 //
9902 // vector_shuffle <6,7,0,1> lhs, rhs
9903 // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 0)
9904
9905 // Avoid scalarizing when both halves are reading from consecutive elements.
9906
9907 // If we're treating 2 element shuffles as legal, also create odd-to-even
9908 // shuffles of neighboring pairs.
9909 //
9910 // vector_shuffle <3,2,7,6> lhs, rhs
9911 // -> concat_vectors vector_shuffle <1, 0> (extract_subvector lhs, 0)
9912 // vector_shuffle <1, 0> (extract_subvector rhs, 2)
9913
9914 SmallVector<SDValue, 16> Pieces;
9915 for (int I = 0, N = ResultVT.getVectorNumElements(); I != N; I += 2) {
9916 if (ShouldUseConsecutiveExtract &&
9917 elementPairIsContiguous(Mask: SVN->getMask(), Elt: I)) {
9918 const int Idx = SVN->getMaskElt(Idx: I);
9919 int VecIdx = Idx < SrcNumElts ? 0 : 1;
9920 int EltIdx = Idx < SrcNumElts ? Idx : Idx - SrcNumElts;
9921 SDValue SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT,
9922 N1: SVN->getOperand(Num: VecIdx),
9923 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
9924 Pieces.push_back(Elt: SubVec);
9925 } else if (elementPairIsOddToEven(Mask: SVN->getMask(), Elt: I) &&
9926 isOperationLegal(Op: ISD::VECTOR_SHUFFLE, VT: PackVT)) {
9927 int Idx0 = SVN->getMaskElt(Idx: I);
9928 int Idx1 = SVN->getMaskElt(Idx: I + 1);
9929
9930 SDValue SrcOp0 = SVN->getOperand(Num: 0);
9931 SDValue SrcOp1 = SrcOp0;
9932 if (Idx0 >= SrcNumElts) {
9933 SrcOp0 = SVN->getOperand(Num: 1);
9934 Idx0 -= SrcNumElts;
9935 }
9936
9937 if (Idx1 >= SrcNumElts) {
9938 SrcOp1 = SVN->getOperand(Num: 1);
9939 Idx1 -= SrcNumElts;
9940 }
9941
9942 int AlignedIdx0 = Idx0 & ~(NewSrcNumElts - 1);
9943 int AlignedIdx1 = Idx1 & ~(NewSrcNumElts - 1);
9944
9945 // Extract nearest even aligned piece.
9946 SDValue SubVec0 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT, N1: SrcOp0,
9947 N2: DAG.getConstant(Val: AlignedIdx0, DL: SL, VT: MVT::i32));
9948 SDValue SubVec1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT, N1: SrcOp1,
9949 N2: DAG.getConstant(Val: AlignedIdx1, DL: SL, VT: MVT::i32));
9950
9951 int NewMaskIdx0 = Idx0 - AlignedIdx0;
9952 int NewMaskIdx1 = Idx1 - AlignedIdx1;
9953
9954 SDValue Result0 = SubVec0;
9955 SDValue Result1 = SubVec0;
9956
9957 if (SubVec0 != SubVec1) {
9958 NewMaskIdx1 += NewSrcNumElts;
9959 Result1 = SubVec1;
9960 } else {
9961 Result1 = DAG.getPOISON(VT: PackVT);
9962 }
9963
9964 SDValue Shuf = DAG.getVectorShuffle(VT: PackVT, dl: SL, N1: Result0, N2: Result1,
9965 Mask: {NewMaskIdx0, NewMaskIdx1});
9966 Pieces.push_back(Elt: Shuf);
9967 } else {
9968 const int Idx0 = SVN->getMaskElt(Idx: I);
9969 const int Idx1 = SVN->getMaskElt(Idx: I + 1);
9970 int VecIdx0 = Idx0 < SrcNumElts ? 0 : 1;
9971 int VecIdx1 = Idx1 < SrcNumElts ? 0 : 1;
9972 int EltIdx0 = Idx0 < SrcNumElts ? Idx0 : Idx0 - SrcNumElts;
9973 int EltIdx1 = Idx1 < SrcNumElts ? Idx1 : Idx1 - SrcNumElts;
9974
9975 SDValue Vec0 = SVN->getOperand(Num: VecIdx0);
9976 SDValue Elt0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec0,
9977 N2: DAG.getSignedConstant(Val: EltIdx0, DL: SL, VT: MVT::i32));
9978
9979 SDValue Vec1 = SVN->getOperand(Num: VecIdx1);
9980 SDValue Elt1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec1,
9981 N2: DAG.getSignedConstant(Val: EltIdx1, DL: SL, VT: MVT::i32));
9982 Pieces.push_back(Elt: DAG.getBuildVector(VT: PackVT, DL: SL, Ops: {Elt0, Elt1}));
9983 }
9984 }
9985
9986 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: ResultVT, Ops: Pieces);
9987}
9988
9989SDValue SITargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
9990 SelectionDAG &DAG) const {
9991 SDValue SVal = Op.getOperand(i: 0);
9992 EVT ResultVT = Op.getValueType();
9993 EVT SValVT = SVal.getValueType();
9994 SDValue UndefVal = DAG.getPOISON(VT: SValVT);
9995 SDLoc SL(Op);
9996
9997 SmallVector<SDValue, 8> VElts;
9998 VElts.push_back(Elt: SVal);
9999 for (int I = 1, E = ResultVT.getVectorNumElements(); I < E; ++I)
10000 VElts.push_back(Elt: UndefVal);
10001
10002 return DAG.getBuildVector(VT: ResultVT, DL: SL, Ops: VElts);
10003}
10004
10005SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op,
10006 SelectionDAG &DAG) const {
10007 SDLoc SL(Op);
10008 EVT VT = Op.getValueType();
10009
10010 if (VT == MVT::v2f16 || VT == MVT::v2i16 || VT == MVT::v2bf16) {
10011 assert(!Subtarget->hasVOP3PInsts() && "this should be legal");
10012
10013 SDValue Lo = Op.getOperand(i: 0);
10014 SDValue Hi = Op.getOperand(i: 1);
10015
10016 // Avoid adding defined bits with the zero_extend.
10017 if (Hi.isUndef()) {
10018 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Lo);
10019 SDValue ExtLo = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Lo);
10020 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: ExtLo);
10021 }
10022
10023 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Hi);
10024 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Hi);
10025
10026 SDValue ShlHi = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Hi,
10027 N2: DAG.getConstant(Val: 16, DL: SL, VT: MVT::i32));
10028 if (Lo.isUndef())
10029 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: ShlHi);
10030
10031 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Lo);
10032 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Lo);
10033
10034 SDValue Or =
10035 DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Lo, N2: ShlHi, Flags: SDNodeFlags::Disjoint);
10036 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Or);
10037 }
10038
10039 // Split into 2-element chunks.
10040 const unsigned NumParts = VT.getVectorNumElements() / 2;
10041 EVT PartVT = MVT::getVectorVT(VT: VT.getVectorElementType().getSimpleVT(), NumElements: 2);
10042 MVT PartIntVT = MVT::getIntegerVT(BitWidth: PartVT.getSizeInBits());
10043
10044 SmallVector<SDValue> Casts;
10045 for (unsigned P = 0; P < NumParts; ++P) {
10046 SDValue Vec = DAG.getBuildVector(
10047 VT: PartVT, DL: SL, Ops: {Op.getOperand(i: P * 2), Op.getOperand(i: P * 2 + 1)});
10048 Casts.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: PartIntVT, Operand: Vec));
10049 }
10050
10051 SDValue Blend =
10052 DAG.getBuildVector(VT: MVT::getVectorVT(VT: PartIntVT, NumElements: NumParts), DL: SL, Ops: Casts);
10053 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Blend);
10054}
10055
10056bool SITargetLowering::isOffsetFoldingLegal(
10057 const GlobalAddressSDNode *GA) const {
10058 // Named barriers have fixed, non-relocated LDS addresses, so a constant
10059 // offset into an array of them can be folded into the address.
10060 if (GA->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
10061 const auto *GV = dyn_cast<GlobalVariable>(Val: GA->getGlobal());
10062 return GV && AMDGPU::isNamedBarrier(GV: *GV);
10063 }
10064
10065 // OSes that use ELF REL relocations (instead of RELA) can only store a
10066 // 32-bit addend in the instruction, so it is not safe to allow offset folding
10067 // which can create arbitrary 64-bit addends. (This is only a problem for
10068 // R_AMDGPU_*32_HI relocations since other relocation types are unaffected by
10069 // the high 32 bits of the addend.)
10070 //
10071 // This should be kept in sync with how HasRelocationAddend is initialized in
10072 // the constructor of ELFAMDGPUAsmBackend.
10073 if (!Subtarget->isAmdHsaOS())
10074 return false;
10075
10076 // We can fold offsets for anything that doesn't require a GOT relocation.
10077 return (GA->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS ||
10078 GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
10079 GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
10080 !shouldEmitGOTReloc(GV: GA->getGlobal());
10081}
10082
10083static SDValue
10084buildPCRelGlobalAddress(SelectionDAG &DAG, const GlobalValue *GV,
10085 const SDLoc &DL, int64_t Offset, EVT PtrVT,
10086 unsigned GAFlags = SIInstrInfo::MO_NONE) {
10087 assert(isInt<32>(Offset + 4) && "32-bit offset is expected!");
10088 // In order to support pc-relative addressing, the PC_ADD_REL_OFFSET SDNode is
10089 // lowered to the following code sequence:
10090 //
10091 // For constant address space:
10092 // s_getpc_b64 s[0:1]
10093 // s_add_u32 s0, s0, $symbol
10094 // s_addc_u32 s1, s1, 0
10095 //
10096 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
10097 // a fixup or relocation is emitted to replace $symbol with a literal
10098 // constant, which is a pc-relative offset from the encoding of the $symbol
10099 // operand to the global variable.
10100 //
10101 // For global address space:
10102 // s_getpc_b64 s[0:1]
10103 // s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
10104 // s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
10105 //
10106 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
10107 // fixups or relocations are emitted to replace $symbol@*@lo and
10108 // $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
10109 // which is a 64-bit pc-relative offset from the encoding of the $symbol
10110 // operand to the global variable.
10111 if (((const GCNSubtarget &)DAG.getSubtarget()).has64BitLiterals()) {
10112 assert(GAFlags != SIInstrInfo::MO_NONE);
10113
10114 SDValue Ptr =
10115 DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i64, offset: Offset, TargetFlags: GAFlags + 2);
10116 return DAG.getNode(Opcode: AMDGPUISD::PC_ADD_REL_OFFSET64, DL, VT: PtrVT, Operand: Ptr);
10117 }
10118
10119 SDValue PtrLo = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: Offset, TargetFlags: GAFlags);
10120 SDValue PtrHi;
10121 if (GAFlags == SIInstrInfo::MO_NONE)
10122 PtrHi = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32);
10123 else
10124 PtrHi = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: Offset, TargetFlags: GAFlags + 1);
10125 return DAG.getNode(Opcode: AMDGPUISD::PC_ADD_REL_OFFSET, DL, VT: PtrVT, N1: PtrLo, N2: PtrHi);
10126}
10127
10128SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
10129 SDValue Op,
10130 SelectionDAG &DAG) const {
10131 GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Val&: Op);
10132 SDLoc DL(GSD);
10133 EVT PtrVT = Op.getValueType();
10134
10135 const GlobalValue *GV = GSD->getGlobal();
10136 const unsigned AS = GSD->getAddressSpace();
10137 if (((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::BARRIER) &&
10138 shouldUseLDSConstAddress(GV)) ||
10139 AS == AMDGPUAS::REGION_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS) {
10140 if (AS == AMDGPUAS::LOCAL_ADDRESS && GV->hasExternalLinkage()) {
10141 const GlobalVariable &GVar = *cast<GlobalVariable>(Val: GV);
10142 // HIP uses an unsized array `extern __shared__ T s[]` or similar
10143 // zero-sized type in other languages to declare the dynamic shared
10144 // memory which size is not known at the compile time. They will be
10145 // allocated by the runtime and placed directly after the static
10146 // allocated ones. They all share the same offset.
10147 if (GVar.getGlobalSize(DL: GVar.getDataLayout()) == 0) {
10148 assert(PtrVT == MVT::i32 && "32-bit pointer is expected.");
10149 // Adjust alignment for that dynamic shared memory array.
10150 Function &F = DAG.getMachineFunction().getFunction();
10151 MFI->setDynLDSAlign(F, GV: GVar);
10152 MFI->setUsesDynamicLDS(true);
10153 return SDValue(
10154 DAG.getMachineNode(Opcode: AMDGPU::GET_GROUPSTATICSIZE, dl: DL, VT: PtrVT), 0);
10155 }
10156 }
10157 return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG);
10158 }
10159
10160 if (AS == AMDGPUAS::BARRIER) {
10161 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: GSD->getOffset(),
10162 TargetFlags: SIInstrInfo::MO_ABS32_LO);
10163 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: GA), 0);
10164 }
10165
10166 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
10167 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: GSD->getOffset(),
10168 TargetFlags: SIInstrInfo::MO_ABS32_LO);
10169 return DAG.getNode(Opcode: AMDGPUISD::LDS, DL, VT: MVT::i32, Operand: GA);
10170 }
10171
10172 if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS()) {
10173 if (Subtarget->has64BitLiterals()) {
10174 SDValue Addr = DAG.getTargetGlobalAddress(
10175 GV, DL, VT: MVT::i64, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS64);
10176 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B64, dl: DL, VT: MVT::i64, Op1: Addr),
10177 0);
10178 }
10179
10180 SDValue AddrLo = DAG.getTargetGlobalAddress(
10181 GV, DL, VT: MVT::i32, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS32_LO);
10182 AddrLo = {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: AddrLo), 0};
10183
10184 SDValue AddrHi = DAG.getTargetGlobalAddress(
10185 GV, DL, VT: MVT::i32, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS32_HI);
10186 AddrHi = {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: AddrHi), 0};
10187
10188 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: AddrLo, N2: AddrHi);
10189 }
10190
10191 if (shouldEmitFixup(GV))
10192 return buildPCRelGlobalAddress(DAG, GV, DL, Offset: GSD->getOffset(), PtrVT);
10193
10194 if (shouldEmitPCReloc(GV))
10195 return buildPCRelGlobalAddress(DAG, GV, DL, Offset: GSD->getOffset(), PtrVT,
10196 GAFlags: SIInstrInfo::MO_REL32);
10197
10198 SDValue GOTAddr = buildPCRelGlobalAddress(DAG, GV, DL, Offset: 0, PtrVT,
10199 GAFlags: SIInstrInfo::MO_GOTPCREL32);
10200 PointerType *PtrTy =
10201 PointerType::get(C&: *DAG.getContext(), AddressSpace: AMDGPUAS::CONSTANT_ADDRESS);
10202 const DataLayout &DataLayout = DAG.getDataLayout();
10203 Align Alignment = DataLayout.getABITypeAlign(Ty: PtrTy);
10204 MachinePointerInfo PtrInfo =
10205 MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction());
10206
10207 return DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: GOTAddr, PtrInfo, Alignment,
10208 MMOFlags: MachineMemOperand::MODereferenceable |
10209 MachineMemOperand::MOInvariant);
10210}
10211
10212SDValue SITargetLowering::LowerExternalSymbol(SDValue Op,
10213 SelectionDAG &DAG) const {
10214 // TODO: Handle this. It should be mostly the same as LowerGlobalAddress.
10215 const Function &Fn = DAG.getMachineFunction().getFunction();
10216 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10217 Fn, "unsupported external symbol", Op.getDebugLoc()));
10218 return DAG.getPOISON(VT: Op.getValueType());
10219}
10220
10221SDValue SITargetLowering::copyToM0(SelectionDAG &DAG, SDValue Chain,
10222 const SDLoc &DL, SDValue V) const {
10223 // We can't use S_MOV_B32 directly, because there is no way to specify m0 as
10224 // the destination register.
10225 //
10226 // We can't use CopyToReg, because MachineCSE won't combine COPY instructions,
10227 // so we will end up with redundant moves to m0.
10228 //
10229 // We use a pseudo to ensure we emit s_mov_b32 with m0 as the direct result.
10230
10231 // A Null SDValue creates a glue result.
10232 SDNode *M0 = DAG.getMachineNode(Opcode: AMDGPU::SI_INIT_M0, dl: DL, VT1: MVT::Other, VT2: MVT::Glue,
10233 Op1: V, Op2: Chain);
10234 return SDValue(M0, 0);
10235}
10236
10237SDValue SITargetLowering::lowerImplicitZextParam(SelectionDAG &DAG, SDValue Op,
10238 MVT VT,
10239 unsigned Offset) const {
10240 SDLoc SL(Op);
10241 SDValue Param = lowerKernargMemParameter(
10242 DAG, VT: MVT::i32, MemVT: MVT::i32, SL, Chain: DAG.getEntryNode(), Offset, Alignment: Align(4), Signed: false);
10243 // The local size values will have the hi 16-bits as zero.
10244 return DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: MVT::i32, N1: Param,
10245 N2: DAG.getValueType(VT));
10246}
10247
10248static SDValue emitNonHSAIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
10249 EVT VT) {
10250 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10251 DAG.getMachineFunction().getFunction(),
10252 "non-hsa intrinsic with hsa target", DL.getDebugLoc()));
10253 return DAG.getPOISON(VT);
10254}
10255
10256static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
10257 EVT VT) {
10258 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10259 DAG.getMachineFunction().getFunction(),
10260 "intrinsic not supported on subtarget", DL.getDebugLoc()));
10261 return DAG.getPOISON(VT);
10262}
10263
10264static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL,
10265 ArrayRef<SDValue> Elts) {
10266 assert(!Elts.empty());
10267 MVT Type;
10268 unsigned NumElts = Elts.size();
10269
10270 if (NumElts <= 12) {
10271 Type = MVT::getVectorVT(VT: MVT::f32, NumElements: NumElts);
10272 } else {
10273 assert(Elts.size() <= 16);
10274 Type = MVT::v16f32;
10275 NumElts = 16;
10276 }
10277
10278 SmallVector<SDValue, 16> VecElts(NumElts);
10279 for (unsigned i = 0; i < Elts.size(); ++i) {
10280 SDValue Elt = Elts[i];
10281 if (Elt.getValueType() != MVT::f32)
10282 Elt = DAG.getBitcast(VT: MVT::f32, V: Elt);
10283 VecElts[i] = Elt;
10284 }
10285 for (unsigned i = Elts.size(); i < NumElts; ++i)
10286 VecElts[i] = DAG.getPOISON(VT: MVT::f32);
10287
10288 if (NumElts == 1)
10289 return VecElts[0];
10290 return DAG.getBuildVector(VT: Type, DL, Ops: VecElts);
10291}
10292
10293static SDValue padEltsToUndef(SelectionDAG &DAG, const SDLoc &DL, EVT CastVT,
10294 SDValue Src, int ExtraElts) {
10295 EVT SrcVT = Src.getValueType();
10296
10297 SmallVector<SDValue, 8> Elts;
10298
10299 if (SrcVT.isVector())
10300 DAG.ExtractVectorElements(Op: Src, Args&: Elts);
10301 else
10302 Elts.push_back(Elt: Src);
10303
10304 SDValue Undef = DAG.getPOISON(VT: SrcVT.getScalarType());
10305 while (ExtraElts--)
10306 Elts.push_back(Elt: Undef);
10307
10308 return DAG.getBuildVector(VT: CastVT, DL, Ops: Elts);
10309}
10310
10311// Re-construct the required return value for a image load intrinsic.
10312// This is more complicated due to the optional use TexFailCtrl which means the
10313// required return type is an aggregate
10314static SDValue constructRetValue(SelectionDAG &DAG, MachineSDNode *Result,
10315 ArrayRef<EVT> ResultTypes, bool IsTexFail,
10316 bool Unpacked, bool IsD16, int DMaskPop,
10317 int NumVDataDwords, bool IsAtomicPacked16Bit,
10318 const SDLoc &DL) {
10319 // Determine the required return type. This is the same regardless of
10320 // IsTexFail flag
10321 EVT ReqRetVT = ResultTypes[0];
10322 int ReqRetNumElts = ReqRetVT.isVector() ? ReqRetVT.getVectorNumElements() : 1;
10323 int NumDataDwords = ((IsD16 && !Unpacked) || IsAtomicPacked16Bit)
10324 ? (ReqRetNumElts + 1) / 2
10325 : ReqRetNumElts;
10326
10327 int MaskPopDwords = (!IsD16 || Unpacked) ? DMaskPop : (DMaskPop + 1) / 2;
10328
10329 MVT DataDwordVT =
10330 NumDataDwords == 1 ? MVT::i32 : MVT::getVectorVT(VT: MVT::i32, NumElements: NumDataDwords);
10331
10332 MVT MaskPopVT =
10333 MaskPopDwords == 1 ? MVT::i32 : MVT::getVectorVT(VT: MVT::i32, NumElements: MaskPopDwords);
10334
10335 SDValue Data(Result, 0);
10336 SDValue TexFail;
10337
10338 if (DMaskPop > 0 && Data.getValueType() != MaskPopVT) {
10339 SDValue ZeroIdx = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
10340 if (MaskPopVT.isVector()) {
10341 Data = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: MaskPopVT,
10342 N1: SDValue(Result, 0), N2: ZeroIdx);
10343 } else {
10344 Data = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MaskPopVT,
10345 N1: SDValue(Result, 0), N2: ZeroIdx);
10346 }
10347 }
10348
10349 if (DataDwordVT.isVector() && !IsAtomicPacked16Bit)
10350 Data = padEltsToUndef(DAG, DL, CastVT: DataDwordVT, Src: Data,
10351 ExtraElts: NumDataDwords - MaskPopDwords);
10352
10353 if (IsD16)
10354 Data = adjustLoadValueTypeImpl(Result: Data, LoadVT: ReqRetVT, DL, DAG, Unpacked);
10355
10356 EVT LegalReqRetVT = ReqRetVT;
10357 if (!ReqRetVT.isVector()) {
10358 if (!Data.getValueType().isInteger())
10359 Data = DAG.getNode(Opcode: ISD::BITCAST, DL,
10360 VT: Data.getValueType().changeTypeToInteger(), Operand: Data);
10361 Data = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ReqRetVT.changeTypeToInteger(), Operand: Data);
10362 } else {
10363 // We need to widen the return vector to a legal type
10364 if ((ReqRetVT.getVectorNumElements() % 2) == 1 &&
10365 ReqRetVT.getVectorElementType().getSizeInBits() == 16) {
10366 LegalReqRetVT =
10367 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ReqRetVT.getVectorElementType(),
10368 NumElements: ReqRetVT.getVectorNumElements() + 1);
10369 }
10370 }
10371 Data = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LegalReqRetVT, Operand: Data);
10372
10373 if (IsTexFail) {
10374 TexFail =
10375 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: SDValue(Result, 0),
10376 N2: DAG.getConstant(Val: MaskPopDwords, DL, VT: MVT::i32));
10377
10378 return DAG.getMergeValues(Ops: {Data, TexFail, SDValue(Result, 1)}, dl: DL);
10379 }
10380
10381 if (Result->getNumValues() == 1)
10382 return Data;
10383
10384 return DAG.getMergeValues(Ops: {Data, SDValue(Result, 1)}, dl: DL);
10385}
10386
10387static bool parseTexFail(SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE,
10388 SDValue *LWE, bool &IsTexFail) {
10389 auto *TexFailCtrlConst = cast<ConstantSDNode>(Val: TexFailCtrl.getNode());
10390
10391 uint64_t Value = TexFailCtrlConst->getZExtValue();
10392 if (Value) {
10393 IsTexFail = true;
10394 }
10395
10396 SDLoc DL(TexFailCtrlConst);
10397 *TFE = DAG.getTargetConstant(Val: (Value & 0x1) ? 1 : 0, DL, VT: MVT::i32);
10398 Value &= ~(uint64_t)0x1;
10399 *LWE = DAG.getTargetConstant(Val: (Value & 0x2) ? 1 : 0, DL, VT: MVT::i32);
10400 Value &= ~(uint64_t)0x2;
10401
10402 return Value == 0;
10403}
10404
10405static void packImage16bitOpsToDwords(SelectionDAG &DAG, SDValue Op,
10406 MVT PackVectorVT,
10407 SmallVectorImpl<SDValue> &PackedAddrs,
10408 unsigned DimIdx, unsigned EndIdx,
10409 unsigned NumGradients) {
10410 SDLoc DL(Op);
10411 for (unsigned I = DimIdx; I < EndIdx; I++) {
10412 SDValue Addr = Op.getOperand(i: I);
10413
10414 // Gradients are packed with undef for each coordinate.
10415 // In <hi 16 bit>,<lo 16 bit> notation, the registers look like this:
10416 // 1D: undef,dx/dh; undef,dx/dv
10417 // 2D: dy/dh,dx/dh; dy/dv,dx/dv
10418 // 3D: dy/dh,dx/dh; undef,dz/dh; dy/dv,dx/dv; undef,dz/dv
10419 if (((I + 1) >= EndIdx) ||
10420 ((NumGradients / 2) % 2 == 1 && (I == DimIdx + (NumGradients / 2) - 1 ||
10421 I == DimIdx + NumGradients - 1))) {
10422 if (Addr.getValueType() != MVT::i16)
10423 Addr = DAG.getBitcast(VT: MVT::i16, V: Addr);
10424 Addr = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Addr);
10425 } else {
10426 Addr = DAG.getBuildVector(VT: PackVectorVT, DL, Ops: {Addr, Op.getOperand(i: I + 1)});
10427 I++;
10428 }
10429 Addr = DAG.getBitcast(VT: MVT::f32, V: Addr);
10430 PackedAddrs.push_back(Elt: Addr);
10431 }
10432}
10433
10434/// Emit a DiagnosticInfoUnsupported for an unsupported image intrinsic and
10435/// return poison values of \p ResultTypes, preserving the chain if present.
10436static SDValue diagnoseUnsupportedImage(SelectionDAG &DAG, SDValue Op,
10437 ArrayRef<EVT> ResultTypes,
10438 const SDLoc &DL, const Twine &Msg) {
10439 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10440 DAG.getMachineFunction().getFunction(), Msg, DL.getDebugLoc()));
10441 return DAG.getErrorMergeValues(ResultTypes, Chain: Op.getOperand(i: 0), dl: DL);
10442}
10443
10444SDValue SITargetLowering::lowerImage(SDValue Op,
10445 const AMDGPU::ImageDimIntrinsicInfo *Intr,
10446 SelectionDAG &DAG, bool WithChain) const {
10447 SDLoc DL(Op);
10448 MachineFunction &MF = DAG.getMachineFunction();
10449 const GCNSubtarget *ST = &MF.getSubtarget<GCNSubtarget>();
10450 unsigned IntrOpcode = Intr->BaseOpcode;
10451 // For image atomic: use no-return opcode if result is unused.
10452 if (Intr->AtomicNoRetBaseOpcode != Intr->BaseOpcode &&
10453 !Op.getNode()->hasAnyUseOfValue(Value: 0))
10454 IntrOpcode = Intr->AtomicNoRetBaseOpcode;
10455 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
10456 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: IntrOpcode);
10457 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(DimEnum: Intr->Dim);
10458 bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI: *Subtarget);
10459 bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI: *Subtarget);
10460 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
10461 bool IsGFX13 = AMDGPU::isGFX13(STI: *Subtarget);
10462
10463 SmallVector<EVT, 3> ResultTypes(Op->values());
10464 SmallVector<EVT, 3> OrigResultTypes(Op->values());
10465 if (BaseOpcode->NoReturn && BaseOpcode->Atomic)
10466 ResultTypes.erase(CI: &ResultTypes[0]);
10467
10468 bool IsD16 = false;
10469 bool IsG16 = false;
10470 bool IsA16 = false;
10471 SDValue VData;
10472 int NumVDataDwords = 0;
10473 bool AdjustRetType = false;
10474 bool IsAtomicPacked16Bit = false;
10475
10476 // Offset of intrinsic arguments
10477 const unsigned ArgOffset = WithChain ? 2 : 1;
10478
10479 unsigned DMask;
10480 unsigned DMaskLanes = 0;
10481
10482 if (BaseOpcode->Atomic) {
10483 VData = Op.getOperand(i: 2);
10484
10485 IsAtomicPacked16Bit =
10486 (IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
10487 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16_NORTN ||
10488 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16 ||
10489 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16_NORTN);
10490
10491 if (!IsAtomicPacked16Bit && VData.getValueSizeInBits() != 32 &&
10492 VData.getValueSizeInBits() != 64) {
10493 return diagnoseUnsupportedImage(DAG, Op, ResultTypes: OrigResultTypes, DL,
10494 Msg: "unsupported image atomic data type");
10495 }
10496
10497 bool Is64Bit = VData.getValueSizeInBits() == 64;
10498 if (BaseOpcode->AtomicX2) {
10499 SDValue VData2 = Op.getOperand(i: 3);
10500 VData = DAG.getBuildVector(VT: Is64Bit ? MVT::v2i64 : MVT::v2i32, DL,
10501 Ops: {VData, VData2});
10502 if (Is64Bit)
10503 VData = DAG.getBitcast(VT: MVT::v4i32, V: VData);
10504
10505 if (!BaseOpcode->NoReturn)
10506 ResultTypes[0] = Is64Bit ? MVT::v2i64 : MVT::v2i32;
10507
10508 DMask = Is64Bit ? 0xf : 0x3;
10509 NumVDataDwords = Is64Bit ? 4 : 2;
10510 } else {
10511 DMask = Is64Bit ? 0x3 : 0x1;
10512 NumVDataDwords = Is64Bit ? 2 : 1;
10513 }
10514 } else {
10515 DMask = Op->getConstantOperandVal(Num: ArgOffset + Intr->DMaskIndex);
10516 DMaskLanes = BaseOpcode->Gather4 ? 4 : llvm::popcount(Value: DMask);
10517
10518 if (BaseOpcode->Store) {
10519 VData = Op.getOperand(i: 2);
10520
10521 MVT StoreVT = VData.getSimpleValueType();
10522 MVT StoreScalarVT = StoreVT.getScalarType();
10523 if (StoreScalarVT != MVT::f16 && StoreScalarVT.getSizeInBits() != 32 &&
10524 StoreScalarVT.getSizeInBits() != 64) {
10525 return diagnoseUnsupportedImage(DAG, Op, ResultTypes: OrigResultTypes, DL,
10526 Msg: "unsupported image store data type");
10527 }
10528 if (StoreScalarVT == MVT::f16) {
10529 if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
10530 return Op; // D16 is unsupported for this instruction
10531
10532 IsD16 = true;
10533 VData = handleD16VData(VData, DAG, ImageStore: true);
10534 }
10535
10536 NumVDataDwords = (VData.getValueType().getSizeInBits() + 31) / 32;
10537 } else if (!BaseOpcode->NoReturn) {
10538 // Work out the num dwords based on the dmask popcount and underlying type
10539 // and whether packing is supported.
10540 MVT LoadVT = ResultTypes[0].getSimpleVT();
10541 MVT LoadScalarVT = LoadVT.getScalarType();
10542 if (LoadScalarVT != MVT::f16 && LoadScalarVT.getSizeInBits() != 32 &&
10543 LoadScalarVT.getSizeInBits() != 64) {
10544 return diagnoseUnsupportedImage(DAG, Op, ResultTypes: OrigResultTypes, DL,
10545 Msg: "unsupported image load data type");
10546 }
10547 if (LoadScalarVT == MVT::f16) {
10548 if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
10549 return Op; // D16 is unsupported for this instruction
10550
10551 IsD16 = true;
10552 }
10553
10554 // Confirm that the return type is large enough for the dmask specified
10555 if ((LoadVT.isVector() && LoadVT.getVectorNumElements() < DMaskLanes) ||
10556 (!LoadVT.isVector() && DMaskLanes > 1))
10557 return Op;
10558
10559 // The sq block of gfx8 and gfx9 do not estimate register use correctly
10560 // for d16 image_gather4, image_gather4_l, and image_gather4_lz
10561 // instructions.
10562 if (IsD16 && !Subtarget->hasUnpackedD16VMem() &&
10563 !(BaseOpcode->Gather4 && Subtarget->hasImageGather4D16Bug()))
10564 NumVDataDwords = (DMaskLanes + 1) / 2;
10565 else
10566 NumVDataDwords = DMaskLanes;
10567
10568 AdjustRetType = true;
10569 }
10570 }
10571
10572 unsigned VAddrEnd = ArgOffset + Intr->VAddrEnd;
10573 SmallVector<SDValue, 4> VAddrs;
10574
10575 // Check for 16 bit addresses or derivatives and pack if true.
10576 MVT VAddrVT =
10577 Op.getOperand(i: ArgOffset + Intr->GradientStart).getSimpleValueType();
10578 MVT VAddrScalarVT = VAddrVT.getScalarType();
10579 MVT GradPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
10580 IsG16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
10581
10582 VAddrVT = Op.getOperand(i: ArgOffset + Intr->CoordStart).getSimpleValueType();
10583 VAddrScalarVT = VAddrVT.getScalarType();
10584 MVT AddrPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
10585 IsA16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
10586
10587 // Push back extra arguments.
10588 for (unsigned I = Intr->VAddrStart; I < Intr->GradientStart; I++) {
10589 if (IsA16 && (Op.getOperand(i: ArgOffset + I).getValueType() == MVT::f16)) {
10590 assert(I == Intr->BiasIndex && "Got unexpected 16-bit extra argument");
10591 // Special handling of bias when A16 is on. Bias is of type half but
10592 // occupies full 32-bit.
10593 SDValue Bias = DAG.getBuildVector(
10594 VT: MVT::v2f16, DL,
10595 Ops: {Op.getOperand(i: ArgOffset + I), DAG.getPOISON(VT: MVT::f16)});
10596 VAddrs.push_back(Elt: Bias);
10597 } else {
10598 assert((!IsA16 || Intr->NumBiasArgs == 0 || I != Intr->BiasIndex) &&
10599 "Bias needs to be converted to 16 bit in A16 mode");
10600 VAddrs.push_back(Elt: Op.getOperand(i: ArgOffset + I));
10601 }
10602 }
10603
10604 if (BaseOpcode->Gradients && !ST->hasG16() && (IsA16 != IsG16)) {
10605 // 16 bit gradients are supported, but are tied to the A16 control
10606 // so both gradients and addresses must be 16 bit
10607 LLVM_DEBUG(
10608 dbgs() << "Failed to lower image intrinsic: 16 bit addresses "
10609 "require 16 bit args for both gradients and addresses");
10610 return Op;
10611 }
10612
10613 if (IsA16) {
10614 if (!ST->hasA16()) {
10615 LLVM_DEBUG(dbgs() << "Failed to lower image intrinsic: Target does not "
10616 "support 16 bit addresses\n");
10617 return Op;
10618 }
10619 }
10620
10621 // We've dealt with incorrect input so we know that if IsA16, IsG16
10622 // are set then we have to compress/pack operands (either address,
10623 // gradient or both)
10624 // In the case where a16 and gradients are tied (no G16 support) then we
10625 // have already verified that both IsA16 and IsG16 are true
10626 if (BaseOpcode->Gradients && IsG16 && ST->hasG16()) {
10627 // Activate g16
10628 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
10629 AMDGPU::getMIMGG16MappingInfo(G: Intr->BaseOpcode);
10630 IntrOpcode = G16MappingInfo->G16; // set new opcode to variant with _g16
10631 }
10632
10633 // Add gradients (packed or unpacked)
10634 if (IsG16) {
10635 // Pack the gradients
10636 // const int PackEndIdx = IsA16 ? VAddrEnd : (ArgOffset + Intr->CoordStart);
10637 packImage16bitOpsToDwords(DAG, Op, PackVectorVT: GradPackVectorVT, PackedAddrs&: VAddrs,
10638 DimIdx: ArgOffset + Intr->GradientStart,
10639 EndIdx: ArgOffset + Intr->CoordStart, NumGradients: Intr->NumGradients);
10640 } else {
10641 for (unsigned I = ArgOffset + Intr->GradientStart;
10642 I < ArgOffset + Intr->CoordStart; I++)
10643 VAddrs.push_back(Elt: Op.getOperand(i: I));
10644 }
10645
10646 // Add addresses (packed or unpacked)
10647 if (IsA16) {
10648 packImage16bitOpsToDwords(DAG, Op, PackVectorVT: AddrPackVectorVT, PackedAddrs&: VAddrs,
10649 DimIdx: ArgOffset + Intr->CoordStart, EndIdx: VAddrEnd,
10650 NumGradients: 0 /* No gradients */);
10651 } else {
10652 // Add uncompressed address
10653 for (unsigned I = ArgOffset + Intr->CoordStart; I < VAddrEnd; I++)
10654 VAddrs.push_back(Elt: Op.getOperand(i: I));
10655 }
10656
10657 // If the register allocator cannot place the address registers contiguously
10658 // without introducing moves, then using the non-sequential address encoding
10659 // is always preferable, since it saves VALU instructions and is usually a
10660 // wash in terms of code size or even better.
10661 //
10662 // However, we currently have no way of hinting to the register allocator that
10663 // MIMG addresses should be placed contiguously when it is possible to do so,
10664 // so force non-NSA for the common 2-address case as a heuristic.
10665 //
10666 // SIShrinkInstructions will convert NSA encodings to non-NSA after register
10667 // allocation when possible.
10668 //
10669 // Partial NSA is allowed on GFX11+ where the final register is a contiguous
10670 // set of the remaining addresses.
10671 const unsigned NSAMaxSize = ST->getNSAMaxSize(HasSampler: BaseOpcode->Sampler);
10672 const bool HasPartialNSAEncoding = ST->hasPartialNSAEncoding();
10673 const bool UseNSA = ST->hasNSAEncoding() &&
10674 VAddrs.size() >= ST->getNSAThreshold(MF) &&
10675 (VAddrs.size() <= NSAMaxSize || HasPartialNSAEncoding);
10676 const bool UsePartialNSA =
10677 UseNSA && HasPartialNSAEncoding && VAddrs.size() > NSAMaxSize;
10678
10679 SDValue VAddr;
10680 if (UsePartialNSA) {
10681 VAddr = getBuildDwordsVector(DAG, DL,
10682 Elts: ArrayRef(VAddrs).drop_front(N: NSAMaxSize - 1));
10683 } else if (!UseNSA) {
10684 VAddr = getBuildDwordsVector(DAG, DL, Elts: VAddrs);
10685 }
10686
10687 SDValue True = DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1);
10688 SDValue False = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1);
10689 SDValue Unorm;
10690 if (!BaseOpcode->Sampler) {
10691 Unorm = True;
10692 } else {
10693 uint64_t UnormConst =
10694 Op.getConstantOperandVal(i: ArgOffset + Intr->UnormIndex);
10695
10696 Unorm = UnormConst ? True : False;
10697 }
10698
10699 SDValue TFE;
10700 SDValue LWE;
10701 SDValue TexFail = Op.getOperand(i: ArgOffset + Intr->TexFailCtrlIndex);
10702 bool IsTexFail = false;
10703 if (!parseTexFail(TexFailCtrl: TexFail, DAG, TFE: &TFE, LWE: &LWE, IsTexFail))
10704 return Op;
10705
10706 if (IsTexFail) {
10707 if (!DMaskLanes) {
10708 // Expecting to get an error flag since TFC is on - and dmask is 0
10709 // Force dmask to be at least 1 otherwise the instruction will fail
10710 DMask = 0x1;
10711 DMaskLanes = 1;
10712 NumVDataDwords = 1;
10713 }
10714 NumVDataDwords += 1;
10715 AdjustRetType = true;
10716 }
10717
10718 // Has something earlier tagged that the return type needs adjusting
10719 // This happens if the instruction is a load or has set TexFailCtrl flags
10720 if (AdjustRetType) {
10721 // NumVDataDwords reflects the true number of dwords required in the return
10722 // type
10723 if (DMaskLanes == 0 && !BaseOpcode->Store) {
10724 // This is a no-op load. This can be eliminated
10725 SDValue Undef = DAG.getPOISON(VT: Op.getValueType());
10726 if (isa<MemSDNode>(Val: Op))
10727 return DAG.getMergeValues(Ops: {Undef, Op.getOperand(i: 0)}, dl: DL);
10728 return Undef;
10729 }
10730
10731 EVT NewVT = NumVDataDwords > 1 ? EVT::getVectorVT(Context&: *DAG.getContext(),
10732 VT: MVT::i32, NumElements: NumVDataDwords)
10733 : MVT::i32;
10734
10735 ResultTypes[0] = NewVT;
10736 if (ResultTypes.size() == 3) {
10737 // Original result was aggregate type used for TexFailCtrl results
10738 // The actual instruction returns as a vector type which has now been
10739 // created. Remove the aggregate result.
10740 ResultTypes.erase(CI: &ResultTypes[1]);
10741 }
10742 }
10743
10744 unsigned CPol = Op.getConstantOperandVal(i: ArgOffset + Intr->CachePolicyIndex);
10745 // Keep GLC only when the atomic's result is actually used.
10746 if (BaseOpcode->Atomic && !BaseOpcode->NoReturn)
10747 CPol |= AMDGPU::CPol::GLC;
10748 if (CPol & ~((IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12) |
10749 AMDGPU::CPol::VOLATILE))
10750 return Op;
10751
10752 SmallVector<SDValue, 26> Ops;
10753 if (BaseOpcode->Store || BaseOpcode->Atomic)
10754 Ops.push_back(Elt: VData); // vdata
10755 if (UsePartialNSA) {
10756 append_range(C&: Ops, R: ArrayRef(VAddrs).take_front(N: NSAMaxSize - 1));
10757 Ops.push_back(Elt: VAddr);
10758 } else if (UseNSA)
10759 append_range(C&: Ops, R&: VAddrs);
10760 else
10761 Ops.push_back(Elt: VAddr);
10762 SDValue Rsrc = Op.getOperand(i: ArgOffset + Intr->RsrcIndex);
10763 EVT RsrcVT = Rsrc.getValueType();
10764 if (RsrcVT != MVT::v4i32 && RsrcVT != MVT::v8i32)
10765 return Op;
10766 Ops.push_back(Elt: Rsrc);
10767 if (BaseOpcode->Sampler) {
10768 SDValue Samp = Op.getOperand(i: ArgOffset + Intr->SampIndex);
10769 if (Samp.getValueType() != MVT::v4i32)
10770 return Op;
10771 Ops.push_back(Elt: Samp);
10772 }
10773 Ops.push_back(Elt: DAG.getTargetConstant(Val: DMask, DL, VT: MVT::i32));
10774 if (IsGFX10Plus)
10775 Ops.push_back(Elt: DAG.getTargetConstant(Val: DimInfo->Encoding, DL, VT: MVT::i32));
10776 if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
10777 Ops.push_back(Elt: Unorm);
10778 Ops.push_back(Elt: DAG.getTargetConstant(Val: CPol, DL, VT: MVT::i32));
10779 Ops.push_back(Elt: IsA16 && // r128, a16 for gfx9
10780 ST->hasFeature(Feature: AMDGPU::FeatureR128A16)
10781 ? True
10782 : False);
10783 if (IsGFX10Plus)
10784 Ops.push_back(Elt: IsA16 ? True : False);
10785
10786 if (!Subtarget->hasGFX90AInsts())
10787 Ops.push_back(Elt: TFE); // tfe
10788 else if (TFE->getAsZExtVal()) {
10789 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10790 DAG.getMachineFunction().getFunction(),
10791 "TFE is not supported on this GPU", DL.getDebugLoc()));
10792 }
10793
10794 if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
10795 Ops.push_back(Elt: LWE); // lwe
10796 if (!IsGFX10Plus)
10797 Ops.push_back(Elt: DimInfo->DA ? True : False);
10798 if (BaseOpcode->HasD16)
10799 Ops.push_back(Elt: IsD16 ? True : False);
10800 if (isa<MemSDNode>(Val: Op))
10801 Ops.push_back(Elt: Op.getOperand(i: 0)); // chain
10802
10803 int NumVAddrDwords =
10804 UseNSA ? VAddrs.size() : VAddr.getValueType().getSizeInBits() / 32;
10805 int Opcode = -1;
10806
10807 if (IsGFX13) {
10808 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx13,
10809 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10810 } else if (IsGFX12Plus) {
10811 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx12,
10812 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10813 } else if (IsGFX11Plus) {
10814 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
10815 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx11NSA
10816 : AMDGPU::MIMGEncGfx11Default,
10817 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10818 } else if (IsGFX10Plus) {
10819 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
10820 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx10NSA
10821 : AMDGPU::MIMGEncGfx10Default,
10822 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10823 } else {
10824 if (Subtarget->hasGFX90AInsts()) {
10825 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx90a,
10826 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10827 if (Opcode == -1) {
10828 return diagnoseUnsupportedImage(
10829 DAG, Op, ResultTypes: OrigResultTypes, DL,
10830 Msg: "requested image instruction is not supported on this GPU");
10831 }
10832 }
10833 if (Opcode == -1 &&
10834 Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
10835 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx8,
10836 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10837 if (Opcode == -1)
10838 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx6,
10839 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10840 }
10841 if (Opcode == -1)
10842 return Op;
10843
10844 MachineSDNode *NewNode = DAG.getMachineNode(Opcode, dl: DL, ResultTys: ResultTypes, Ops);
10845 if (auto *MemOp = dyn_cast<MemSDNode>(Val&: Op)) {
10846 MachineMemOperand *MemRef = MemOp->getMemOperand();
10847 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
10848 }
10849
10850 if (BaseOpcode->NoReturn) {
10851 if (BaseOpcode->Atomic)
10852 return DAG.getMergeValues(
10853 Ops: {DAG.getPOISON(VT: OrigResultTypes[0]), SDValue(NewNode, 0)}, dl: DL);
10854
10855 return SDValue(NewNode, 0);
10856 }
10857
10858 if (BaseOpcode->AtomicX2) {
10859 SmallVector<SDValue, 1> Elt;
10860 DAG.ExtractVectorElements(Op: SDValue(NewNode, 0), Args&: Elt, Start: 0, Count: 1);
10861 return DAG.getMergeValues(Ops: {Elt[0], SDValue(NewNode, 1)}, dl: DL);
10862 }
10863
10864 return constructRetValue(DAG, Result: NewNode, ResultTypes: OrigResultTypes, IsTexFail,
10865 Unpacked: Subtarget->hasUnpackedD16VMem(), IsD16, DMaskPop: DMaskLanes,
10866 NumVDataDwords, IsAtomicPacked16Bit, DL);
10867}
10868
10869SDValue SITargetLowering::lowerSBuffer(EVT VT, EVT MemVT, SDLoc DL,
10870 SDValue Chain, SDValue Rsrc,
10871 SDValue Offset, SDValue CachePolicy,
10872 SelectionDAG &DAG,
10873 MachineMemOperand *MMO) const {
10874 MachineFunction &MF = DAG.getMachineFunction();
10875 bool HasChainResult = MMO != nullptr;
10876
10877 // SBUFFER_LOAD only produces values that fill whole SGPRs, apart from the
10878 // subword loads below.
10879 bool IsSubwordLoad = (MemVT == MVT::i8 || MemVT == MVT::i16) &&
10880 Subtarget->hasScalarSubwordLoads();
10881 if ((!isTypeLegal(VT) || VT.getSizeInBits() % 32 != 0) && !IsSubwordLoad) {
10882 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10883 MF.getFunction(), "unsupported s_buffer_load result type",
10884 DL.getDebugLoc()));
10885 EVT ResultTypes[] = {VT, MVT::Other};
10886 return DAG.getErrorMergeValues(
10887 ResultTypes: ArrayRef(ResultTypes, HasChainResult ? 2 : 1), Chain, dl: DL);
10888 }
10889
10890 if (!HasChainResult) {
10891 const DataLayout &DataLayout = DAG.getDataLayout();
10892 Align Alignment =
10893 DataLayout.getABITypeAlign(Ty: MemVT.getTypeForEVT(Context&: *DAG.getContext()));
10894
10895 MMO = MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
10896 F: MachineMemOperand::MOLoad |
10897 MachineMemOperand::MODereferenceable |
10898 MachineMemOperand::MOInvariant,
10899 Size: MemVT.getStoreSize(), BaseAlignment: Alignment);
10900 }
10901
10902 if (!Offset->isDivergent()) {
10903 SDValue Ops[] = {Chain, Rsrc, Offset, CachePolicy};
10904
10905 // Lower llvm.amdgcn.*s.buffer.load.{i,u}N intrinsics. First, generate
10906 // s_buffer_load_u* for signed and unsigned load instructions. Next, DAG
10907 // combiner tries to merge the s_buffer_load_uN with a sext instruction
10908 // (performSignExtendInRegCombine()) and it replaces s_buffer_load_uN with
10909 // s_buffer_load_iN.
10910 auto HandleScalarSubwordLoads = [&](unsigned Opcode) -> SDValue {
10911 SDValue BufferLoad = DAG.getMemIntrinsicNode(
10912 Opcode, dl: DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops, MemVT, MMO);
10913 SDValue LoadVal = DAG.getAnyExtOrTrunc(
10914 Op: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MemVT, Operand: BufferLoad), DL, VT);
10915 if (HasChainResult)
10916 return DAG.getMergeValues(Ops: {LoadVal, BufferLoad.getValue(R: 1)}, dl: DL);
10917 return LoadVal;
10918 };
10919 if (MemVT == MVT::i8 && Subtarget->hasScalarSubwordLoads())
10920 return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_UBYTE);
10921
10922 if (MemVT == MVT::i16 && Subtarget->hasScalarSubwordLoads())
10923 return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_USHORT);
10924
10925 // Widen vec3 load to vec4. Only 32-bit elements have a vec4 pattern.
10926 if (VT.isVector() && VT.getVectorNumElements() == 3 &&
10927 VT.getVectorElementType().getSizeInBits() == 32 &&
10928 !Subtarget->hasScalarDwordx3Loads()) {
10929 EVT WidenedVT =
10930 EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(), NumElements: 4);
10931 auto WidenedOp = DAG.getMemIntrinsicNode(
10932 Opcode: AMDGPUISD::SBUFFER_LOAD, dl: DL, VTList: DAG.getVTList(VT1: WidenedVT, VT2: MVT::Other),
10933 Ops, MemVT: WidenedVT,
10934 MMO: MF.getMachineMemOperand(MMO, Offset: 0, Size: WidenedVT.getStoreSize()));
10935 auto Subvector = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: WidenedOp,
10936 N2: DAG.getVectorIdxConstant(Val: 0, DL));
10937 if (HasChainResult)
10938 return DAG.getMergeValues(Ops: {Subvector, WidenedOp.getValue(R: 1)}, dl: DL);
10939 return Subvector;
10940 }
10941
10942 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::SBUFFER_LOAD, dl: DL,
10943 VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), Ops, MemVT,
10944 MMO);
10945 }
10946
10947 // We have a divergent offset. Emit a MUBUF buffer load instead. We can
10948 // assume that the buffer is unswizzled.
10949 SDValue Ops[] = {
10950 Chain, // Chain
10951 Rsrc, // rsrc
10952 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
10953 {}, // voffset
10954 {}, // soffset
10955 {}, // offset
10956 CachePolicy, // cachepolicy
10957 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
10958 };
10959 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
10960 Subtarget->hasScalarSubwordLoads()) {
10961 setBufferOffsets(CombinedOffset: Offset, DAG, Offsets: &Ops[3], Alignment: Align(4));
10962 SDValue Load = handleByteShortBufferLoads(DAG, LoadVT: MemVT, DL, Ops, MMO);
10963 SDValue LoadVal = DAG.getAnyExtOrTrunc(Op: Load.getOperand(i: 0), DL, VT);
10964 if (HasChainResult)
10965 return DAG.getMergeValues(Ops: {LoadVal, Load.getOperand(i: 1)}, dl: DL);
10966 return LoadVal;
10967 }
10968
10969 SmallVector<SDValue, 4> Loads;
10970 unsigned NumLoads = 1;
10971 MVT LoadVT = VT.getSimpleVT();
10972 unsigned NumElts = LoadVT.isVector() ? LoadVT.getVectorNumElements() : 1;
10973 assert((LoadVT.getScalarType() == MVT::i32 ||
10974 LoadVT.getScalarType() == MVT::f32));
10975
10976 if (NumElts == 8 || NumElts == 16) {
10977 NumLoads = NumElts / 4;
10978 LoadVT = MVT::getVectorVT(VT: LoadVT.getScalarType(), NumElements: 4);
10979 }
10980
10981 SDVTList VTList = DAG.getVTList(VTs: {LoadVT, MVT::Other});
10982
10983 // Use the alignment to ensure that the required offsets will fit into the
10984 // immediate offsets.
10985 setBufferOffsets(CombinedOffset: Offset, DAG, Offsets: &Ops[3],
10986 Alignment: NumLoads > 1 ? Align(16 * NumLoads) : Align(4));
10987
10988 uint64_t InstOffset = Ops[5]->getAsZExtVal();
10989 unsigned LoadSize = LoadVT.getStoreSize();
10990 for (unsigned i = 0; i < NumLoads; ++i) {
10991 Ops[5] = DAG.getTargetConstant(Val: InstOffset + 16 * i, DL, VT: MVT::i32);
10992 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(MMO, Offset: 16 * i, Size: LoadSize);
10993 Loads.push_back(Elt: getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_LOAD, DL, VTList, Ops,
10994 MemVT: LoadVT, MMO: LoadMMO, DAG));
10995 }
10996
10997 if (NumElts == 8 || NumElts == 16) {
10998 SDValue LoadVal = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: Loads);
10999 if (HasChainResult) {
11000 SmallVector<SDValue, 4> LoadChains;
11001 for (SDValue Load : Loads)
11002 LoadChains.push_back(Elt: Load.getValue(R: 1));
11003 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: LoadChains);
11004 return DAG.getMergeValues(Ops: {LoadVal, Chain}, dl: DL);
11005 }
11006 return LoadVal;
11007 }
11008
11009 return Loads[0];
11010}
11011
11012SDValue SITargetLowering::lowerWaveID(SelectionDAG &DAG, SDValue Op) const {
11013 // With architected SGPRs, waveIDinGroup is in TTMP8[29:25].
11014 if (!Subtarget->hasArchitectedSGPRs())
11015 return {};
11016 SDLoc SL(Op);
11017 MVT VT = MVT::i32;
11018 SDValue TTMP8 = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: AMDGPU::TTMP8, VT);
11019 return DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT, N1: TTMP8,
11020 N2: DAG.getConstant(Val: 25, DL: SL, VT), N3: DAG.getConstant(Val: 5, DL: SL, VT));
11021}
11022
11023SDValue SITargetLowering::lowerConstHwRegRead(SelectionDAG &DAG, SDValue Op,
11024 AMDGPU::Hwreg::Id HwReg,
11025 unsigned LowBit,
11026 unsigned Width) const {
11027 SDLoc SL(Op);
11028 using namespace AMDGPU::Hwreg;
11029 return {DAG.getMachineNode(
11030 Opcode: AMDGPU::S_GETREG_B32_const, dl: SL, VT: MVT::i32,
11031 Op1: DAG.getTargetConstant(Val: HwregEncoding::encode(Values: HwReg, Values: LowBit, Values: Width),
11032 DL: SL, VT: MVT::i32)),
11033 0};
11034}
11035
11036SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
11037 unsigned Dim,
11038 const ArgDescriptor &Arg) const {
11039 SDLoc SL(Op);
11040 MachineFunction &MF = DAG.getMachineFunction();
11041 unsigned MaxID = Subtarget->getMaxWorkitemID(Kernel: MF.getFunction(), Dimension: Dim);
11042 if (MaxID == 0)
11043 return DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
11044
11045 // It's undefined behavior if a function marked with the amdgpu-no-*
11046 // attributes uses the corresponding intrinsic.
11047 if (!Arg)
11048 return DAG.getPOISON(VT: Op->getValueType(ResNo: 0));
11049
11050 SDValue Val = loadInputValue(DAG, RC: &AMDGPU::VGPR_32RegClass, VT: MVT::i32,
11051 SL: SDLoc(DAG.getEntryNode()), Arg);
11052
11053 // Don't bother inserting AssertZext for packed IDs since we're emitting the
11054 // masking operations anyway.
11055 //
11056 // TODO: We could assert the top bit is 0 for the source copy.
11057 if (Arg.isMasked())
11058 return Val;
11059
11060 // Preserve the known bits after expansion to a copy.
11061 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: llvm::bit_width(Value: MaxID));
11062 return DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: MVT::i32, N1: Val,
11063 N2: DAG.getValueType(SmallVT));
11064}
11065
11066SDValue SITargetLowering::lowerFromFP8(SDValue Op, bool IsBF8,
11067 SelectionDAG &DAG) const {
11068 SDLoc SL(Op);
11069 SDValue Src = Op.getOperand(i: 0);
11070 EVT DstVT = Op.getValueType();
11071 bool IsF16 = DstVT.getVectorElementType() == MVT::f16;
11072 assert((!IsF16 || Subtarget->hasFP8F16ConversionInsts()) &&
11073 "fp8/bf8 -> f16 conversion requires FP8F16ConversionInsts");
11074
11075 unsigned Opc;
11076 if (IsF16)
11077 Opc = IsBF8 ? AMDGPUISD::CVT_PK_F16_BF8 : AMDGPUISD::CVT_PK_F16_FP8;
11078 else
11079 Opc = IsBF8 ? AMDGPUISD::CVT_PK_F32_BF8 : AMDGPUISD::CVT_PK_F32_FP8;
11080
11081 // Pack the two i8 lanes into the integer type the packed HW node reads. The
11082 // f16 form takes i16 and the f32 form takes i32. v2i8 bitcasts to i16
11083 // directly and the f32 node reads the low half of an any-extended i32.
11084 EVT PackedVT =
11085 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DstVT.getScalarSizeInBits());
11086 SDValue AsI16 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Src);
11087 SDValue Packed = DAG.getAnyExtOrTrunc(Op: AsI16, DL: SL, VT: PackedVT);
11088 return DAG.getNode(Opcode: Opc, DL: SL, VT: DstVT, Operand: Packed);
11089}
11090
11091SDValue
11092SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
11093 SelectionDAG &DAG) const {
11094 // Handle the OCP FP8 formats (E4M3FN, E5M2) and unsigned E5M3 on subtargets
11095 // with matching HW conversions. Other formats use the generic expansion.
11096 APFloatBase::Semantics FPSemantic =
11097 static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(i: 1));
11098 const bool IsFP8 = FPSemantic == APFloatBase::S_Float8E4M3FN;
11099 const bool IsBF8 = FPSemantic == APFloatBase::S_Float8E5M2;
11100 const bool IsE5M3 = FPSemantic == APFloatBase::S_Float8E5M3FNU;
11101 const bool HasE5M3ConversionInsts =
11102 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
11103 const bool IsSupported = IsFP8 || IsBF8 || (IsE5M3 && HasE5M3ConversionInsts);
11104 if (!IsSupported)
11105 return SDValue();
11106
11107 EVT DstVT = Op.getValueType();
11108 if (IsE5M3) {
11109 if (DstVT.getScalarType() != MVT::f32)
11110 return SDValue();
11111
11112 SDLoc SL(Op);
11113 SDValue Src = Op.getOperand(i: 0);
11114 assert((!DstVT.isVector() || DstVT == MVT::v2f32) &&
11115 "only the v2f32 vector result is custom lowered");
11116
11117 if (DstVT.isVector())
11118 Src = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Src);
11119 Src = DAG.getAnyExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32);
11120
11121 auto ConvertByte = [&](unsigned ByteSel) {
11122 return DAG.getNode(Opcode: AMDGPUISD::CVT_F32_FP8_E5M3, DL: SL, VT: MVT::f32, N1: Src,
11123 N2: DAG.getTargetConstant(Val: ByteSel, DL: SL, VT: MVT::i32));
11124 };
11125
11126 if (!DstVT.isVector())
11127 return ConvertByte(0);
11128 return DAG.getBuildVector(VT: DstVT, DL: SL, Ops: {ConvertByte(0), ConvertByte(1)});
11129 }
11130
11131 if (!DstVT.isVector()) {
11132 SDValue Src = Op.getOperand(i: 0);
11133 if (Src.getValueType() != MVT::i32) {
11134 SDLoc SL(Op);
11135 SDValue SrcI32 = DAG.getAnyExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32);
11136 return DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL: SL, VT: DstVT, N1: SrcI32,
11137 N2: Op.getOperand(i: 1));
11138 }
11139 return Op;
11140 }
11141
11142 EVT EltVT = DstVT.getVectorElementType();
11143 if (EltVT == MVT::f16 || EltVT == MVT::f32)
11144 return lowerFromFP8(Op, IsBF8, DAG);
11145 return SDValue();
11146}
11147
11148SDValue SITargetLowering::lowerToFP8(SDValue Op, bool IsBF8, bool IsE5M3,
11149 SelectionDAG &DAG) const {
11150 SDLoc SL(Op);
11151 SDValue Src = Op.getOperand(i: 0);
11152 EVT ResVT = Op.getValueType();
11153 bool IsF16 = Src.getValueType().getScalarType() == MVT::f16;
11154 assert((!IsF16 || Subtarget->hasF16FP8ConversionInsts()) &&
11155 "f16 -> fp8/bf8 conversion requires F16FP8ConversionInsts");
11156 assert((!ResVT.isVector() || ResVT == MVT::v2i8) &&
11157 "only the v2i8 vector result is custom lowered");
11158
11159 if (IsF16) {
11160 unsigned Opc =
11161 IsBF8 ? AMDGPUISD::CVT_PK_BF8_F16 : AMDGPUISD::CVT_PK_FP8_F16;
11162 SDValue Bytes = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i16, Operand: Src);
11163 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ResVT, Operand: Bytes);
11164 }
11165
11166 unsigned Opc = IsBF8 ? AMDGPUISD::CVT_PK_BF8_F32
11167 : IsE5M3 ? AMDGPUISD::CVT_PK_FP8_F32_E5M3
11168 : AMDGPUISD::CVT_PK_FP8_F32;
11169 SDValue PoisonI32 = DAG.getPOISON(VT: MVT::i32);
11170 SDValue WordSel = DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i1);
11171
11172 if (!ResVT.isVector()) {
11173 // Convert one lane, the second is unused. Feed it the same source so the
11174 // instruction does not read an undefined register.
11175 SDValue Packed =
11176 DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: Src, N2: Src, N3: PoisonI32, N4: WordSel);
11177 return DAG.getAnyExtOrTrunc(Op: Packed, DL: SL, VT: ResVT);
11178 }
11179
11180 SDValue A = DAG.getExtractVectorElt(DL: SL, VT: MVT::f32, Vec: Src, Idx: 0);
11181 SDValue B = DAG.getExtractVectorElt(DL: SL, VT: MVT::f32, Vec: Src, Idx: 1);
11182 SDValue Packed = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: A, N2: B, N3: PoisonI32, N4: WordSel);
11183 SDValue Bytes = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i16, Operand: Packed);
11184 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ResVT, Operand: Bytes);
11185}
11186
11187SDValue
11188SITargetLowering::LowerCONVERT_TO_ARBITRARY_FP(SDValue Op,
11189 SelectionDAG &DAG) const {
11190 // The OCP FP8 formats (E4M3FN, E5M2) and unsigned E5M3 map to HW conversions
11191 // on subtargets that support them. Everything else uses generic expansion.
11192 APFloatBase::Semantics Sem =
11193 static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(i: 1));
11194 const bool IsFP8 = Sem == APFloatBase::S_Float8E4M3FN;
11195 const bool IsBF8 = Sem == APFloatBase::S_Float8E5M2;
11196 const bool IsE5M3 = Sem == APFloatBase::S_Float8E5M3FNU;
11197 const bool HasE5M3ConversionInsts =
11198 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
11199 const bool IsSupported = IsFP8 || IsBF8 || (IsE5M3 && HasE5M3ConversionInsts);
11200 if (!IsSupported)
11201 return SDValue();
11202
11203 // The HW conversions only support nearest-even. The OCP conversions do not
11204 // saturate. The unsigned E5M3 conversion always clamps out-of-range inputs,
11205 // which also refines the non-saturating form where those inputs are poison.
11206 if (static_cast<RoundingMode>(Op.getConstantOperandVal(i: 2)) !=
11207 RoundingMode::NearestTiesToEven)
11208 return SDValue();
11209 if (!IsE5M3 && Op.getConstantOperandVal(i: 3) != 0)
11210 return SDValue();
11211
11212 EVT SrcEltVT = Op.getOperand(i: 0).getValueType().getScalarType();
11213 // The f32 form is built here rather than by a tablegen pattern because the
11214 // HW result is i32 while the node result is i16 after the i8 promotion.
11215 if (SrcEltVT == MVT::f32)
11216 return lowerToFP8(Op, IsBF8, IsE5M3, DAG);
11217 if (!IsE5M3 && SrcEltVT == MVT::f16 &&
11218 Subtarget->hasF16FP8ConversionInsts()) {
11219 // A scalar conversion is selected from the generic node by tablegen, only
11220 // the illegal v2i8 result type needs lowering here.
11221 if (!Op.getValueType().isVector())
11222 return Op;
11223 return lowerToFP8(Op, IsBF8, IsE5M3: false, DAG);
11224 }
11225 return SDValue();
11226}
11227
11228SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
11229 SelectionDAG &DAG) const {
11230 MachineFunction &MF = DAG.getMachineFunction();
11231 auto *MFI = MF.getInfo<SIMachineFunctionInfo>();
11232
11233 EVT VT = Op.getValueType();
11234 SDLoc DL(Op);
11235 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
11236
11237 // TODO: Should this propagate fast-math-flags?
11238
11239 switch (IntrinsicID) {
11240 case Intrinsic::amdgcn_wave_reduce_min:
11241 case Intrinsic::amdgcn_wave_reduce_umin:
11242 case Intrinsic::amdgcn_wave_reduce_fmin:
11243 case Intrinsic::amdgcn_wave_reduce_max:
11244 case Intrinsic::amdgcn_wave_reduce_umax:
11245 case Intrinsic::amdgcn_wave_reduce_fmax:
11246 case Intrinsic::amdgcn_wave_reduce_add:
11247 case Intrinsic::amdgcn_wave_reduce_fadd:
11248 case Intrinsic::amdgcn_wave_reduce_sub:
11249 case Intrinsic::amdgcn_wave_reduce_fsub:
11250 case Intrinsic::amdgcn_wave_reduce_and:
11251 case Intrinsic::amdgcn_wave_reduce_or:
11252 case Intrinsic::amdgcn_wave_reduce_xor: {
11253 EVT SrcVT = Op.getOperand(i: 1).getValueType();
11254 if (SrcVT.getFixedSizeInBits() == 16) {
11255 bool IsFPOp = SrcVT.isFloatingPoint();
11256 bool NeedsSignExt = IntrinsicID == Intrinsic::amdgcn_wave_reduce_min ||
11257 IntrinsicID == Intrinsic::amdgcn_wave_reduce_max ||
11258 IntrinsicID == Intrinsic::amdgcn_wave_reduce_add ||
11259 IntrinsicID == Intrinsic::amdgcn_wave_reduce_sub;
11260 unsigned ExtOpc = IsFPOp ? ISD::FP_EXTEND
11261 : NeedsSignExt ? ISD::SIGN_EXTEND
11262 : ISD::ZERO_EXTEND;
11263 auto SrcType = IsFPOp ? MVT::f16 : MVT::i16;
11264 auto ExtType = IsFPOp ? MVT::f32 : MVT::i32;
11265 SDValue ExtendedSrc = DAG.getNode(Opcode: ExtOpc, DL, VT: ExtType, Operand: Op.getOperand(i: 1));
11266 SDValue Strategy = Op.getOperand(i: 2);
11267 SDValue Result = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: ExtType,
11268 N1: Op.getOperand(i: 0), N2: ExtendedSrc, N3: Strategy);
11269 if (IsFPOp)
11270 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: SrcType, N1: Result,
11271 N2: DAG.getTargetConstant(Val: 1, DL, VT: MVT::i32));
11272 else
11273 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: SrcType, Operand: Result);
11274 }
11275 return SDValue();
11276 }
11277 case Intrinsic::amdgcn_implicit_buffer_ptr: {
11278 if (getSubtarget()->isAmdHsaOrMesa(F: MF.getFunction()))
11279 return emitNonHSAIntrinsicError(DAG, DL, VT);
11280 return getPreloadedValue(DAG, MFI: *MFI, VT,
11281 PVID: AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR);
11282 }
11283 case Intrinsic::amdgcn_dispatch_ptr:
11284 case Intrinsic::amdgcn_queue_ptr: {
11285 if (!Subtarget->isAmdHsaOrMesa(F: MF.getFunction())) {
11286 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11287 MF.getFunction(), "unsupported hsa intrinsic without hsa target",
11288 DL.getDebugLoc()));
11289 return DAG.getPOISON(VT);
11290 }
11291
11292 auto RegID = IntrinsicID == Intrinsic::amdgcn_dispatch_ptr
11293 ? AMDGPUFunctionArgInfo::DISPATCH_PTR
11294 : AMDGPUFunctionArgInfo::QUEUE_PTR;
11295 return getPreloadedValue(DAG, MFI: *MFI, VT, PVID: RegID);
11296 }
11297 case Intrinsic::amdgcn_implicitarg_ptr: {
11298 if (MFI->isEntryFunction())
11299 return getImplicitArgPtr(DAG, SL: DL);
11300 return getPreloadedValue(DAG, MFI: *MFI, VT,
11301 PVID: AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
11302 }
11303 case Intrinsic::amdgcn_kernarg_segment_ptr: {
11304 if (!AMDGPU::isKernel(F: MF.getFunction())) {
11305 // This only makes sense to call in a kernel, so just lower to null.
11306 return DAG.getConstant(Val: 0, DL, VT);
11307 }
11308
11309 return getPreloadedValue(DAG, MFI: *MFI, VT,
11310 PVID: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
11311 }
11312 case Intrinsic::amdgcn_dispatch_id: {
11313 return getPreloadedValue(DAG, MFI: *MFI, VT, PVID: AMDGPUFunctionArgInfo::DISPATCH_ID);
11314 }
11315 case Intrinsic::amdgcn_rcp:
11316 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT, Operand: Op.getOperand(i: 1));
11317 case Intrinsic::amdgcn_rsq:
11318 return DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: Op.getOperand(i: 1));
11319 case Intrinsic::amdgcn_rsq_legacy:
11320 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
11321 return emitRemovedIntrinsicError(DAG, DL, VT);
11322 return SDValue();
11323 case Intrinsic::amdgcn_rcp_legacy:
11324 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
11325 return emitRemovedIntrinsicError(DAG, DL, VT);
11326 return DAG.getNode(Opcode: AMDGPUISD::RCP_LEGACY, DL, VT, Operand: Op.getOperand(i: 1));
11327 case Intrinsic::amdgcn_fma_legacy:
11328 case Intrinsic::amdgcn_sudot4:
11329 case Intrinsic::amdgcn_sudot8:
11330 case Intrinsic::amdgcn_tanh:
11331 return SDValue();
11332 case Intrinsic::amdgcn_rsq_clamp: {
11333 if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
11334 return DAG.getNode(Opcode: AMDGPUISD::RSQ_CLAMP, DL, VT, Operand: Op.getOperand(i: 1));
11335
11336 Type *Type = VT.getTypeForEVT(Context&: *DAG.getContext());
11337 APFloat Max = APFloat::getLargest(Sem: Type->getFltSemantics());
11338 APFloat Min = APFloat::getLargest(Sem: Type->getFltSemantics(), Negative: true);
11339
11340 SDValue Rsq = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: Op.getOperand(i: 1));
11341 SDValue Tmp =
11342 DAG.getNode(Opcode: ISD::FMINNUM, DL, VT, N1: Rsq, N2: DAG.getConstantFP(Val: Max, DL, VT));
11343 return DAG.getNode(Opcode: ISD::FMAXNUM, DL, VT, N1: Tmp,
11344 N2: DAG.getConstantFP(Val: Min, DL, VT));
11345 }
11346 case Intrinsic::r600_read_ngroups_x:
11347 if (Subtarget->isAmdHsaOS())
11348 return emitNonHSAIntrinsicError(DAG, DL, VT);
11349
11350 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
11351 Offset: SI::KernelInputOffsets::NGROUPS_X, Alignment: Align(4),
11352 Signed: false);
11353 case Intrinsic::r600_read_ngroups_y:
11354 if (Subtarget->isAmdHsaOS())
11355 return emitNonHSAIntrinsicError(DAG, DL, VT);
11356
11357 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
11358 Offset: SI::KernelInputOffsets::NGROUPS_Y, Alignment: Align(4),
11359 Signed: false);
11360 case Intrinsic::r600_read_ngroups_z:
11361 if (Subtarget->isAmdHsaOS())
11362 return emitNonHSAIntrinsicError(DAG, DL, VT);
11363
11364 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
11365 Offset: SI::KernelInputOffsets::NGROUPS_Z, Alignment: Align(4),
11366 Signed: false);
11367 case Intrinsic::r600_read_local_size_x:
11368 if (Subtarget->isAmdHsaOS())
11369 return emitNonHSAIntrinsicError(DAG, DL, VT);
11370
11371 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
11372 Offset: SI::KernelInputOffsets::LOCAL_SIZE_X);
11373 case Intrinsic::r600_read_local_size_y:
11374 if (Subtarget->isAmdHsaOS())
11375 return emitNonHSAIntrinsicError(DAG, DL, VT);
11376
11377 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
11378 Offset: SI::KernelInputOffsets::LOCAL_SIZE_Y);
11379 case Intrinsic::r600_read_local_size_z:
11380 if (Subtarget->isAmdHsaOS())
11381 return emitNonHSAIntrinsicError(DAG, DL, VT);
11382
11383 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
11384 Offset: SI::KernelInputOffsets::LOCAL_SIZE_Z);
11385 case Intrinsic::amdgcn_workgroup_id_x:
11386 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
11387 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_X,
11388 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X,
11389 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X);
11390 case Intrinsic::amdgcn_workgroup_id_y:
11391 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
11392 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,
11393 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y,
11394 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y);
11395 case Intrinsic::amdgcn_workgroup_id_z:
11396 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
11397 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,
11398 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z,
11399 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z);
11400 case Intrinsic::amdgcn_cluster_id_x:
11401 return Subtarget->hasClusters()
11402 ? getPreloadedValue(DAG, MFI: *MFI, VT,
11403 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_X)
11404 : DAG.getPOISON(VT);
11405 case Intrinsic::amdgcn_cluster_id_y:
11406 return Subtarget->hasClusters()
11407 ? getPreloadedValue(DAG, MFI: *MFI, VT,
11408 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_Y)
11409 : DAG.getPOISON(VT);
11410 case Intrinsic::amdgcn_cluster_id_z:
11411 return Subtarget->hasClusters()
11412 ? getPreloadedValue(DAG, MFI: *MFI, VT,
11413 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_Z)
11414 : DAG.getPOISON(VT);
11415 case Intrinsic::amdgcn_cluster_workgroup_id_x:
11416 return Subtarget->hasClusters()
11417 ? getPreloadedValue(
11418 DAG, MFI: *MFI, VT,
11419 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X)
11420 : DAG.getPOISON(VT);
11421 case Intrinsic::amdgcn_cluster_workgroup_id_y:
11422 return Subtarget->hasClusters()
11423 ? getPreloadedValue(
11424 DAG, MFI: *MFI, VT,
11425 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y)
11426 : DAG.getPOISON(VT);
11427 case Intrinsic::amdgcn_cluster_workgroup_id_z:
11428 return Subtarget->hasClusters()
11429 ? getPreloadedValue(
11430 DAG, MFI: *MFI, VT,
11431 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z)
11432 : DAG.getPOISON(VT);
11433 case Intrinsic::amdgcn_cluster_workgroup_flat_id:
11434 return Subtarget->hasClusters()
11435 ? lowerConstHwRegRead(DAG, Op, HwReg: AMDGPU::Hwreg::ID_IB_STS2, LowBit: 21, Width: 4)
11436 : SDValue();
11437 case Intrinsic::amdgcn_cluster_workgroup_max_id_x:
11438 return Subtarget->hasClusters()
11439 ? getPreloadedValue(
11440 DAG, MFI: *MFI, VT,
11441 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X)
11442 : DAG.getPOISON(VT);
11443 case Intrinsic::amdgcn_cluster_workgroup_max_id_y:
11444 return Subtarget->hasClusters()
11445 ? getPreloadedValue(
11446 DAG, MFI: *MFI, VT,
11447 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y)
11448 : DAG.getPOISON(VT);
11449 case Intrinsic::amdgcn_cluster_workgroup_max_id_z:
11450 return Subtarget->hasClusters()
11451 ? getPreloadedValue(
11452 DAG, MFI: *MFI, VT,
11453 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z)
11454 : DAG.getPOISON(VT);
11455 case Intrinsic::amdgcn_cluster_workgroup_max_flat_id:
11456 return Subtarget->hasClusters()
11457 ? getPreloadedValue(
11458 DAG, MFI: *MFI, VT,
11459 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID)
11460 : DAG.getPOISON(VT);
11461 case Intrinsic::amdgcn_wave_id:
11462 return lowerWaveID(DAG, Op);
11463 case Intrinsic::amdgcn_lds_kernel_id: {
11464 if (MFI->isEntryFunction())
11465 return getLDSKernelId(DAG, SL: DL);
11466 return getPreloadedValue(DAG, MFI: *MFI, VT,
11467 PVID: AMDGPUFunctionArgInfo::LDS_KERNEL_ID);
11468 }
11469 case Intrinsic::amdgcn_workitem_id_x:
11470 return lowerWorkitemID(DAG, Op, Dim: 0, Arg: MFI->getArgInfo().WorkItemIDX);
11471 case Intrinsic::amdgcn_workitem_id_y:
11472 return lowerWorkitemID(DAG, Op, Dim: 1, Arg: MFI->getArgInfo().WorkItemIDY);
11473 case Intrinsic::amdgcn_workitem_id_z:
11474 return lowerWorkitemID(DAG, Op, Dim: 2, Arg: MFI->getArgInfo().WorkItemIDZ);
11475 case Intrinsic::amdgcn_wavefrontsize:
11476 return DAG.getConstant(Val: MF.getSubtarget<GCNSubtarget>().getWavefrontSize(),
11477 DL: SDLoc(Op), VT: MVT::i32);
11478 case Intrinsic::amdgcn_s_buffer_load: {
11479 unsigned CPol = Op.getConstantOperandVal(i: 3);
11480 // s_buffer_load, because of how it's optimized, can't be volatile
11481 // so reject ones with the volatile bit set.
11482 if (CPol & ~((Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12)
11483 ? AMDGPU::CPol::ALL
11484 : AMDGPU::CPol::ALL_pregfx12))
11485 return Op;
11486 return lowerSBuffer(VT, MemVT: VT, DL, Chain: DAG.getEntryNode(), Rsrc: Op.getOperand(i: 1),
11487 Offset: Op.getOperand(i: 2), CachePolicy: Op.getOperand(i: 3), DAG);
11488 }
11489 case Intrinsic::amdgcn_fdiv_fast:
11490 return lowerFDIV_FAST(Op, DAG);
11491 case Intrinsic::amdgcn_sin:
11492 return DAG.getNode(Opcode: AMDGPUISD::SIN_HW, DL, VT, Operand: Op.getOperand(i: 1));
11493
11494 case Intrinsic::amdgcn_cos:
11495 return DAG.getNode(Opcode: AMDGPUISD::COS_HW, DL, VT, Operand: Op.getOperand(i: 1));
11496
11497 case Intrinsic::amdgcn_mul_u24:
11498 return DAG.getNode(Opcode: AMDGPUISD::MUL_U24, DL, VT, N1: Op.getOperand(i: 1),
11499 N2: Op.getOperand(i: 2));
11500 case Intrinsic::amdgcn_mul_i24:
11501 return DAG.getNode(Opcode: AMDGPUISD::MUL_I24, DL, VT, N1: Op.getOperand(i: 1),
11502 N2: Op.getOperand(i: 2));
11503
11504 case Intrinsic::amdgcn_log_clamp: {
11505 if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
11506 return SDValue();
11507
11508 return emitRemovedIntrinsicError(DAG, DL, VT);
11509 }
11510 case Intrinsic::amdgcn_fract:
11511 return DAG.getNode(Opcode: AMDGPUISD::FRACT, DL, VT, Operand: Op.getOperand(i: 1));
11512
11513 case Intrinsic::amdgcn_class: {
11514 SDValue Src = Op.getOperand(i: 1);
11515 EVT SrcVT = Src.getValueType();
11516 bool IsLegal = SrcVT == MVT::f32 || SrcVT == MVT::f64 ||
11517 (SrcVT == MVT::f16 && Subtarget->has16BitInsts());
11518 if (!IsLegal) {
11519 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11520 DAG.getMachineFunction().getFunction(),
11521 "llvm.amdgcn.class only supports f16, f32, and f64",
11522 DL.getDebugLoc()));
11523 return DAG.getPOISON(VT);
11524 }
11525 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT, N1: Src, N2: Op.getOperand(i: 2));
11526 }
11527 case Intrinsic::amdgcn_div_fmas:
11528 return DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL, VT, N1: Op.getOperand(i: 1),
11529 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), N4: Op.getOperand(i: 4));
11530
11531 case Intrinsic::amdgcn_div_fixup:
11532 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL, VT, N1: Op.getOperand(i: 1),
11533 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11534
11535 case Intrinsic::amdgcn_div_scale: {
11536 const ConstantSDNode *Param = cast<ConstantSDNode>(Val: Op.getOperand(i: 3));
11537
11538 // Translate to the operands expected by the machine instruction. The
11539 // first parameter must be the same as the first instruction.
11540 SDValue Numerator = Op.getOperand(i: 1);
11541 SDValue Denominator = Op.getOperand(i: 2);
11542
11543 // Note this order is opposite of the machine instruction's operations,
11544 // which is s0.f = Quotient, s1.f = Denominator, s2.f = Numerator. The
11545 // intrinsic has the numerator as the first operand to match a normal
11546 // division operation.
11547
11548 SDValue Src0 = Param->isAllOnes() ? Numerator : Denominator;
11549
11550 return DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL, VTList: Op->getVTList(), N1: Src0,
11551 N2: Denominator, N3: Numerator);
11552 }
11553 case Intrinsic::amdgcn_ballot:
11554 return lowerBALLOTIntrinsic(TLI: *this, N: Op.getNode(), DAG);
11555 case Intrinsic::amdgcn_fmed3:
11556 return DAG.getNode(Opcode: AMDGPUISD::FMED3, DL, VT, N1: Op.getOperand(i: 1),
11557 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), Flags: Op->getFlags());
11558 case Intrinsic::amdgcn_fdot2:
11559 return DAG.getNode(Opcode: AMDGPUISD::FDOT2, DL, VT, N1: Op.getOperand(i: 1),
11560 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), N4: Op.getOperand(i: 4));
11561 case Intrinsic::amdgcn_fmul_legacy:
11562 return DAG.getNode(Opcode: AMDGPUISD::FMUL_LEGACY, DL, VT, N1: Op.getOperand(i: 1),
11563 N2: Op.getOperand(i: 2));
11564 case Intrinsic::amdgcn_sbfe:
11565 case Intrinsic::amdgcn_ubfe:
11566 return lowerBFEIntrinsic(Op, DAG, IntrinsicID);
11567 case Intrinsic::amdgcn_cvt_pkrtz:
11568 case Intrinsic::amdgcn_cvt_pknorm_i16:
11569 case Intrinsic::amdgcn_cvt_pknorm_u16:
11570 case Intrinsic::amdgcn_cvt_pk_i16:
11571 case Intrinsic::amdgcn_cvt_pk_u16: {
11572 // FIXME: Stop adding cast if v2f16/v2i16 are legal.
11573 EVT VT = Op.getValueType();
11574 unsigned Opcode;
11575
11576 if (IntrinsicID == Intrinsic::amdgcn_cvt_pkrtz)
11577 Opcode = AMDGPUISD::CVT_PKRTZ_F16_F32;
11578 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_i16)
11579 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
11580 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_u16)
11581 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
11582 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pk_i16)
11583 Opcode = AMDGPUISD::CVT_PK_I16_I32;
11584 else
11585 Opcode = AMDGPUISD::CVT_PK_U16_U32;
11586
11587 if (isTypeLegal(VT))
11588 return DAG.getNode(Opcode, DL, VT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
11589
11590 SDValue Node =
11591 DAG.getNode(Opcode, DL, VT: MVT::i32, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
11592 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Node);
11593 }
11594 case Intrinsic::amdgcn_fmad_ftz:
11595 return DAG.getNode(Opcode: AMDGPUISD::FMAD_FTZ, DL, VT, N1: Op.getOperand(i: 1),
11596 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11597
11598 case Intrinsic::amdgcn_if_break:
11599 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::SI_IF_BREAK, dl: DL, VT,
11600 Op1: Op->getOperand(Num: 1), Op2: Op->getOperand(Num: 2)),
11601 0);
11602
11603 case Intrinsic::amdgcn_groupstaticsize: {
11604 Triple::OSType OS = getTargetMachine().getTargetTriple().getOS();
11605 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL)
11606 return Op;
11607
11608 const Module *M = MF.getFunction().getParent();
11609 const GlobalValue *GV =
11610 Intrinsic::getDeclarationIfExists(M, id: Intrinsic::amdgcn_groupstaticsize);
11611 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: 0,
11612 TargetFlags: SIInstrInfo::MO_ABS32_LO);
11613 return {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: GA), 0};
11614 }
11615 case Intrinsic::amdgcn_is_shared:
11616 case Intrinsic::amdgcn_is_private: {
11617 SDLoc SL(Op);
11618 SDValue SrcVec =
11619 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
11620 SDValue SrcHi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: SrcVec,
11621 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
11622
11623 unsigned AS = (IntrinsicID == Intrinsic::amdgcn_is_shared)
11624 ? AMDGPUAS::LOCAL_ADDRESS
11625 : AMDGPUAS::PRIVATE_ADDRESS;
11626 if (AS == AMDGPUAS::PRIVATE_ADDRESS &&
11627 Subtarget->hasGloballyAddressableScratch()) {
11628 SDValue FlatScratchBaseHi(
11629 DAG.getMachineNode(
11630 Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32,
11631 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, VT: MVT::i32)),
11632 0);
11633 // Test bits 63..58 against the aperture address.
11634 return DAG.getSetCC(
11635 DL: SL, VT: MVT::i1,
11636 LHS: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: SrcHi, N2: FlatScratchBaseHi),
11637 RHS: DAG.getConstant(Val: 1u << 26, DL: SL, VT: MVT::i32), Cond: ISD::SETULT);
11638 }
11639
11640 SDValue Aperture = getSegmentAperture(AS, DL: SL, DAG);
11641 return DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: SrcHi, RHS: Aperture, Cond: ISD::SETEQ);
11642 }
11643 case Intrinsic::amdgcn_perm:
11644 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: Op.getOperand(i: 1),
11645 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11646 case Intrinsic::amdgcn_reloc_constant: {
11647 Module *M = MF.getFunction().getParent();
11648 const MDNode *Metadata = cast<MDNodeSDNode>(Val: Op.getOperand(i: 1))->getMD();
11649 auto SymbolName = cast<MDString>(Val: Metadata->getOperand(I: 0))->getString();
11650 auto *RelocSymbol = cast<GlobalVariable>(
11651 Val: M->getOrInsertGlobal(Name: SymbolName, Ty: Type::getInt32Ty(C&: M->getContext())));
11652 SDValue GA = DAG.getTargetGlobalAddress(GV: RelocSymbol, DL, VT: MVT::i32, offset: 0,
11653 TargetFlags: SIInstrInfo::MO_ABS32_LO);
11654 return {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: GA), 0};
11655 }
11656 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
11657 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
11658 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
11659 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
11660 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
11661 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
11662 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
11663 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8: {
11664 if (Op.getOperand(i: 4).getValueType() == MVT::i32)
11665 return SDValue();
11666
11667 SDLoc SL(Op);
11668 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 4), DL: SL, VT: MVT::i32);
11669 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
11670 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1), N3: Op.getOperand(i: 2),
11671 N4: Op.getOperand(i: 3), N5: IndexKeyi32);
11672 }
11673 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
11674 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
11675 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
11676 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
11677 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
11678 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
11679 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
11680 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8: {
11681 if (Op.getOperand(i: 4).getValueType() == MVT::i64)
11682 return SDValue();
11683
11684 SDLoc SL(Op);
11685 auto IndexKeyi64 =
11686 Op.getOperand(i: 4).getValueType() == MVT::v2i32
11687 ? DAG.getBitcast(VT: MVT::i64, V: Op.getOperand(i: 4))
11688 : DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 4), DL: SL, VT: MVT::i64);
11689 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
11690 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
11691 Op.getOperand(i: 3), IndexKeyi64, Op.getOperand(i: 5),
11692 Op.getOperand(i: 6)});
11693 }
11694 case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
11695 case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
11696 case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
11697 case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
11698 case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
11699 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8: {
11700 EVT IndexKeyTy = IntrinsicID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8
11701 ? MVT::i64
11702 : MVT::i32;
11703 if (Op.getOperand(i: 6).getValueType() == IndexKeyTy)
11704 return SDValue();
11705
11706 SDLoc SL(Op);
11707 auto IndexKey =
11708 Op.getOperand(i: 6).getValueType().isVector()
11709 ? DAG.getBitcast(VT: IndexKeyTy, V: Op.getOperand(i: 6))
11710 : DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 6), DL: SL, VT: IndexKeyTy);
11711 SmallVector<SDValue> Args{
11712 Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
11713 Op.getOperand(i: 3), Op.getOperand(i: 4), Op.getOperand(i: 5),
11714 IndexKey, Op.getOperand(i: 7), Op.getOperand(i: 8)};
11715 if (IntrinsicID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8)
11716 Args.push_back(Elt: Op.getOperand(i: 9));
11717 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(), Ops: Args);
11718 }
11719 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
11720 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
11721 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4: {
11722 if (Op.getOperand(i: 6).getValueType() == MVT::i32)
11723 return SDValue();
11724
11725 SDLoc SL(Op);
11726 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 6), DL: SL, VT: MVT::i32);
11727 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
11728 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
11729 Op.getOperand(i: 3), Op.getOperand(i: 4), Op.getOperand(i: 5),
11730 IndexKeyi32, Op.getOperand(i: 7)});
11731 }
11732 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
11733 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
11734 unsigned AFmt = (unsigned)Op.getConstantOperandVal(i: 1);
11735 unsigned BFmt = (unsigned)Op.getConstantOperandVal(i: 3);
11736 unsigned AScaleFmt = (unsigned)Op.getConstantOperandVal(i: 8);
11737 unsigned BScaleFmt = (unsigned)Op.getConstantOperandVal(i: 11);
11738 if (!AMDGPU::isValidWMMAScaleFmtCombination(AFmt, AScale: AScaleFmt, BFmt,
11739 BScale: BScaleFmt)) {
11740 DAG.getMachineFunction().getFunction().getContext().emitError(
11741 ErrorStr: "invalid matrix and scale format combination in wmma call");
11742 Op->print(OS&: errs());
11743 errs() << '\n';
11744 }
11745 return SDValue();
11746 }
11747 case Intrinsic::amdgcn_readlane:
11748 case Intrinsic::amdgcn_readfirstlane:
11749 case Intrinsic::amdgcn_writelane:
11750 case Intrinsic::amdgcn_permlane16:
11751 case Intrinsic::amdgcn_permlanex16:
11752 case Intrinsic::amdgcn_permlane64:
11753 case Intrinsic::amdgcn_set_inactive:
11754 case Intrinsic::amdgcn_set_inactive_chain_arg:
11755 case Intrinsic::amdgcn_mov_dpp8:
11756 case Intrinsic::amdgcn_update_dpp:
11757 case Intrinsic::amdgcn_permlane_bcast:
11758 case Intrinsic::amdgcn_permlane_up:
11759 case Intrinsic::amdgcn_permlane_down:
11760 case Intrinsic::amdgcn_permlane_xor:
11761 return lowerLaneOp(TLI: *this, N: Op.getNode(), DAG);
11762 case Intrinsic::amdgcn_dead: {
11763 SmallVector<SDValue, 8> Poisons;
11764 for (const EVT ValTy : Op.getNode()->values())
11765 Poisons.push_back(Elt: DAG.getPOISON(VT: ValTy));
11766 return DAG.getMergeValues(Ops: Poisons, dl: SDLoc(Op));
11767 }
11768 case Intrinsic::amdgcn_wave_shuffle:
11769 return lowerWaveShuffle(TLI: *this, N: Op.getNode(), DAG);
11770 default:
11771 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
11772 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrinsicID))
11773 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: false);
11774
11775 return Op;
11776 }
11777}
11778
11779// On targets not supporting constant in soffset field, turn zero to
11780// SGPR_NULL to avoid generating an extra s_mov with zero.
11781static SDValue selectSOffset(SDValue SOffset, SelectionDAG &DAG,
11782 const GCNSubtarget *Subtarget) {
11783 if (Subtarget->hasRestrictedSOffset() && isNullConstant(V: SOffset))
11784 return DAG.getRegister(Reg: AMDGPU::SGPR_NULL, VT: MVT::i32);
11785 return SOffset;
11786}
11787
11788SDValue SITargetLowering::lowerRawBufferAtomicIntrin(SDValue Op,
11789 SelectionDAG &DAG,
11790 unsigned NewOpcode) const {
11791 SDLoc DL(Op);
11792
11793 SDValue VData = Op.getOperand(i: 2);
11794 if (VData.getValueSizeInBits() != 32 && VData.getValueSizeInBits() != 64) {
11795 SmallVector<EVT, 2> ResultTypes(Op->values());
11796 return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
11797 Msg: "unsupported buffer atomic data type");
11798 }
11799 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
11800 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
11801 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
11802 SDValue Ops[] = {
11803 Op.getOperand(i: 0), // Chain
11804 VData, // vdata
11805 Rsrc, // rsrc
11806 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
11807 VOffset, // voffset
11808 SOffset, // soffset
11809 Offset, // offset
11810 Op.getOperand(i: 6), // cachepolicy
11811 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
11812 };
11813
11814 auto *M = cast<MemSDNode>(Val&: Op);
11815
11816 EVT MemVT = VData.getValueType();
11817 return DAG.getMemIntrinsicNode(Opcode: NewOpcode, dl: DL, VTList: Op->getVTList(), Ops, MemVT,
11818 MMO: M->getMemOperand());
11819}
11820
11821SDValue
11822SITargetLowering::lowerStructBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG,
11823 unsigned NewOpcode) const {
11824 SDLoc DL(Op);
11825
11826 SDValue VData = Op.getOperand(i: 2);
11827 if (VData.getValueSizeInBits() != 32 && VData.getValueSizeInBits() != 64) {
11828 SmallVector<EVT, 2> ResultTypes(Op->values());
11829 return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
11830 Msg: "unsupported buffer atomic data type");
11831 }
11832 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
11833 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
11834 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
11835 SDValue Ops[] = {
11836 Op.getOperand(i: 0), // Chain
11837 VData, // vdata
11838 Rsrc, // rsrc
11839 Op.getOperand(i: 4), // vindex
11840 VOffset, // voffset
11841 SOffset, // soffset
11842 Offset, // offset
11843 Op.getOperand(i: 7), // cachepolicy
11844 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
11845 };
11846
11847 auto *M = cast<MemSDNode>(Val&: Op);
11848
11849 EVT MemVT = VData.getValueType();
11850 return DAG.getMemIntrinsicNode(Opcode: NewOpcode, dl: DL, VTList: Op->getVTList(), Ops, MemVT,
11851 MMO: M->getMemOperand());
11852}
11853
11854static void initializeM0ToZeroForClusterLoad(SDValue Op, SelectionDAG &DAG,
11855 SDLoc DL) {
11856 SDNode *N = Op.getNode();
11857 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
11858 unsigned NumOperands = N->getNumOperands();
11859 if (N->getOperand(Num: NumOperands - 1) == Zero)
11860 return;
11861 SmallVector<SDValue, 7> Ops(N->ops());
11862 Ops[NumOperands - 1] = Zero; // M0 = 0
11863 DAG.UpdateNodeOperands(N, Ops);
11864}
11865
11866SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
11867 SelectionDAG &DAG) const {
11868 unsigned IntrID = Op.getConstantOperandVal(i: 1);
11869 SDLoc DL(Op);
11870
11871 switch (IntrID) {
11872 case Intrinsic::amdgcn_cluster_load_b32:
11873 case Intrinsic::amdgcn_cluster_load_b64:
11874 case Intrinsic::amdgcn_cluster_load_b128: {
11875 if (Subtarget->hasGFX1250_STRICT())
11876 initializeM0ToZeroForClusterLoad(Op, DAG, DL);
11877 return SDValue();
11878 }
11879 case Intrinsic::amdgcn_ds_ordered_add:
11880 case Intrinsic::amdgcn_ds_ordered_swap: {
11881 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11882 SDValue Chain = M->getOperand(Num: 0);
11883 SDValue M0 = M->getOperand(Num: 2);
11884 SDValue Value = M->getOperand(Num: 3);
11885 unsigned IndexOperand = M->getConstantOperandVal(Num: 7);
11886 unsigned WaveRelease = M->getConstantOperandVal(Num: 8);
11887 unsigned WaveDone = M->getConstantOperandVal(Num: 9);
11888
11889 unsigned OrderedCountIndex = IndexOperand & 0x3f;
11890 IndexOperand &= ~0x3f;
11891 unsigned CountDw = 0;
11892
11893 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10) {
11894 CountDw = (IndexOperand >> 24) & 0xf;
11895 IndexOperand &= ~(0xf << 24);
11896
11897 if (CountDw < 1 || CountDw > 4) {
11898 const Function &Fn = DAG.getMachineFunction().getFunction();
11899 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11900 Fn, "ds_ordered_count: dword count must be between 1 and 4",
11901 DL.getDebugLoc()));
11902 CountDw = 1;
11903 }
11904 }
11905
11906 if (IndexOperand) {
11907 const Function &Fn = DAG.getMachineFunction().getFunction();
11908 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11909 Fn, "ds_ordered_count: bad index operand", DL.getDebugLoc()));
11910 }
11911
11912 if (WaveDone && !WaveRelease) {
11913 // TODO: Move this to IR verifier
11914 const Function &Fn = DAG.getMachineFunction().getFunction();
11915 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11916 Fn, "ds_ordered_count: wave_done requires wave_release",
11917 DL.getDebugLoc()));
11918 }
11919
11920 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
11921 unsigned ShaderType =
11922 SIInstrInfo::getDSShaderTypeValue(MF: DAG.getMachineFunction());
11923 unsigned Offset0 = OrderedCountIndex << 2;
11924 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (Instruction << 4);
11925
11926 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10)
11927 Offset1 |= (CountDw - 1) << 6;
11928
11929 if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX11)
11930 Offset1 |= ShaderType << 2;
11931
11932 unsigned Offset = Offset0 | (Offset1 << 8);
11933
11934 SDValue Ops[] = {
11935 Chain, Value, DAG.getTargetConstant(Val: Offset, DL, VT: MVT::i16),
11936 copyToM0(DAG, Chain, DL, V: M0).getValue(R: 1), // Glue
11937 };
11938 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::DS_ORDERED_COUNT, dl: DL,
11939 VTList: M->getVTList(), Ops, MemVT: M->getMemoryVT(),
11940 MMO: M->getMemOperand());
11941 }
11942 case Intrinsic::amdgcn_ptr_s_buffer_load: {
11943 unsigned CPol = Op.getConstantOperandVal(i: 4);
11944 if (CPol & ~((Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12)
11945 ? AMDGPU::CPol::ALL
11946 : AMDGPU::CPol::ALL_pregfx12))
11947 return Op;
11948
11949 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11950 return lowerSBuffer(
11951 VT: Op.getValueType(), MemVT: M->getMemoryVT(), DL, Chain: Op.getOperand(i: 0),
11952 Rsrc: bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG), Offset: Op.getOperand(i: 3),
11953 CachePolicy: Op.getOperand(i: 4), DAG, MMO: M->getMemOperand());
11954 }
11955 case Intrinsic::amdgcn_raw_buffer_load:
11956 case Intrinsic::amdgcn_raw_ptr_buffer_load:
11957 case Intrinsic::amdgcn_raw_atomic_buffer_load:
11958 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
11959 case Intrinsic::amdgcn_raw_buffer_load_format:
11960 case Intrinsic::amdgcn_raw_ptr_buffer_load_format: {
11961 const bool IsFormat =
11962 IntrID == Intrinsic::amdgcn_raw_buffer_load_format ||
11963 IntrID == Intrinsic::amdgcn_raw_ptr_buffer_load_format;
11964
11965 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
11966 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 3), DAG);
11967 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 4), DAG, Subtarget);
11968 SDValue Ops[] = {
11969 Op.getOperand(i: 0), // Chain
11970 Rsrc, // rsrc
11971 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
11972 VOffset, // voffset
11973 SOffset, // soffset
11974 Offset, // offset
11975 Op.getOperand(i: 5), // cachepolicy, swizzled buffer
11976 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
11977 };
11978
11979 auto *M = cast<MemSDNode>(Val&: Op);
11980 return lowerIntrinsicLoad(M, IsFormat, DAG, Ops);
11981 }
11982 case Intrinsic::amdgcn_struct_buffer_load:
11983 case Intrinsic::amdgcn_struct_ptr_buffer_load:
11984 case Intrinsic::amdgcn_struct_buffer_load_format:
11985 case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
11986 case Intrinsic::amdgcn_struct_atomic_buffer_load:
11987 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
11988 const bool IsFormat =
11989 IntrID == Intrinsic::amdgcn_struct_buffer_load_format ||
11990 IntrID == Intrinsic::amdgcn_struct_ptr_buffer_load_format;
11991
11992 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
11993 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
11994 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
11995 SDValue Ops[] = {
11996 Op.getOperand(i: 0), // Chain
11997 Rsrc, // rsrc
11998 Op.getOperand(i: 3), // vindex
11999 VOffset, // voffset
12000 SOffset, // soffset
12001 Offset, // offset
12002 Op.getOperand(i: 6), // cachepolicy, swizzled buffer
12003 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
12004 };
12005
12006 return lowerIntrinsicLoad(M: cast<MemSDNode>(Val&: Op), IsFormat, DAG, Ops);
12007 }
12008 case Intrinsic::amdgcn_raw_tbuffer_load:
12009 case Intrinsic::amdgcn_raw_ptr_tbuffer_load: {
12010 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12011 EVT LoadVT = Op.getValueType();
12012 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
12013 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 3), DAG);
12014 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 4), DAG, Subtarget);
12015
12016 SDValue Ops[] = {
12017 Op.getOperand(i: 0), // Chain
12018 Rsrc, // rsrc
12019 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
12020 VOffset, // voffset
12021 SOffset, // soffset
12022 Offset, // offset
12023 Op.getOperand(i: 5), // format
12024 Op.getOperand(i: 6), // cachepolicy, swizzled buffer
12025 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
12026 };
12027
12028 if (LoadVT.getScalarSizeInBits() == 16)
12029 return adjustLoadValueType(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
12030 Ops);
12031 return getMemIntrinsicNode(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
12032 VTList: Op->getVTList(), Ops, MemVT: LoadVT, MMO: M->getMemOperand(),
12033 DAG);
12034 }
12035 case Intrinsic::amdgcn_struct_tbuffer_load:
12036 case Intrinsic::amdgcn_struct_ptr_tbuffer_load: {
12037 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12038 EVT LoadVT = Op.getValueType();
12039 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
12040 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
12041 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
12042
12043 SDValue Ops[] = {
12044 Op.getOperand(i: 0), // Chain
12045 Rsrc, // rsrc
12046 Op.getOperand(i: 3), // vindex
12047 VOffset, // voffset
12048 SOffset, // soffset
12049 Offset, // offset
12050 Op.getOperand(i: 6), // format
12051 Op.getOperand(i: 7), // cachepolicy, swizzled buffer
12052 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
12053 };
12054
12055 if (LoadVT.getScalarSizeInBits() == 16)
12056 return adjustLoadValueType(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
12057 Ops);
12058 return getMemIntrinsicNode(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
12059 VTList: Op->getVTList(), Ops, MemVT: LoadVT, MMO: M->getMemOperand(),
12060 DAG);
12061 }
12062 case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
12063 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
12064 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FADD);
12065 case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
12066 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
12067 return lowerStructBufferAtomicIntrin(Op, DAG,
12068 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FADD);
12069 case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
12070 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
12071 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMIN);
12072 case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
12073 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
12074 return lowerStructBufferAtomicIntrin(Op, DAG,
12075 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMIN);
12076 case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
12077 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
12078 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMAX);
12079 case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
12080 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
12081 return lowerStructBufferAtomicIntrin(Op, DAG,
12082 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMAX);
12083 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
12084 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
12085 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SWAP);
12086 case Intrinsic::amdgcn_raw_buffer_atomic_add:
12087 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
12088 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_ADD);
12089 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
12090 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
12091 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SUB);
12092 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
12093 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
12094 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMIN);
12095 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
12096 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
12097 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMIN);
12098 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
12099 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
12100 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMAX);
12101 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
12102 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
12103 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMAX);
12104 case Intrinsic::amdgcn_raw_buffer_atomic_and:
12105 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
12106 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_AND);
12107 case Intrinsic::amdgcn_raw_buffer_atomic_or:
12108 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
12109 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_OR);
12110 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
12111 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
12112 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_XOR);
12113 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
12114 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
12115 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_INC);
12116 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
12117 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
12118 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_DEC);
12119 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
12120 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
12121 return lowerStructBufferAtomicIntrin(Op, DAG,
12122 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SWAP);
12123 case Intrinsic::amdgcn_struct_buffer_atomic_add:
12124 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
12125 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_ADD);
12126 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
12127 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
12128 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SUB);
12129 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
12130 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
12131 return lowerStructBufferAtomicIntrin(Op, DAG,
12132 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMIN);
12133 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
12134 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
12135 return lowerStructBufferAtomicIntrin(Op, DAG,
12136 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMIN);
12137 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
12138 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
12139 return lowerStructBufferAtomicIntrin(Op, DAG,
12140 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMAX);
12141 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
12142 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
12143 return lowerStructBufferAtomicIntrin(Op, DAG,
12144 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMAX);
12145 case Intrinsic::amdgcn_struct_buffer_atomic_and:
12146 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
12147 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_AND);
12148 case Intrinsic::amdgcn_struct_buffer_atomic_or:
12149 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
12150 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_OR);
12151 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
12152 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
12153 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_XOR);
12154 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
12155 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
12156 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_INC);
12157 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
12158 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
12159 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_DEC);
12160 case Intrinsic::amdgcn_raw_buffer_atomic_sub_clamp_u32:
12161 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
12162 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_CSUB);
12163 case Intrinsic::amdgcn_struct_buffer_atomic_sub_clamp_u32:
12164 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
12165 return lowerStructBufferAtomicIntrin(Op, DAG,
12166 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_CSUB);
12167 case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
12168 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
12169 return lowerRawBufferAtomicIntrin(Op, DAG,
12170 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_COND_SUB_U32);
12171 case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
12172 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
12173 return lowerStructBufferAtomicIntrin(Op, DAG,
12174 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_COND_SUB_U32);
12175 case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
12176 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap: {
12177 SDValue Src = Op.getOperand(i: 2);
12178 if (Src.getValueSizeInBits() != 32 && Src.getValueSizeInBits() != 64) {
12179 SmallVector<EVT, 2> ResultTypes(Op->values());
12180 return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
12181 Msg: "unsupported buffer atomic data type");
12182 }
12183 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 4), DAG);
12184 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
12185 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
12186 SDValue Ops[] = {
12187 Op.getOperand(i: 0), // Chain
12188 Op.getOperand(i: 2), // src
12189 Op.getOperand(i: 3), // cmp
12190 Rsrc, // rsrc
12191 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
12192 VOffset, // voffset
12193 SOffset, // soffset
12194 Offset, // offset
12195 Op.getOperand(i: 7), // cachepolicy
12196 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
12197 };
12198 EVT VT = Op.getValueType();
12199 auto *M = cast<MemSDNode>(Val&: Op);
12200
12201 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, dl: DL,
12202 VTList: Op->getVTList(), Ops, MemVT: VT,
12203 MMO: M->getMemOperand());
12204 }
12205 case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
12206 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap: {
12207 SDValue Src = Op.getOperand(i: 2);
12208 if (Src.getValueSizeInBits() != 32 && Src.getValueSizeInBits() != 64) {
12209 SmallVector<EVT, 2> ResultTypes(Op->values());
12210 return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
12211 Msg: "unsupported buffer atomic data type");
12212 }
12213 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op->getOperand(Num: 4), DAG);
12214 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 6), DAG);
12215 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 7), DAG, Subtarget);
12216 SDValue Ops[] = {
12217 Op.getOperand(i: 0), // Chain
12218 Op.getOperand(i: 2), // src
12219 Op.getOperand(i: 3), // cmp
12220 Rsrc, // rsrc
12221 Op.getOperand(i: 5), // vindex
12222 VOffset, // voffset
12223 SOffset, // soffset
12224 Offset, // offset
12225 Op.getOperand(i: 8), // cachepolicy
12226 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
12227 };
12228 EVT VT = Op.getValueType();
12229 auto *M = cast<MemSDNode>(Val&: Op);
12230
12231 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, dl: DL,
12232 VTList: Op->getVTList(), Ops, MemVT: VT,
12233 MMO: M->getMemOperand());
12234 }
12235 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
12236 case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
12237 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12238 SDValue NodePtr = M->getOperand(Num: 2);
12239 SDValue RayExtent = M->getOperand(Num: 3);
12240 SDValue InstanceMask = M->getOperand(Num: 4);
12241 SDValue RayOrigin = M->getOperand(Num: 5);
12242 SDValue RayDir = M->getOperand(Num: 6);
12243 SDValue Offsets = M->getOperand(Num: 7);
12244 SDValue TDescr = M->getOperand(Num: 8);
12245
12246 assert(NodePtr.getValueType() == MVT::i64);
12247 assert(RayDir.getValueType() == MVT::v3f32);
12248
12249 bool IsBVH8 = IntrID == Intrinsic::amdgcn_image_bvh8_intersect_ray;
12250 const unsigned NumVDataDwords = 10;
12251 const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12;
12252 int Opcode = AMDGPU::getMIMGOpcode(
12253 BaseOpcode: IsBVH8 ? AMDGPU::IMAGE_BVH8_INTERSECT_RAY
12254 : AMDGPU::IMAGE_BVH_DUAL_INTERSECT_RAY,
12255 MIMGEncoding: AMDGPU::MIMGEncGfx12, VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
12256 assert(Opcode != -1);
12257
12258 SmallVector<SDValue, 7> Ops;
12259 Ops.push_back(Elt: NodePtr);
12260 Ops.push_back(Elt: DAG.getBuildVector(
12261 VT: MVT::v2i32, DL,
12262 Ops: {DAG.getBitcast(VT: MVT::i32, V: RayExtent),
12263 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: InstanceMask)}));
12264 Ops.push_back(Elt: RayOrigin);
12265 Ops.push_back(Elt: RayDir);
12266 Ops.push_back(Elt: Offsets);
12267 Ops.push_back(Elt: TDescr);
12268 Ops.push_back(Elt: M->getChain());
12269
12270 auto *NewNode = DAG.getMachineNode(Opcode, dl: DL, VTs: M->getVTList(), Ops);
12271 MachineMemOperand *MemRef = M->getMemOperand();
12272 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
12273 return SDValue(NewNode, 0);
12274 }
12275 case Intrinsic::amdgcn_image_bvh_intersect_ray: {
12276 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12277 SDValue NodePtr = M->getOperand(Num: 2);
12278 SDValue RayExtent = M->getOperand(Num: 3);
12279 SDValue RayOrigin = M->getOperand(Num: 4);
12280 SDValue RayDir = M->getOperand(Num: 5);
12281 SDValue RayInvDir = M->getOperand(Num: 6);
12282 SDValue TDescr = M->getOperand(Num: 7);
12283
12284 assert(NodePtr.getValueType() == MVT::i32 ||
12285 NodePtr.getValueType() == MVT::i64);
12286 assert(RayDir.getValueType() == MVT::v3f16 ||
12287 RayDir.getValueType() == MVT::v3f32);
12288
12289 const bool IsGFX11 = AMDGPU::isGFX11(STI: *Subtarget);
12290 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI: *Subtarget);
12291 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
12292 const bool IsA16 = RayDir.getValueType().getVectorElementType() == MVT::f16;
12293 const bool Is64 = NodePtr.getValueType() == MVT::i64;
12294 const unsigned NumVDataDwords = 4;
12295 const unsigned NumVAddrDwords = IsA16 ? (Is64 ? 9 : 8) : (Is64 ? 12 : 11);
12296 const unsigned NumVAddrs = IsGFX11Plus ? (IsA16 ? 4 : 5) : NumVAddrDwords;
12297 const bool UseNSA = (Subtarget->hasNSAEncoding() &&
12298 NumVAddrs <= Subtarget->getNSAMaxSize()) ||
12299 IsGFX12Plus;
12300 const unsigned BaseOpcodes[2][2] = {
12301 {AMDGPU::IMAGE_BVH_INTERSECT_RAY, AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16},
12302 {AMDGPU::IMAGE_BVH64_INTERSECT_RAY,
12303 AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16}};
12304 int Opcode;
12305 if (UseNSA) {
12306 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: BaseOpcodes[Is64][IsA16],
12307 MIMGEncoding: IsGFX12Plus ? AMDGPU::MIMGEncGfx12
12308 : IsGFX11 ? AMDGPU::MIMGEncGfx11NSA
12309 : AMDGPU::MIMGEncGfx10NSA,
12310 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
12311 } else {
12312 assert(!IsGFX12Plus);
12313 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: BaseOpcodes[Is64][IsA16],
12314 MIMGEncoding: IsGFX11 ? AMDGPU::MIMGEncGfx11Default
12315 : AMDGPU::MIMGEncGfx10Default,
12316 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
12317 }
12318 assert(Opcode != -1);
12319
12320 SmallVector<SDValue, 16> Ops;
12321
12322 auto packLanes = [&DAG, &Ops, &DL](SDValue Op, bool IsAligned) {
12323 SmallVector<SDValue, 3> Lanes;
12324 DAG.ExtractVectorElements(Op, Args&: Lanes, Start: 0, Count: 3);
12325 if (Lanes[0].getValueSizeInBits() == 32) {
12326 for (unsigned I = 0; I < 3; ++I)
12327 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: Lanes[I]));
12328 } else {
12329 if (IsAligned) {
12330 Ops.push_back(Elt: DAG.getBitcast(
12331 VT: MVT::i32,
12332 V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Lanes[0], Lanes[1]})));
12333 Ops.push_back(Elt: Lanes[2]);
12334 } else {
12335 SDValue Elt0 = Ops.pop_back_val();
12336 Ops.push_back(Elt: DAG.getBitcast(
12337 VT: MVT::i32, V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Elt0, Lanes[0]})));
12338 Ops.push_back(Elt: DAG.getBitcast(
12339 VT: MVT::i32,
12340 V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Lanes[1], Lanes[2]})));
12341 }
12342 }
12343 };
12344
12345 if (UseNSA && IsGFX11Plus) {
12346 Ops.push_back(Elt: NodePtr);
12347 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: RayExtent));
12348 Ops.push_back(Elt: RayOrigin);
12349 if (IsA16) {
12350 SmallVector<SDValue, 3> DirLanes, InvDirLanes, MergedLanes;
12351 DAG.ExtractVectorElements(Op: RayDir, Args&: DirLanes, Start: 0, Count: 3);
12352 DAG.ExtractVectorElements(Op: RayInvDir, Args&: InvDirLanes, Start: 0, Count: 3);
12353 for (unsigned I = 0; I < 3; ++I) {
12354 MergedLanes.push_back(Elt: DAG.getBitcast(
12355 VT: MVT::i32, V: DAG.getBuildVector(VT: MVT::v2f16, DL,
12356 Ops: {DirLanes[I], InvDirLanes[I]})));
12357 }
12358 Ops.push_back(Elt: DAG.getBuildVector(VT: MVT::v3i32, DL, Ops: MergedLanes));
12359 } else {
12360 Ops.push_back(Elt: RayDir);
12361 Ops.push_back(Elt: RayInvDir);
12362 }
12363 } else {
12364 if (Is64)
12365 DAG.ExtractVectorElements(Op: DAG.getBitcast(VT: MVT::v2i32, V: NodePtr), Args&: Ops, Start: 0,
12366 Count: 2);
12367 else
12368 Ops.push_back(Elt: NodePtr);
12369
12370 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: RayExtent));
12371 packLanes(RayOrigin, true);
12372 packLanes(RayDir, true);
12373 packLanes(RayInvDir, false);
12374 }
12375
12376 if (!UseNSA) {
12377 // Build a single vector containing all the operands so far prepared.
12378 if (NumVAddrDwords > 12) {
12379 SDValue Undef = DAG.getPOISON(VT: MVT::i32);
12380 Ops.append(NumInputs: 16 - Ops.size(), Elt: Undef);
12381 }
12382 assert(Ops.size() >= 8 && Ops.size() <= 12);
12383 SDValue MergedOps =
12384 DAG.getBuildVector(VT: MVT::getVectorVT(VT: MVT::i32, NumElements: Ops.size()), DL, Ops);
12385 Ops.clear();
12386 Ops.push_back(Elt: MergedOps);
12387 }
12388
12389 Ops.push_back(Elt: TDescr);
12390 Ops.push_back(Elt: DAG.getTargetConstant(Val: IsA16, DL, VT: MVT::i1));
12391 Ops.push_back(Elt: M->getChain());
12392
12393 auto *NewNode = DAG.getMachineNode(Opcode, dl: DL, VTs: M->getVTList(), Ops);
12394 MachineMemOperand *MemRef = M->getMemOperand();
12395 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
12396 return SDValue(NewNode, 0);
12397 }
12398 case Intrinsic::amdgcn_global_atomic_fmin_num:
12399 case Intrinsic::amdgcn_global_atomic_fmax_num:
12400 case Intrinsic::amdgcn_flat_atomic_fmin_num:
12401 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
12402 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12403 SDValue Ops[] = {
12404 M->getOperand(Num: 0), // Chain
12405 M->getOperand(Num: 2), // Ptr
12406 M->getOperand(Num: 3) // Value
12407 };
12408 unsigned Opcode = 0;
12409 switch (IntrID) {
12410 case Intrinsic::amdgcn_global_atomic_fmin_num:
12411 case Intrinsic::amdgcn_flat_atomic_fmin_num: {
12412 Opcode = ISD::ATOMIC_LOAD_FMIN;
12413 break;
12414 }
12415 case Intrinsic::amdgcn_global_atomic_fmax_num:
12416 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
12417 Opcode = ISD::ATOMIC_LOAD_FMAX;
12418 break;
12419 }
12420 default:
12421 llvm_unreachable("unhandled atomic opcode");
12422 }
12423 return DAG.getAtomic(Opcode, dl: SDLoc(Op), MemVT: M->getMemoryVT(), VTList: M->getVTList(),
12424 Ops, MMO: M->getMemOperand());
12425 }
12426 case Intrinsic::amdgcn_s_alloc_vgpr: {
12427 SDValue NumVGPRs = Op.getOperand(i: 2);
12428 if (!NumVGPRs->isDivergent())
12429 return Op;
12430
12431 SDValue ReadFirstLaneID =
12432 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
12433 NumVGPRs = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
12434 N1: ReadFirstLaneID, N2: NumVGPRs);
12435
12436 return DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL, VTList: Op->getVTList(),
12437 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1), N3: NumVGPRs);
12438 }
12439 case Intrinsic::amdgcn_s_get_barrier_state:
12440 case Intrinsic::amdgcn_s_get_named_barrier_state: {
12441 SDValue Chain = Op->getOperand(Num: 0);
12442 SmallVector<SDValue, 2> Ops;
12443 unsigned Opc;
12444
12445 if (isa<ConstantSDNode>(Val: Op->getOperand(Num: 2))) {
12446 uint64_t BarID = cast<ConstantSDNode>(Val: Op->getOperand(Num: 2))->getZExtValue();
12447 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state)
12448 BarID = BarID & 0x3F;
12449 Opc = AMDGPU::S_GET_BARRIER_STATE_IMM;
12450 SDValue K = DAG.getTargetConstant(Val: BarID, DL, VT: MVT::i32);
12451 Ops.push_back(Elt: K);
12452 Ops.push_back(Elt: Chain);
12453 } else {
12454 Opc = AMDGPU::S_GET_BARRIER_STATE_M0;
12455 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
12456 SDValue M0Val = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Op->getOperand(Num: 2),
12457 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
12458 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
12459 } else
12460 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: Op->getOperand(Num: 2)).getValue(R: 0));
12461 }
12462
12463 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
12464 return SDValue(NewMI, 0);
12465 }
12466 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
12467 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
12468 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
12469 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12470 SDValue Chain = Op->getOperand(Num: 0);
12471 SDValue Ptr = Op->getOperand(Num: 2);
12472 EVT VT = Op->getValueType(ResNo: 0);
12473 return DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl: DL, MemVT: MII->getMemoryVT(), VT,
12474 Chain, Ptr, MMO: MII->getMemOperand());
12475 }
12476 case Intrinsic::amdgcn_av_load_b128: {
12477 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12478 SDValue Chain = Op->getOperand(Num: 0);
12479 SDValue Ptr = Op->getOperand(Num: 2);
12480 EVT VT = Op->getValueType(ResNo: 0);
12481 // Lower to a regular ISD::LOAD. The MachineMemOperand carries Monotonic
12482 // ordering and syncscope so that SIMemoryLegalizer sets cache policy bits.
12483 // Address space filtering in the load_global/load_flat PatFrags selects
12484 // the correct GLOBAL vs FLAT instruction.
12485 return DAG.getLoad(VT, dl: DL, Chain, Ptr, MMO: MII->getMemOperand());
12486 }
12487 case Intrinsic::amdgcn_flat_load_monitor_b32:
12488 case Intrinsic::amdgcn_flat_load_monitor_b64:
12489 case Intrinsic::amdgcn_flat_load_monitor_b128: {
12490 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12491 SDValue Chain = Op->getOperand(Num: 0);
12492 SDValue Ptr = Op->getOperand(Num: 2);
12493 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::FLAT_LOAD_MONITOR, dl: DL,
12494 VTList: Op->getVTList(), Ops: {Chain, Ptr},
12495 MemVT: MII->getMemoryVT(), MMO: MII->getMemOperand());
12496 }
12497 case Intrinsic::amdgcn_global_load_monitor_b32:
12498 case Intrinsic::amdgcn_global_load_monitor_b64:
12499 case Intrinsic::amdgcn_global_load_monitor_b128: {
12500 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12501 SDValue Chain = Op->getOperand(Num: 0);
12502 SDValue Ptr = Op->getOperand(Num: 2);
12503 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::GLOBAL_LOAD_MONITOR, dl: DL,
12504 VTList: Op->getVTList(), Ops: {Chain, Ptr},
12505 MemVT: MII->getMemoryVT(), MMO: MII->getMemOperand());
12506 }
12507 default:
12508
12509 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
12510 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrID))
12511 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: true);
12512
12513 return SDValue();
12514 }
12515}
12516
12517// Call DAG.getMemIntrinsicNode for a load, but first widen a dwordx3 type to
12518// dwordx4 if on SI and handle TFE loads.
12519SDValue SITargetLowering::getMemIntrinsicNode(unsigned Opcode, const SDLoc &DL,
12520 SDVTList VTList,
12521 ArrayRef<SDValue> Ops, EVT MemVT,
12522 MachineMemOperand *MMO,
12523 SelectionDAG &DAG) const {
12524 LLVMContext &C = *DAG.getContext();
12525 MachineFunction &MF = DAG.getMachineFunction();
12526 EVT VT = VTList.VTs[0];
12527
12528 assert(VTList.NumVTs == 2 || VTList.NumVTs == 3);
12529 bool IsTFE = VTList.NumVTs == 3;
12530 if (IsTFE) {
12531 unsigned NumValueDWords = divideCeil(Numerator: VT.getSizeInBits(), Denominator: 32);
12532 unsigned NumOpDWords = NumValueDWords + 1;
12533 EVT OpDWordsVT = EVT::getVectorVT(Context&: C, VT: MVT::i32, NumElements: NumOpDWords);
12534 SDVTList OpDWordsVTList = DAG.getVTList(VT1: OpDWordsVT, VT2: VTList.VTs[2]);
12535 MachineMemOperand *OpDWordsMMO =
12536 MF.getMachineMemOperand(MMO, Offset: 0, Size: NumOpDWords * 4);
12537 SDValue Op = getMemIntrinsicNode(Opcode, DL, VTList: OpDWordsVTList, Ops,
12538 MemVT: OpDWordsVT, MMO: OpDWordsMMO, DAG);
12539 auto [Value, Status] = splitTFEValueAndStatus(Op, VT, DL, DAG);
12540 return DAG.getMergeValues(Ops: {Value, Status, SDValue(Op.getNode(), 1)}, dl: DL);
12541 }
12542
12543 if (!Subtarget->hasDwordx3LoadStores() &&
12544 (VT == MVT::v3i32 || VT == MVT::v3f32)) {
12545 EVT WidenedVT = EVT::getVectorVT(Context&: C, VT: VT.getVectorElementType(), NumElements: 4);
12546 EVT WidenedMemVT = EVT::getVectorVT(Context&: C, VT: MemVT.getVectorElementType(), NumElements: 4);
12547 MachineMemOperand *WidenedMMO = MF.getMachineMemOperand(MMO, Offset: 0, Size: 16);
12548 SDVTList WidenedVTList = DAG.getVTList(VT1: WidenedVT, VT2: VTList.VTs[1]);
12549 SDValue Op = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: WidenedVTList, Ops,
12550 MemVT: WidenedMemVT, MMO: WidenedMMO);
12551 SDValue Value = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Op,
12552 N2: DAG.getVectorIdxConstant(Val: 0, DL));
12553 return DAG.getMergeValues(Ops: {Value, SDValue(Op.getNode(), 1)}, dl: DL);
12554 }
12555
12556 return DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList, Ops, MemVT, MMO);
12557}
12558
12559SDValue SITargetLowering::handleD16VData(SDValue VData, SelectionDAG &DAG,
12560 bool ImageStore) const {
12561 EVT StoreVT = VData.getValueType();
12562
12563 // No change for f16 and legal vector D16 types.
12564 if (!StoreVT.isVector())
12565 return VData;
12566
12567 SDLoc DL(VData);
12568 unsigned NumElements = StoreVT.getVectorNumElements();
12569
12570 if (Subtarget->hasUnpackedD16VMem()) {
12571 // We need to unpack the packed data to store.
12572 EVT IntStoreVT = StoreVT.changeTypeToInteger();
12573 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
12574
12575 EVT EquivStoreVT =
12576 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements);
12577 SDValue ZExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EquivStoreVT, Operand: IntVData);
12578 return DAG.UnrollVectorOp(N: ZExt.getNode());
12579 }
12580
12581 // The sq block of gfx8.1 does not estimate register use correctly for d16
12582 // image store instructions. The data operand is computed as if it were not a
12583 // d16 image instruction.
12584 if (ImageStore && Subtarget->hasImageStoreD16Bug()) {
12585 // Bitcast to i16
12586 EVT IntStoreVT = StoreVT.changeTypeToInteger();
12587 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
12588
12589 // Decompose into scalars
12590 SmallVector<SDValue, 4> Elts;
12591 DAG.ExtractVectorElements(Op: IntVData, Args&: Elts);
12592
12593 // Group pairs of i16 into v2i16 and bitcast to i32
12594 SmallVector<SDValue, 4> PackedElts;
12595 for (unsigned I = 0; I < Elts.size() / 2; I += 1) {
12596 SDValue Pair =
12597 DAG.getBuildVector(VT: MVT::v2i16, DL, Ops: {Elts[I * 2], Elts[I * 2 + 1]});
12598 SDValue IntPair = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Pair);
12599 PackedElts.push_back(Elt: IntPair);
12600 }
12601 if ((NumElements % 2) == 1) {
12602 // Handle v3i16
12603 unsigned I = Elts.size() / 2;
12604 SDValue Pair = DAG.getBuildVector(VT: MVT::v2i16, DL,
12605 Ops: {Elts[I * 2], DAG.getPOISON(VT: MVT::i16)});
12606 SDValue IntPair = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Pair);
12607 PackedElts.push_back(Elt: IntPair);
12608 }
12609
12610 // Pad using UNDEF
12611 PackedElts.resize(N: Elts.size(), NV: DAG.getPOISON(VT: MVT::i32));
12612
12613 // Build final vector
12614 EVT VecVT =
12615 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: PackedElts.size());
12616 return DAG.getBuildVector(VT: VecVT, DL, Ops: PackedElts);
12617 }
12618
12619 if (NumElements == 3) {
12620 EVT IntStoreVT =
12621 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StoreVT.getStoreSizeInBits());
12622 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
12623
12624 EVT WidenedStoreVT = EVT::getVectorVT(
12625 Context&: *DAG.getContext(), VT: StoreVT.getVectorElementType(), NumElements: NumElements + 1);
12626 EVT WidenedIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
12627 BitWidth: WidenedStoreVT.getStoreSizeInBits());
12628 SDValue ZExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenedIntVT, Operand: IntVData);
12629 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WidenedStoreVT, Operand: ZExt);
12630 }
12631
12632 assert(isTypeLegal(StoreVT));
12633 return VData;
12634}
12635
12636static bool isAsyncLDSDMA(Intrinsic::ID Intr) {
12637 switch (Intr) {
12638 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
12639 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
12640 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
12641 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds:
12642 case Intrinsic::amdgcn_load_async_to_lds:
12643 case Intrinsic::amdgcn_global_load_async_lds:
12644 return true;
12645 }
12646 return false;
12647}
12648
12649SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
12650 SelectionDAG &DAG) const {
12651 SDLoc DL(Op);
12652 SDValue Chain = Op.getOperand(i: 0);
12653 unsigned IntrinsicID = Op.getConstantOperandVal(i: 1);
12654
12655 switch (IntrinsicID) {
12656 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
12657 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
12658 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
12659 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
12660 if (Subtarget->hasGFX1250_STRICT())
12661 initializeM0ToZeroForClusterLoad(Op, DAG, DL);
12662 return SDValue();
12663 }
12664 case Intrinsic::amdgcn_exp_compr: {
12665 SDValue Src0 = Op.getOperand(i: 4);
12666 SDValue Src1 = Op.getOperand(i: 5);
12667 // Hack around illegal type on SI by directly selecting it.
12668 if (isTypeLegal(VT: Src0.getValueType()))
12669 return SDValue();
12670
12671 const ConstantSDNode *Done = cast<ConstantSDNode>(Val: Op.getOperand(i: 6));
12672 SDValue Undef = DAG.getPOISON(VT: MVT::f32);
12673 const SDValue Ops[] = {
12674 Op.getOperand(i: 2), // tgt
12675 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: Src0), // src0
12676 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: Src1), // src1
12677 Undef, // src2
12678 Undef, // src3
12679 Op.getOperand(i: 7), // vm
12680 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // compr
12681 Op.getOperand(i: 3), // en
12682 Op.getOperand(i: 0) // Chain
12683 };
12684
12685 unsigned Opc = Done->isZero() ? AMDGPU::EXP : AMDGPU::EXP_DONE;
12686 return SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops), 0);
12687 }
12688
12689 case Intrinsic::amdgcn_struct_tbuffer_store:
12690 case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
12691 SDValue VData = Op.getOperand(i: 2);
12692 bool IsD16 = (VData.getValueType().getScalarSizeInBits() == 16);
12693 if (IsD16)
12694 VData = handleD16VData(VData, DAG);
12695 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
12696 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
12697 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
12698 SDValue Ops[] = {
12699 Chain,
12700 VData, // vdata
12701 Rsrc, // rsrc
12702 Op.getOperand(i: 4), // vindex
12703 VOffset, // voffset
12704 SOffset, // soffset
12705 Offset, // offset
12706 Op.getOperand(i: 7), // format
12707 Op.getOperand(i: 8), // cachepolicy, swizzled buffer
12708 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
12709 };
12710 unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16
12711 : AMDGPUISD::TBUFFER_STORE_FORMAT;
12712 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12713 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12714 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12715 }
12716
12717 case Intrinsic::amdgcn_raw_tbuffer_store:
12718 case Intrinsic::amdgcn_raw_ptr_tbuffer_store: {
12719 SDValue VData = Op.getOperand(i: 2);
12720 bool IsD16 = (VData.getValueType().getScalarSizeInBits() == 16);
12721 if (IsD16)
12722 VData = handleD16VData(VData, DAG);
12723 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
12724 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
12725 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
12726 SDValue Ops[] = {
12727 Chain,
12728 VData, // vdata
12729 Rsrc, // rsrc
12730 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
12731 VOffset, // voffset
12732 SOffset, // soffset
12733 Offset, // offset
12734 Op.getOperand(i: 6), // format
12735 Op.getOperand(i: 7), // cachepolicy, swizzled buffer
12736 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
12737 };
12738 unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16
12739 : AMDGPUISD::TBUFFER_STORE_FORMAT;
12740 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12741 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12742 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12743 }
12744
12745 case Intrinsic::amdgcn_raw_buffer_store:
12746 case Intrinsic::amdgcn_raw_ptr_buffer_store:
12747 case Intrinsic::amdgcn_raw_buffer_store_format:
12748 case Intrinsic::amdgcn_raw_ptr_buffer_store_format: {
12749 const bool IsFormat =
12750 IntrinsicID == Intrinsic::amdgcn_raw_buffer_store_format ||
12751 IntrinsicID == Intrinsic::amdgcn_raw_ptr_buffer_store_format;
12752
12753 SDValue VData = Op.getOperand(i: 2);
12754 EVT VDataVT = VData.getValueType();
12755 EVT EltType = VDataVT.getScalarType();
12756 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
12757
12758 if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
12759 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
12760 DAG.getMachineFunction().getFunction(),
12761 "unsupported sub-dword format buffer store", DL.getDebugLoc()));
12762 return Chain;
12763 }
12764
12765 if (IsD16) {
12766 VData = handleD16VData(VData, DAG);
12767 VDataVT = VData.getValueType();
12768 }
12769
12770 if (!isTypeLegal(VT: VDataVT)) {
12771 VData =
12772 DAG.getNode(Opcode: ISD::BITCAST, DL,
12773 VT: getEquivalentMemType(Context&: *DAG.getContext(), VT: VDataVT), Operand: VData);
12774 }
12775
12776 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
12777 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
12778 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
12779 SDValue Ops[] = {
12780 Chain,
12781 VData,
12782 Rsrc,
12783 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
12784 VOffset, // voffset
12785 SOffset, // soffset
12786 Offset, // offset
12787 Op.getOperand(i: 6), // cachepolicy, swizzled buffer
12788 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
12789 };
12790 unsigned Opc =
12791 IsFormat ? AMDGPUISD::BUFFER_STORE_FORMAT : AMDGPUISD::BUFFER_STORE;
12792 Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 : Opc;
12793 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12794
12795 // Handle BUFFER_STORE_BYTE/SHORT overloaded intrinsics
12796 if (!IsD16 && !VDataVT.isVector() && EltType.getSizeInBits() < 32)
12797 return handleByteShortBufferStores(DAG, VDataType: VDataVT, DL, Ops, M);
12798
12799 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12800 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12801 }
12802
12803 case Intrinsic::amdgcn_struct_buffer_store:
12804 case Intrinsic::amdgcn_struct_ptr_buffer_store:
12805 case Intrinsic::amdgcn_struct_buffer_store_format:
12806 case Intrinsic::amdgcn_struct_ptr_buffer_store_format: {
12807 const bool IsFormat =
12808 IntrinsicID == Intrinsic::amdgcn_struct_buffer_store_format ||
12809 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_store_format;
12810
12811 SDValue VData = Op.getOperand(i: 2);
12812 EVT VDataVT = VData.getValueType();
12813 EVT EltType = VDataVT.getScalarType();
12814 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
12815
12816 if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
12817 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
12818 DAG.getMachineFunction().getFunction(),
12819 "unsupported sub-dword format buffer store", DL.getDebugLoc()));
12820 return Chain;
12821 }
12822
12823 if (IsD16) {
12824 VData = handleD16VData(VData, DAG);
12825 VDataVT = VData.getValueType();
12826 }
12827
12828 if (!isTypeLegal(VT: VDataVT)) {
12829 VData =
12830 DAG.getNode(Opcode: ISD::BITCAST, DL,
12831 VT: getEquivalentMemType(Context&: *DAG.getContext(), VT: VDataVT), Operand: VData);
12832 }
12833
12834 auto Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
12835 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
12836 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
12837 SDValue Ops[] = {
12838 Chain,
12839 VData,
12840 Rsrc,
12841 Op.getOperand(i: 4), // vindex
12842 VOffset, // voffset
12843 SOffset, // soffset
12844 Offset, // offset
12845 Op.getOperand(i: 7), // cachepolicy, swizzled buffer
12846 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
12847 };
12848 unsigned Opc =
12849 !IsFormat ? AMDGPUISD::BUFFER_STORE : AMDGPUISD::BUFFER_STORE_FORMAT;
12850 Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 : Opc;
12851 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12852
12853 // Handle BUFFER_STORE_BYTE/SHORT overloaded intrinsics
12854 EVT VDataType = VData.getValueType().getScalarType();
12855 if (!IsD16 && !VDataVT.isVector() && EltType.getSizeInBits() < 32)
12856 return handleByteShortBufferStores(DAG, VDataType, DL, Ops, M);
12857
12858 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12859 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12860 }
12861 case Intrinsic::amdgcn_raw_buffer_load_lds:
12862 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
12863 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
12864 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
12865 case Intrinsic::amdgcn_struct_buffer_load_lds:
12866 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
12867 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
12868 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
12869 unsigned Opc;
12870 bool HasVIndex =
12871 IntrinsicID == Intrinsic::amdgcn_struct_buffer_load_lds ||
12872 IntrinsicID == Intrinsic::amdgcn_struct_buffer_load_async_lds ||
12873 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_load_lds ||
12874 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds;
12875 unsigned OpOffset = HasVIndex ? 1 : 0;
12876 SDValue VOffset = Op.getOperand(i: 5 + OpOffset);
12877 bool HasVOffset = !isNullConstant(V: VOffset);
12878 unsigned Size = Op->getConstantOperandVal(Num: 4);
12879
12880 switch (Size) {
12881 default:
12882 return SDValue();
12883 case 1:
12884 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
12885 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
12886 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
12887 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
12888 break;
12889 case 2:
12890 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
12891 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
12892 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
12893 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
12894 break;
12895 case 4:
12896 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
12897 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
12898 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
12899 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
12900 break;
12901 case 12:
12902 if (!Subtarget->hasLDSLoadB96_B128())
12903 return SDValue();
12904 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
12905 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
12906 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
12907 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
12908 break;
12909 case 16:
12910 if (!Subtarget->hasLDSLoadB96_B128())
12911 return SDValue();
12912 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
12913 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
12914 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
12915 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
12916 break;
12917 }
12918
12919 SDValue M0Val = copyToM0(DAG, Chain, DL, V: Op.getOperand(i: 3));
12920
12921 SmallVector<SDValue, 8> Ops;
12922
12923 if (HasVIndex && HasVOffset)
12924 Ops.push_back(Elt: DAG.getBuildVector(VT: MVT::v2i32, DL,
12925 Ops: {Op.getOperand(i: 5), // VIndex
12926 VOffset}));
12927 else if (HasVIndex)
12928 Ops.push_back(Elt: Op.getOperand(i: 5));
12929 else if (HasVOffset)
12930 Ops.push_back(Elt: VOffset);
12931
12932 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
12933 Ops.push_back(Elt: Rsrc);
12934 Ops.push_back(Elt: Op.getOperand(i: 6 + OpOffset)); // soffset
12935 Ops.push_back(Elt: Op.getOperand(i: 7 + OpOffset)); // imm offset
12936 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
12937 unsigned Aux = Op.getConstantOperandVal(i: 8 + OpOffset);
12938 Ops.push_back(Elt: DAG.getTargetConstant(
12939 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12),
12940 DL, VT: MVT::i8)); // cpol
12941 Ops.push_back(Elt: DAG.getTargetConstant(
12942 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::SWZ : AMDGPU::CPol::SWZ_pregfx12)
12943 ? 1
12944 : 0,
12945 DL, VT: MVT::i8)); // swz
12946 Ops.push_back(
12947 Elt: DAG.getTargetConstant(Val: isAsyncLDSDMA(Intr: IntrinsicID), DL, VT: MVT::i8));
12948 Ops.push_back(Elt: M0Val.getValue(R: 0)); // Chain
12949 Ops.push_back(Elt: M0Val.getValue(R: 1)); // Glue
12950
12951 auto *M = cast<MemSDNode>(Val&: Op);
12952 auto *Load = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: M->getVTList(), Ops);
12953 DAG.setNodeMemRefs(N: Load, NewMemRefs: M->memoperands());
12954
12955 return SDValue(Load, 0);
12956 }
12957 // Buffers are handled by LowerBufferFatPointers, and we're going to go
12958 // for "trust me" that the remaining cases are global pointers until
12959 // such time as we can put two mem operands on an intrinsic.
12960 case Intrinsic::amdgcn_load_to_lds:
12961 case Intrinsic::amdgcn_load_async_to_lds:
12962 case Intrinsic::amdgcn_global_load_lds:
12963 case Intrinsic::amdgcn_global_load_async_lds: {
12964 if (!Subtarget->hasVMemToLDSLoad())
12965 return SDValue();
12966
12967 unsigned Opc;
12968 unsigned Size = Op->getConstantOperandVal(Num: 4);
12969 switch (Size) {
12970 default:
12971 return SDValue();
12972 case 1:
12973 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
12974 break;
12975 case 2:
12976 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
12977 break;
12978 case 4:
12979 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
12980 break;
12981 case 12:
12982 if (!Subtarget->hasLDSLoadB96_B128())
12983 return SDValue();
12984 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
12985 break;
12986 case 16:
12987 if (!Subtarget->hasLDSLoadB96_B128())
12988 return SDValue();
12989 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
12990 break;
12991 }
12992
12993 SDValue M0Val = copyToM0(DAG, Chain, DL, V: Op.getOperand(i: 3));
12994
12995 SmallVector<SDValue, 6> Ops;
12996
12997 SDValue Addr = Op.getOperand(i: 2); // Global ptr
12998 SDValue VOffset;
12999 // Try to split SAddr and VOffset. Global and LDS pointers share the same
13000 // immediate offset, so we cannot use a regular SelectGlobalSAddr().
13001 if (Addr->isDivergent() && Addr->isAnyAdd()) {
13002 SDValue LHS = Addr.getOperand(i: 0);
13003 SDValue RHS = Addr.getOperand(i: 1);
13004
13005 if (LHS->isDivergent())
13006 std::swap(a&: LHS, b&: RHS);
13007
13008 if (!LHS->isDivergent() && RHS.getOpcode() == ISD::ZERO_EXTEND &&
13009 RHS.getOperand(i: 0).getValueType() == MVT::i32) {
13010 // add (i64 sgpr), (zero_extend (i32 vgpr))
13011 Addr = LHS;
13012 VOffset = RHS.getOperand(i: 0);
13013 }
13014 }
13015
13016 Ops.push_back(Elt: Addr);
13017 if (!Addr->isDivergent()) {
13018 Opc = AMDGPU::getGlobalSaddrOp(Opcode: Opc);
13019 if (!VOffset)
13020 VOffset =
13021 SDValue(DAG.getMachineNode(Opcode: AMDGPU::V_MOV_B32_e32, dl: DL, VT: MVT::i32,
13022 Op1: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32)),
13023 0);
13024 Ops.push_back(Elt: VOffset);
13025 }
13026
13027 Ops.push_back(Elt: Op.getOperand(i: 5)); // Offset
13028
13029 unsigned Aux = Op.getConstantOperandVal(i: 6);
13030 Ops.push_back(Elt: DAG.getTargetConstant(Val: Aux & ~AMDGPU::CPol::VIRTUAL_BITS, DL,
13031 VT: MVT::i32)); // CPol
13032 Ops.push_back(
13033 Elt: DAG.getTargetConstant(Val: isAsyncLDSDMA(Intr: IntrinsicID), DL, VT: MVT::i8));
13034
13035 Ops.push_back(Elt: M0Val.getValue(R: 0)); // Chain
13036 Ops.push_back(Elt: M0Val.getValue(R: 1)); // Glue
13037
13038 auto *M = cast<MemSDNode>(Val&: Op);
13039 auto *Load = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
13040 DAG.setNodeMemRefs(N: Load, NewMemRefs: M->memoperands());
13041
13042 return SDValue(Load, 0);
13043 }
13044 case Intrinsic::amdgcn_end_cf:
13045 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::SI_END_CF, dl: DL, VT: MVT::Other,
13046 Op1: Op->getOperand(Num: 2), Op2: Chain),
13047 0);
13048 case Intrinsic::amdgcn_s_barrier_signal_var: {
13049 // Member count of 0 means to re-use a previous member count,
13050 // which, if the named barrier is statically chosen, means we can use
13051 // the immarg form. Otherwisee, fall through to constructiong M0 as for
13052 // s_barrier_init.
13053 SDValue CntOp = Op->getOperand(Num: 3);
13054 auto *CntC = dyn_cast<ConstantSDNode>(Val&: CntOp);
13055 if (CntC && CntC->isZero()) {
13056 SDValue Chain = Op->getOperand(Num: 0);
13057 SDValue BarOp = Op->getOperand(Num: 2);
13058 SmallVector<SDValue, 2> Ops;
13059
13060 std::optional<uint64_t> BarVal;
13061 if (auto *C = dyn_cast<ConstantSDNode>(Val&: BarOp))
13062 BarVal = C->getZExtValue();
13063 else if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: BarOp))
13064 if (auto Addr = AMDGPUMachineFunctionInfo::get32BitAbsoluteAddress(
13065 GV: *GA->getGlobal(), AS: AMDGPUAS::BARRIER))
13066 BarVal = *Addr + GA->getOffset();
13067
13068 if (BarVal) {
13069 unsigned BarID = *BarVal & 0x3F;
13070 Ops.push_back(Elt: DAG.getTargetConstant(Val: BarID, DL, VT: MVT::i32));
13071 Ops.push_back(Elt: Chain);
13072 auto *NewMI = DAG.getMachineNode(Opcode: AMDGPU::S_BARRIER_SIGNAL_IMM, dl: DL,
13073 VTs: Op->getVTList(), Ops);
13074 return SDValue(NewMI, 0);
13075 }
13076 }
13077 [[fallthrough]];
13078 }
13079 case Intrinsic::amdgcn_s_barrier_init: {
13080 // these two intrinsics have two operands: barrier pointer and member count
13081 SDValue Chain = Op->getOperand(Num: 0);
13082 SmallVector<SDValue, 2> Ops;
13083 SDValue BarOp = Op->getOperand(Num: 2);
13084 SDValue CntOp = Op->getOperand(Num: 3);
13085 SDValue M0Val;
13086 unsigned Opc = IntrinsicID == Intrinsic::amdgcn_s_barrier_init
13087 ? AMDGPU::S_BARRIER_INIT_M0
13088 : AMDGPU::S_BARRIER_SIGNAL_M0;
13089 // extract the BarrierID from bits 0-5 of BarOp
13090 SDValue BarID = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: BarOp,
13091 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
13092 // Member count should be put into M0[ShAmt:+6]
13093 // Barrier ID should be put into M0[5:0]
13094 SDValue MemberCnt = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: CntOp,
13095 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
13096 constexpr unsigned ShAmt = 16;
13097 M0Val = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: MemberCnt,
13098 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT: MVT::i32, DL));
13099
13100 M0Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M0Val, N2: BarID);
13101
13102 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
13103
13104 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
13105 return SDValue(NewMI, 0);
13106 }
13107 case Intrinsic::amdgcn_s_wakeup_barrier: {
13108 if (!Subtarget->hasSWakeupBarrier())
13109 return SDValue();
13110 [[fallthrough]];
13111 }
13112 case Intrinsic::amdgcn_s_barrier_join: {
13113 // these three intrinsics have one operand: barrier pointer
13114 SDValue Chain = Op->getOperand(Num: 0);
13115 SmallVector<SDValue, 2> Ops;
13116 SDValue BarOp = Op->getOperand(Num: 2);
13117 unsigned Opc;
13118
13119 if (isa<ConstantSDNode>(Val: BarOp)) {
13120 uint64_t BarVal = cast<ConstantSDNode>(Val&: BarOp)->getZExtValue();
13121 switch (IntrinsicID) {
13122 default:
13123 return SDValue();
13124 case Intrinsic::amdgcn_s_barrier_join:
13125 Opc = AMDGPU::S_BARRIER_JOIN_IMM;
13126 break;
13127 case Intrinsic::amdgcn_s_wakeup_barrier:
13128 Opc = AMDGPU::S_WAKEUP_BARRIER_IMM;
13129 break;
13130 }
13131 // extract the BarrierID from bits 0-5 of the immediate
13132 unsigned BarID = BarVal & 0x3F;
13133 SDValue K = DAG.getTargetConstant(Val: BarID, DL, VT: MVT::i32);
13134 Ops.push_back(Elt: K);
13135 Ops.push_back(Elt: Chain);
13136 } else {
13137 switch (IntrinsicID) {
13138 default:
13139 return SDValue();
13140 case Intrinsic::amdgcn_s_barrier_join:
13141 Opc = AMDGPU::S_BARRIER_JOIN_M0;
13142 break;
13143 case Intrinsic::amdgcn_s_wakeup_barrier:
13144 Opc = AMDGPU::S_WAKEUP_BARRIER_M0;
13145 break;
13146 }
13147 // extract the BarrierID from bits 0-5 of BarOp, copy to M0[5:0]
13148 SDValue M0Val = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: BarOp,
13149 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
13150 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
13151 }
13152
13153 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
13154 return SDValue(NewMI, 0);
13155 }
13156 case Intrinsic::amdgcn_s_prefetch_data:
13157 case Intrinsic::amdgcn_s_prefetch_inst: {
13158 // For non-global address space preserve the chain and remove the call.
13159 if (!AMDGPU::isFlatGlobalAddrSpace(AS: cast<MemSDNode>(Val&: Op)->getAddressSpace()))
13160 return Op.getOperand(i: 0);
13161 return Op;
13162 }
13163 case Intrinsic::amdgcn_s_buffer_prefetch_data: {
13164 SDValue Ops[] = {
13165 Chain, bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG),
13166 Op.getOperand(i: 3), // offset
13167 Op.getOperand(i: 4), // length
13168 };
13169
13170 MemSDNode *M = cast<MemSDNode>(Val&: Op);
13171 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::SBUFFER_PREFETCH_DATA, dl: DL,
13172 VTList: Op->getVTList(), Ops, MemVT: M->getMemoryVT(),
13173 MMO: M->getMemOperand());
13174 }
13175 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
13176 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
13177 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
13178 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
13179 SDValue Chain = Op->getOperand(Num: 0);
13180 SDValue Ptr = Op->getOperand(Num: 2);
13181 SDValue Val = Op->getOperand(Num: 3);
13182 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MII->getMemoryVT(), Chain, Ptr: Val,
13183 Val: Ptr, MMO: MII->getMemOperand());
13184 }
13185 case Intrinsic::amdgcn_av_store_b128: {
13186 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
13187 SDValue Chain = Op->getOperand(Num: 0);
13188 SDValue Ptr = Op->getOperand(Num: 2);
13189 SDValue Val = Op->getOperand(Num: 3);
13190 return DAG.getStore(Chain, dl: DL, Val, Ptr, MMO: MII->getMemOperand());
13191 }
13192 default: {
13193 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
13194 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrinsicID))
13195 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: true);
13196
13197 return Op;
13198 }
13199 }
13200}
13201
13202// Return whether the operation has NoUnsignedWrap property.
13203static bool isNoUnsignedWrap(SDValue Addr) {
13204 return (Addr.getOpcode() == ISD::ADD &&
13205 Addr->getFlags().hasNoUnsignedWrap()) ||
13206 Addr->getOpcode() == ISD::OR;
13207}
13208
13209bool SITargetLowering::shouldPreservePtrArith(const Function &F,
13210 EVT PtrVT) const {
13211 return PtrVT == MVT::i64;
13212}
13213
13214bool SITargetLowering::canTransformPtrArithOutOfBounds(const Function &F,
13215 EVT PtrVT) const {
13216 return true;
13217}
13218
13219// The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
13220// offset (the offset that is included in bounds checking and swizzling, to be
13221// split between the instruction's voffset and immoffset fields) and soffset
13222// (the offset that is excluded from bounds checking and swizzling, to go in
13223// the instruction's soffset field). This function takes the first kind of
13224// offset and figures out how to split it between voffset and immoffset.
13225std::pair<SDValue, SDValue>
13226SITargetLowering::splitBufferOffsets(SDValue Offset, SelectionDAG &DAG) const {
13227 SDLoc DL(Offset);
13228 const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(ST: *Subtarget);
13229 SDValue N0 = Offset;
13230 ConstantSDNode *C1 = nullptr;
13231
13232 if ((C1 = dyn_cast<ConstantSDNode>(Val&: N0)))
13233 N0 = SDValue();
13234 else if (DAG.isBaseWithConstantOffset(Op: N0)) {
13235 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
13236 // being added, so we can only safely match a 32-bit addition with no
13237 // unsigned overflow.
13238 bool CheckNUW = Subtarget->hasGFX1250Insts();
13239 if (!CheckNUW || isNoUnsignedWrap(Addr: N0)) {
13240 C1 = cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
13241 N0 = N0.getOperand(i: 0);
13242 }
13243 }
13244
13245 if (C1) {
13246 unsigned ImmOffset = C1->getZExtValue();
13247 // If the immediate value is too big for the immoffset field, put only bits
13248 // that would normally fit in the immoffset field. The remaining value that
13249 // is copied/added for the voffset field is a large power of 2, and it
13250 // stands more chance of being CSEd with the copy/add for another similar
13251 // load/store.
13252 // However, do not do that rounding down if that is a negative
13253 // number, as it appears to be illegal to have a negative offset in the
13254 // vgpr, even if adding the immediate offset makes it positive.
13255 unsigned Overflow = ImmOffset & ~MaxImm;
13256 ImmOffset -= Overflow;
13257 if ((int32_t)Overflow < 0) {
13258 Overflow += ImmOffset;
13259 ImmOffset = 0;
13260 }
13261 C1 = cast<ConstantSDNode>(Val: DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32));
13262 if (Overflow) {
13263 auto OverflowVal = DAG.getConstant(Val: Overflow, DL, VT: MVT::i32);
13264 if (!N0)
13265 N0 = OverflowVal;
13266 else {
13267 SDValue Ops[] = {N0, OverflowVal};
13268 N0 = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, Ops);
13269 }
13270 }
13271 }
13272 if (!N0)
13273 N0 = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13274 if (!C1)
13275 C1 = cast<ConstantSDNode>(Val: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
13276 return {N0, SDValue(C1, 0)};
13277}
13278
13279// Analyze a combined offset from an amdgcn_s_buffer_load intrinsic and store
13280// the three offsets (voffset, soffset and instoffset) into the SDValue[3] array
13281// pointed to by Offsets.
13282void SITargetLowering::setBufferOffsets(SDValue CombinedOffset,
13283 SelectionDAG &DAG, SDValue *Offsets,
13284 Align Alignment) const {
13285 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
13286 SDLoc DL(CombinedOffset);
13287 if (auto *C = dyn_cast<ConstantSDNode>(Val&: CombinedOffset)) {
13288 uint32_t Imm = C->getZExtValue();
13289 uint32_t SOffset, ImmOffset;
13290 if (TII->splitMUBUFOffset(Imm, SOffset, ImmOffset, Alignment)) {
13291 Offsets[0] = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13292 Offsets[1] = DAG.getConstant(Val: SOffset, DL, VT: MVT::i32);
13293 Offsets[2] = DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32);
13294 return;
13295 }
13296 }
13297 if (DAG.isBaseWithConstantOffset(Op: CombinedOffset)) {
13298 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
13299 // being added, so we can only safely match a 32-bit addition with no
13300 // unsigned overflow.
13301 bool CheckNUW = Subtarget->hasGFX1250Insts();
13302 SDValue N0 = CombinedOffset.getOperand(i: 0);
13303 SDValue N1 = CombinedOffset.getOperand(i: 1);
13304 uint32_t SOffset, ImmOffset;
13305 int Offset = cast<ConstantSDNode>(Val&: N1)->getSExtValue();
13306 if (Offset >= 0 && (!CheckNUW || isNoUnsignedWrap(Addr: CombinedOffset)) &&
13307 TII->splitMUBUFOffset(Imm: Offset, SOffset, ImmOffset, Alignment)) {
13308 Offsets[0] = N0;
13309 Offsets[1] = DAG.getConstant(Val: SOffset, DL, VT: MVT::i32);
13310 Offsets[2] = DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32);
13311 return;
13312 }
13313 }
13314
13315 SDValue SOffsetZero = Subtarget->hasRestrictedSOffset()
13316 ? DAG.getRegister(Reg: AMDGPU::SGPR_NULL, VT: MVT::i32)
13317 : DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13318
13319 Offsets[0] = CombinedOffset;
13320 Offsets[1] = SOffsetZero;
13321 Offsets[2] = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32);
13322}
13323
13324SDValue SITargetLowering::bufferRsrcPtrToVector(SDValue MaybePointer,
13325 SelectionDAG &DAG) const {
13326 if (!MaybePointer.getValueType().isScalarInteger())
13327 return MaybePointer;
13328
13329 SDValue Rsrc = DAG.getBitcast(VT: MVT::v4i32, V: MaybePointer);
13330 return Rsrc;
13331}
13332
13333// Wrap a global or flat pointer into a buffer intrinsic using the flags
13334// specified in the intrinsic.
13335SDValue SITargetLowering::lowerPointerAsRsrcIntrin(SDNode *Op,
13336 SelectionDAG &DAG) const {
13337 SDLoc Loc(Op);
13338
13339 SDValue Pointer = Op->getOperand(Num: 1);
13340 SDValue Stride = Op->getOperand(Num: 2);
13341 SDValue NumRecords = Op->getOperand(Num: 3);
13342 SDValue Flags = Op->getOperand(Num: 4);
13343
13344 SDValue ExtStride = DAG.getAnyExtOrTrunc(Op: Stride, DL: Loc, VT: MVT::i32);
13345 SDValue Rsrc;
13346
13347 if (Subtarget->getBufferResourceNumRecordsWidth() == 45) {
13348 NumRecords = DAG.getZExtOrTrunc(Op: NumRecords, DL: Loc, VT: MVT::i64);
13349 NumRecords = DAG.getNode(Opcode: ISD::AND, DL: Loc, VT: MVT::i64, N1: NumRecords,
13350 N2: DAG.getConstant(Val: (1ULL << 45) - 1, DL: Loc, VT: MVT::i64));
13351 SDValue Zero = DAG.getConstant(Val: 0, DL: Loc, VT: MVT::i32);
13352 // Build the lower 64-bit value, which has a 57-bit base and the lower 7-bit
13353 // num_records.
13354 SDValue ExtPointer = DAG.getAnyExtOrTrunc(Op: Pointer, DL: Loc, VT: MVT::i64);
13355 SDValue NumRecordsLHS =
13356 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i64, N1: NumRecords,
13357 N2: DAG.getShiftAmountConstant(Val: 57, VT: MVT::i32, DL: Loc));
13358 SDValue LowHalf =
13359 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: ExtPointer, N2: NumRecordsLHS);
13360
13361 // Build the higher 64-bit value, which has the higher 38-bit num_records,
13362 // 6-bit zero (omit), 16-bit stride and scale and 4-bit flag.
13363 SDValue NumRecordsRHS =
13364 DAG.getNode(Opcode: ISD::SRL, DL: Loc, VT: MVT::i64, N1: NumRecords,
13365 N2: DAG.getShiftAmountConstant(Val: 7, VT: MVT::i32, DL: Loc));
13366 SDValue ShiftedStride =
13367 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: ExtStride,
13368 N2: DAG.getShiftAmountConstant(Val: 12, VT: MVT::i32, DL: Loc));
13369 SDValue ExtShiftedStrideVec =
13370 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i32, N1: Zero, N2: ShiftedStride);
13371 SDValue ExtShiftedStride =
13372 DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i64, Operand: ExtShiftedStrideVec);
13373 SDValue ShiftedFlags =
13374 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: Flags,
13375 N2: DAG.getShiftAmountConstant(Val: 28, VT: MVT::i32, DL: Loc));
13376 SDValue ExtShiftedFlagsVec =
13377 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i32, N1: Zero, N2: ShiftedFlags);
13378 SDValue ExtShiftedFlags =
13379 DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i64, Operand: ExtShiftedFlagsVec);
13380 SDValue CombinedFields =
13381 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: NumRecordsRHS, N2: ExtShiftedStride);
13382 SDValue HighHalf =
13383 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: CombinedFields, N2: ExtShiftedFlags);
13384
13385 Rsrc = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i64, N1: LowHalf, N2: HighHalf);
13386 } else {
13387 NumRecords = DAG.getZExtOrTrunc(Op: NumRecords, DL: Loc, VT: MVT::i32);
13388 auto [LowHalf, HighHalf] =
13389 DAG.SplitScalar(N: Pointer, DL: Loc, LoVT: MVT::i32, HiVT: MVT::i32);
13390 SDValue Mask = DAG.getConstant(Val: 0x0000ffff, DL: Loc, VT: MVT::i32);
13391 SDValue Masked = DAG.getNode(Opcode: ISD::AND, DL: Loc, VT: MVT::i32, N1: HighHalf, N2: Mask);
13392 SDValue ShiftedStride =
13393 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: ExtStride,
13394 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: Loc));
13395 SDValue NewHighHalf =
13396 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i32, N1: Masked, N2: ShiftedStride);
13397
13398 Rsrc = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v4i32, N1: LowHalf, N2: NewHighHalf,
13399 N3: NumRecords, N4: Flags);
13400 }
13401
13402 SDValue RsrcPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i128, Operand: Rsrc);
13403 return RsrcPtr;
13404}
13405
13406// Handle 8 bit and 16 bit buffer loads
13407SDValue SITargetLowering::handleByteShortBufferLoads(SelectionDAG &DAG,
13408 EVT LoadVT, SDLoc DL,
13409 ArrayRef<SDValue> Ops,
13410 MachineMemOperand *MMO,
13411 bool IsTFE) const {
13412 EVT IntVT = LoadVT.changeTypeToInteger();
13413
13414 if (IsTFE) {
13415 unsigned Opc = (LoadVT.getScalarType() == MVT::i8)
13416 ? AMDGPUISD::BUFFER_LOAD_UBYTE_TFE
13417 : AMDGPUISD::BUFFER_LOAD_USHORT_TFE;
13418 MachineFunction &MF = DAG.getMachineFunction();
13419 MachineMemOperand *OpMMO = MF.getMachineMemOperand(MMO, Offset: 0, Size: 8);
13420 SDVTList VTs = DAG.getVTList(VT1: MVT::v2i32, VT2: MVT::Other);
13421 SDValue Op = getMemIntrinsicNode(Opcode: Opc, DL, VTList: VTs, Ops, MemVT: MVT::v2i32, MMO: OpMMO, DAG);
13422 SDValue Status = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
13423 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
13424 SDValue Data = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
13425 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
13426 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: Data);
13427 SDValue Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: Trunc);
13428 return DAG.getMergeValues(Ops: {Value, Status, SDValue(Op.getNode(), 1)}, dl: DL);
13429 }
13430
13431 unsigned Opc = LoadVT.getScalarType() == MVT::i8
13432 ? AMDGPUISD::BUFFER_LOAD_UBYTE
13433 : AMDGPUISD::BUFFER_LOAD_USHORT;
13434
13435 SDVTList ResList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
13436 SDValue BufferLoad =
13437 DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: ResList, Ops, MemVT: IntVT, MMO);
13438 SDValue LoadVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: BufferLoad);
13439 LoadVal = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: LoadVal);
13440
13441 return DAG.getMergeValues(Ops: {LoadVal, BufferLoad.getValue(R: 1)}, dl: DL);
13442}
13443
13444// Handle 8 bit and 16 bit buffer stores
13445SDValue SITargetLowering::handleByteShortBufferStores(SelectionDAG &DAG,
13446 EVT VDataType, SDLoc DL,
13447 SDValue Ops[],
13448 MemSDNode *M) const {
13449 if (VDataType == MVT::f16 || VDataType == MVT::bf16)
13450 Ops[1] = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Ops[1]);
13451
13452 SDValue BufferStoreExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Ops[1]);
13453 Ops[1] = BufferStoreExt;
13454 unsigned Opc = (VDataType == MVT::i8) ? AMDGPUISD::BUFFER_STORE_BYTE
13455 : AMDGPUISD::BUFFER_STORE_SHORT;
13456 ArrayRef<SDValue> OpsRef = ArrayRef(&Ops[0], 9);
13457 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: M->getVTList(), Ops: OpsRef, MemVT: VDataType,
13458 MMO: M->getMemOperand());
13459}
13460
13461static SDValue getLoadExtOrTrunc(SelectionDAG &DAG, ISD::LoadExtType ExtType,
13462 SDValue Op, const SDLoc &SL, EVT VT) {
13463 if (VT.bitsLT(VT: Op.getValueType()))
13464 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
13465
13466 switch (ExtType) {
13467 case ISD::SEXTLOAD:
13468 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SL, VT, Operand: Op);
13469 case ISD::ZEXTLOAD:
13470 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT, Operand: Op);
13471 case ISD::EXTLOAD:
13472 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT, Operand: Op);
13473 case ISD::NON_EXTLOAD:
13474 return Op;
13475 }
13476
13477 llvm_unreachable("invalid ext type");
13478}
13479
13480// Try to turn 8 and 16-bit scalar loads into SMEM eligible 32-bit loads.
13481// TODO: Skip this on GFX12 which does have scalar sub-dword loads.
13482SDValue SITargetLowering::widenLoad(LoadSDNode *Ld,
13483 DAGCombinerInfo &DCI) const {
13484 SelectionDAG &DAG = DCI.DAG;
13485 if (Ld->getAlign() < Align(4) || Ld->isDivergent())
13486 return SDValue();
13487
13488 // FIXME: Constant loads should all be marked invariant.
13489 unsigned AS = Ld->getAddressSpace();
13490 if (AS != AMDGPUAS::CONSTANT_ADDRESS &&
13491 AS != AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
13492 (AS != AMDGPUAS::GLOBAL_ADDRESS || !Ld->isInvariant()))
13493 return SDValue();
13494
13495 // Don't do this early, since it may interfere with adjacent load merging for
13496 // illegal types. We can avoid losing alignment information for exotic types
13497 // pre-legalize.
13498 EVT MemVT = Ld->getMemoryVT();
13499 if ((MemVT.isSimple() && !DCI.isAfterLegalizeDAG()) ||
13500 MemVT.getSizeInBits() >= 32)
13501 return SDValue();
13502
13503 SDLoc SL(Ld);
13504
13505 assert((!MemVT.isVector() || Ld->getExtensionType() == ISD::NON_EXTLOAD) &&
13506 "unexpected vector extload");
13507
13508 // TODO: Drop only high part of range.
13509 SDValue Ptr = Ld->getBasePtr();
13510 SDValue NewLoad = DAG.getLoad(
13511 AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT: MVT::i32, dl: SL, Chain: Ld->getChain(), Ptr,
13512 Offset: Ld->getOffset(), PtrInfo: Ld->getPointerInfo(), MemVT: MVT::i32, Alignment: Ld->getAlign(),
13513 MMOFlags: Ld->getMemOperand()->getFlags(), Metadata: Ld->getAAInfo()); // Drop ranges
13514
13515 EVT TruncVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
13516 if (MemVT.isFloatingPoint()) {
13517 assert(Ld->getExtensionType() == ISD::NON_EXTLOAD &&
13518 "unexpected fp extload");
13519 TruncVT = MemVT.changeTypeToInteger();
13520 }
13521
13522 SDValue Cvt = NewLoad;
13523 if (Ld->getExtensionType() == ISD::SEXTLOAD) {
13524 Cvt = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SL, VT: MVT::i32, N1: NewLoad,
13525 N2: DAG.getValueType(TruncVT));
13526 } else if (Ld->getExtensionType() == ISD::ZEXTLOAD ||
13527 Ld->getExtensionType() == ISD::NON_EXTLOAD) {
13528 Cvt = DAG.getZeroExtendInReg(Op: NewLoad, DL: SL, VT: TruncVT);
13529 } else {
13530 assert(Ld->getExtensionType() == ISD::EXTLOAD);
13531 }
13532
13533 EVT VT = Ld->getValueType(ResNo: 0);
13534 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
13535
13536 DCI.AddToWorklist(N: Cvt.getNode());
13537
13538 // We may need to handle exotic cases, such as i16->i64 extloads, so insert
13539 // the appropriate extension from the 32-bit load.
13540 Cvt = getLoadExtOrTrunc(DAG, ExtType: Ld->getExtensionType(), Op: Cvt, SL, VT: IntVT);
13541 DCI.AddToWorklist(N: Cvt.getNode());
13542
13543 // Handle conversion back to floating point if necessary.
13544 Cvt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Cvt);
13545
13546 return DAG.getMergeValues(Ops: {Cvt, NewLoad.getValue(R: 1)}, dl: SL);
13547}
13548
13549static bool addressMayBeAccessedAsPrivate(const MachineMemOperand *MMO,
13550 const SIMachineFunctionInfo &Info) {
13551 // TODO: Should check if the address can definitely not access stack.
13552 if (Info.isEntryFunction())
13553 return Info.getUserSGPRInfo().hasFlatScratchInit();
13554 return true;
13555}
13556
13557SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
13558 SDLoc DL(Op);
13559 LoadSDNode *Load = cast<LoadSDNode>(Val&: Op);
13560 ISD::LoadExtType ExtType = Load->getExtensionType();
13561 EVT MemVT = Load->getMemoryVT();
13562 MachineMemOperand *MMO = Load->getMemOperand();
13563
13564 if (ExtType == ISD::NON_EXTLOAD && MemVT.getSizeInBits() < 32) {
13565 if (MemVT == MVT::i16 && isTypeLegal(VT: MVT::i16))
13566 return SDValue();
13567
13568 // FIXME: Copied from PPC
13569 // First, load into 32 bits, then truncate to 1 bit.
13570
13571 SDValue Chain = Load->getChain();
13572 SDValue BasePtr = Load->getBasePtr();
13573
13574 EVT RealMemVT = (MemVT == MVT::i1) ? MVT::i8 : MVT::i16;
13575
13576 SDValue NewLD = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: MVT::i32, Chain, Ptr: BasePtr,
13577 MemVT: RealMemVT, MMO);
13578
13579 if (!MemVT.isVector()) {
13580 SDValue Ops[] = {DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MemVT, Operand: NewLD),
13581 NewLD.getValue(R: 1)};
13582
13583 return DAG.getMergeValues(Ops, dl: DL);
13584 }
13585
13586 SmallVector<SDValue, 3> Elts;
13587 for (unsigned I = 0, N = MemVT.getVectorNumElements(); I != N; ++I) {
13588 SDValue Elt = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: NewLD,
13589 N2: DAG.getConstant(Val: I, DL, VT: MVT::i32));
13590
13591 Elts.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Elt));
13592 }
13593
13594 SDValue Ops[] = {DAG.getBuildVector(VT: MemVT, DL, Ops: Elts), NewLD.getValue(R: 1)};
13595
13596 return DAG.getMergeValues(Ops, dl: DL);
13597 }
13598
13599 if (!MemVT.isVector())
13600 return SDValue();
13601
13602 assert(Op.getValueType().getVectorElementType() == MVT::i32 &&
13603 "Custom lowering for non-i32 vectors hasn't been implemented.");
13604
13605 Align Alignment = Load->getAlign();
13606 unsigned AS = Load->getAddressSpace();
13607 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
13608 AS == AMDGPUAS::FLAT_ADDRESS &&
13609 Alignment.value() < MemVT.getStoreSize() && MemVT.getSizeInBits() > 32) {
13610 return SplitVectorLoad(Op, DAG);
13611 }
13612
13613 MachineFunction &MF = DAG.getMachineFunction();
13614 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
13615 // If there is a possibility that flat instruction access scratch memory
13616 // then we need to use the same legalization rules we use for private.
13617 if (AS == AMDGPUAS::FLAT_ADDRESS &&
13618 !Subtarget->hasMultiDwordFlatScratchAddressing())
13619 AS = addressMayBeAccessedAsPrivate(MMO: Load->getMemOperand(), Info: *MFI)
13620 ? AMDGPUAS::PRIVATE_ADDRESS
13621 : AMDGPUAS::GLOBAL_ADDRESS;
13622
13623 unsigned NumElements = MemVT.getVectorNumElements();
13624
13625 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
13626 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
13627 (AS == AMDGPUAS::GLOBAL_ADDRESS &&
13628 Subtarget->getScalarizeGlobalBehavior() && Load->isSimple() &&
13629 (Load->isInvariant() || isMemOpHasNoClobberedMemOperand(N: Load)))) {
13630 if ((!Op->isDivergent() || AMDGPU::isUniformMMO(MMO)) &&
13631 Alignment >= Align(4) && NumElements < 32) {
13632 if (MemVT.isPow2VectorType() ||
13633 (Subtarget->hasScalarDwordx3Loads() && NumElements == 3))
13634 return SDValue();
13635 return WidenOrSplitVectorLoad(Op, DAG);
13636 }
13637 // Non-uniform loads will be selected to MUBUF instructions, so they
13638 // have the same legalization requirements as global and private
13639 // loads.
13640 //
13641 }
13642 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
13643 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
13644 AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS) {
13645 if (NumElements > 4)
13646 return SplitVectorLoad(Op, DAG);
13647 // v3 loads not supported on SI.
13648 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
13649 return WidenOrSplitVectorLoad(Op, DAG);
13650
13651 // v3 and v4 loads are supported for private and global memory.
13652 return SDValue();
13653 }
13654 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
13655 // Depending on the setting of the private_element_size field in the
13656 // resource descriptor, we can only make private accesses up to a certain
13657 // size.
13658 switch (Subtarget->getMaxPrivateElementSize()) {
13659 case 4: {
13660 auto [Op0, Op1] = scalarizeVectorLoad(LD: Load, DAG);
13661 return DAG.getMergeValues(Ops: {Op0, Op1}, dl: DL);
13662 }
13663 case 8:
13664 if (NumElements > 2)
13665 return SplitVectorLoad(Op, DAG);
13666 return SDValue();
13667 case 16:
13668 // Same as global/flat
13669 if (NumElements > 4)
13670 return SplitVectorLoad(Op, DAG);
13671 // v3 loads not supported on SI.
13672 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
13673 return WidenOrSplitVectorLoad(Op, DAG);
13674
13675 return SDValue();
13676 default:
13677 llvm_unreachable("unsupported private_element_size");
13678 }
13679 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
13680 unsigned Fast = 0;
13681 auto Flags = Load->getMemOperand()->getFlags();
13682 if (allowsMisalignedMemoryAccessesImpl(Size: MemVT.getSizeInBits(), AddrSpace: AS,
13683 Alignment: Load->getAlign(), Flags, IsFast: &Fast) &&
13684 Fast > 1)
13685 return SDValue();
13686
13687 if (MemVT.isVector())
13688 return SplitVectorLoad(Op, DAG);
13689 }
13690
13691 if (!allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
13692 VT: MemVT, MMO: *Load->getMemOperand())) {
13693 auto [Op0, Op1] = expandUnalignedLoad(LD: Load, DAG);
13694 return DAG.getMergeValues(Ops: {Op0, Op1}, dl: DL);
13695 }
13696
13697 return SDValue();
13698}
13699
13700SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
13701 EVT VT = Op.getValueType();
13702 if (VT.getSizeInBits() == 128 || VT.getSizeInBits() == 256 ||
13703 VT.getSizeInBits() == 512)
13704 return splitTernaryVectorOp(Op, DAG);
13705
13706 assert(VT.getSizeInBits() == 64);
13707
13708 SDLoc DL(Op);
13709 SDValue Cond = DAG.getFreeze(V: Op.getOperand(i: 0));
13710
13711 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13712 SDValue One = DAG.getConstant(Val: 1, DL, VT: MVT::i32);
13713
13714 SDValue LHS = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
13715 SDValue RHS = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 2));
13716
13717 SDValue Lo0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: LHS, N2: Zero);
13718 SDValue Lo1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: RHS, N2: Zero);
13719
13720 SDValue Lo = DAG.getSelect(DL, VT: MVT::i32, Cond, LHS: Lo0, RHS: Lo1);
13721
13722 SDValue Hi0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: LHS, N2: One);
13723 SDValue Hi1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: RHS, N2: One);
13724
13725 SDValue Hi = DAG.getSelect(DL, VT: MVT::i32, Cond, LHS: Hi0, RHS: Hi1);
13726
13727 SDValue Res = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Lo, Hi});
13728 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Res);
13729}
13730
13731// Catch division cases where we can use shortcuts with rcp and rsq
13732// instructions.
13733SDValue SITargetLowering::lowerFastUnsafeFDIV(SDValue Op,
13734 SelectionDAG &DAG) const {
13735 SDLoc SL(Op);
13736 SDValue LHS = Op.getOperand(i: 0);
13737 SDValue RHS = Op.getOperand(i: 1);
13738 EVT VT = Op.getValueType();
13739 const SDNodeFlags Flags = Op->getFlags();
13740
13741 bool AllowInaccurateRcp = Flags.hasApproximateFuncs();
13742
13743 if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
13744 // Without !fpmath accuracy information, we can't do more because we don't
13745 // know exactly whether rcp is accurate enough to meet !fpmath requirement.
13746 // f16 is always accurate enough
13747 if (!AllowInaccurateRcp && VT != MVT::f16 && VT != MVT::bf16)
13748 return SDValue();
13749
13750 if (CLHS->isOne()) {
13751 // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
13752 // the CI documentation has a worst case error of 1 ulp.
13753 // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to
13754 // use it as long as we aren't trying to use denormals.
13755 //
13756 // v_rcp_f16 and v_rsq_f16 DO support denormals and 0.51ulp.
13757
13758 // 1.0 / sqrt(x) -> rsq(x)
13759
13760 // XXX - Is afn sufficient to do this for f64? The maximum ULP
13761 // error seems really high at 2^29 ULP.
13762 // 1.0 / x -> rcp(x)
13763 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: RHS);
13764 }
13765
13766 // Same as for 1.0, but expand the sign out of the constant.
13767 if (CLHS->isMinusOne()) {
13768 // -1.0 / x -> rcp (fneg x)
13769 SDValue FNegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
13770 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: FNegRHS);
13771 }
13772 }
13773
13774 // For f16 and bf16 require afn or arcp.
13775 // For f32 require afn.
13776 if (!AllowInaccurateRcp &&
13777 ((VT != MVT::f16 && VT != MVT::bf16) || !Flags.hasAllowReciprocal()))
13778 return SDValue();
13779
13780 // Turn into multiply by the reciprocal.
13781 // x / y -> x * (1.0 / y)
13782 SDValue Recip = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: RHS);
13783 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: LHS, N2: Recip, Flags);
13784}
13785
13786SDValue SITargetLowering::lowerFastUnsafeFDIV64(SDValue Op,
13787 SelectionDAG &DAG) const {
13788 SDLoc SL(Op);
13789 SDValue X = Op.getOperand(i: 0);
13790 SDValue Y = Op.getOperand(i: 1);
13791 EVT VT = Op.getValueType();
13792 const SDNodeFlags Flags = Op->getFlags();
13793
13794 bool AllowInaccurateDiv = Flags.hasApproximateFuncs();
13795 if (!AllowInaccurateDiv)
13796 return SDValue();
13797
13798 const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: X);
13799 bool IsNegRcp = CLHS && CLHS->isMinusOne();
13800
13801 // Pull out the negation so it folds for free into the source modifiers.
13802 if (IsNegRcp)
13803 X = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
13804
13805 SDValue NegY = IsNegRcp ? Y : DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Y);
13806 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
13807
13808 SDValue R = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: Y);
13809 if (IsNegRcp)
13810 R = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: R);
13811
13812 SDValue Tmp0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: R, N3: One);
13813
13814 R = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp0, N2: R, N3: R);
13815 SDValue Tmp1 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: R, N3: One);
13816 R = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp1, N2: R, N3: R);
13817
13818 // Skip the last 2 correction terms for reciprocal.
13819 if (IsNegRcp || (CLHS && CLHS->isOne()))
13820 return R;
13821
13822 SDValue Ret = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: R);
13823 SDValue Tmp2 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: Ret, N3: X);
13824 return DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp2, N2: R, N3: Ret);
13825}
13826
13827static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
13828 EVT VT, SDValue A, SDValue B, SDValue GlueChain,
13829 SDNodeFlags Flags) {
13830 if (GlueChain->getNumValues() <= 1) {
13831 return DAG.getNode(Opcode, DL: SL, VT, N1: A, N2: B, Flags);
13832 }
13833
13834 assert(GlueChain->getNumValues() == 3);
13835
13836 SDVTList VTList = DAG.getVTList(VT1: VT, VT2: MVT::Other, VT3: MVT::Glue);
13837 switch (Opcode) {
13838 default:
13839 llvm_unreachable("no chain equivalent for opcode");
13840 case ISD::FMUL:
13841 Opcode = AMDGPUISD::FMUL_W_CHAIN;
13842 break;
13843 }
13844
13845 return DAG.getNode(Opcode, DL: SL, VTList,
13846 Ops: {GlueChain.getValue(R: 1), A, B, GlueChain.getValue(R: 2)},
13847 Flags);
13848}
13849
13850static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
13851 EVT VT, SDValue A, SDValue B, SDValue C,
13852 SDValue GlueChain, SDNodeFlags Flags) {
13853 if (GlueChain->getNumValues() <= 1) {
13854 return DAG.getNode(Opcode, DL: SL, VT, Ops: {A, B, C}, Flags);
13855 }
13856
13857 assert(GlueChain->getNumValues() == 3);
13858
13859 SDVTList VTList = DAG.getVTList(VT1: VT, VT2: MVT::Other, VT3: MVT::Glue);
13860 switch (Opcode) {
13861 default:
13862 llvm_unreachable("no chain equivalent for opcode");
13863 case ISD::FMA:
13864 Opcode = AMDGPUISD::FMA_W_CHAIN;
13865 break;
13866 }
13867
13868 return DAG.getNode(Opcode, DL: SL, VTList,
13869 Ops: {GlueChain.getValue(R: 1), A, B, C, GlueChain.getValue(R: 2)},
13870 Flags);
13871}
13872
13873SDValue SITargetLowering::LowerFDIV16(SDValue Op, SelectionDAG &DAG) const {
13874 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
13875 return FastLowered;
13876
13877 SDLoc SL(Op);
13878 EVT VT = Op.getValueType();
13879 SDValue LHS = Op.getOperand(i: 0);
13880 SDValue RHS = Op.getOperand(i: 1);
13881
13882 SDValue LHSExt = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: LHS);
13883 SDValue RHSExt = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: RHS);
13884
13885 if (VT == MVT::bf16) {
13886 SDValue ExtDiv =
13887 DAG.getNode(Opcode: ISD::FDIV, DL: SL, VT: MVT::f32, N1: LHSExt, N2: RHSExt, Flags: Op->getFlags());
13888 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::bf16, N1: ExtDiv,
13889 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32));
13890 }
13891
13892 assert(VT == MVT::f16);
13893
13894 // a32.u = opx(V_CVT_F32_F16, a.u); // CVT to F32
13895 // b32.u = opx(V_CVT_F32_F16, b.u); // CVT to F32
13896 // r32.u = opx(V_RCP_F32, b32.u); // rcp = 1 / d
13897 // q32.u = opx(V_MUL_F32, a32.u, r32.u); // q = n * rcp
13898 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
13899 // q32.u = opx(V_MAD_F32, e32.u, r32.u, q32.u); // q = n * rcp
13900 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
13901 // tmp.u = opx(V_MUL_F32, e32.u, r32.u);
13902 // tmp.u = opx(V_AND_B32, tmp.u, 0xff800000)
13903 // q32.u = opx(V_ADD_F32, tmp.u, q32.u);
13904 // q16.u = opx(V_CVT_F16_F32, q32.u);
13905 // q16.u = opx(V_DIV_FIXUP_F16, q16.u, b.u, a.u); // q = touchup(q, d, n)
13906
13907 // We will use ISD::FMA on targets that don't support ISD::FMAD.
13908 unsigned FMADOpCode =
13909 isOperationLegal(Op: ISD::FMAD, VT: MVT::f32) ? ISD::FMAD : ISD::FMA;
13910 SDValue NegRHSExt = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: RHSExt);
13911 SDValue Rcp =
13912 DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: RHSExt, Flags: Op->getFlags());
13913 SDValue Quot =
13914 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: LHSExt, N2: Rcp, Flags: Op->getFlags());
13915 SDValue Err = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: NegRHSExt, N2: Quot, N3: LHSExt,
13916 Flags: Op->getFlags());
13917 Quot = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: Err, N2: Rcp, N3: Quot, Flags: Op->getFlags());
13918 Err = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: NegRHSExt, N2: Quot, N3: LHSExt,
13919 Flags: Op->getFlags());
13920 SDValue Tmp = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: Err, N2: Rcp, Flags: Op->getFlags());
13921 SDValue TmpCast = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Tmp);
13922 TmpCast = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: TmpCast,
13923 N2: DAG.getConstant(Val: 0xff800000, DL: SL, VT: MVT::i32));
13924 Tmp = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: TmpCast);
13925 Quot = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f32, N1: Tmp, N2: Quot, Flags: Op->getFlags());
13926 SDValue RDst = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::f16, N1: Quot,
13927 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32));
13928 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f16, N1: RDst, N2: RHS, N3: LHS,
13929 Flags: Op->getFlags());
13930}
13931
13932// Faster 2.5 ULP division that does not support denormals.
13933SDValue SITargetLowering::lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const {
13934 SDNodeFlags Flags = Op->getFlags();
13935 SDLoc SL(Op);
13936 SDValue LHS = Op.getOperand(i: 1);
13937 SDValue RHS = Op.getOperand(i: 2);
13938
13939 // TODO: The combiner should probably handle elimination of redundant fabs.
13940 SDValue r1 = DAG.SignBitIsZeroFP(Op: RHS)
13941 ? RHS
13942 : DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: MVT::f32, Operand: RHS, Flags);
13943
13944 const APFloat K0Val(0x1p+96f);
13945 const SDValue K0 = DAG.getConstantFP(Val: K0Val, DL: SL, VT: MVT::f32);
13946
13947 const APFloat K1Val(0x1p-32f);
13948 const SDValue K1 = DAG.getConstantFP(Val: K1Val, DL: SL, VT: MVT::f32);
13949
13950 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f32);
13951
13952 EVT SetCCVT =
13953 getSetCCResultType(DL: DAG.getDataLayout(), Ctx&: *DAG.getContext(), VT: MVT::f32);
13954
13955 SDValue r2 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: r1, RHS: K0, Cond: ISD::SETOGT);
13956
13957 SDValue r3 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f32, N1: r2, N2: K1, N3: One, Flags);
13958
13959 r1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: RHS, N2: r3, Flags);
13960
13961 // rcp does not support denormals.
13962 SDValue r0 = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: r1, Flags);
13963
13964 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: LHS, N2: r0, Flags);
13965
13966 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: r3, N2: Mul, Flags);
13967}
13968
13969// Returns immediate value for setting the F32 denorm mode when using the
13970// S_DENORM_MODE instruction.
13971static SDValue getSPDenormModeValue(uint32_t SPDenormMode, SelectionDAG &DAG,
13972 const SIMachineFunctionInfo *Info,
13973 const GCNSubtarget *ST) {
13974 assert(ST->hasDenormModeInst() && "Requires S_DENORM_MODE");
13975 uint32_t DPDenormModeDefault = Info->getMode().fpDenormModeDPValue();
13976 uint32_t Mode = SPDenormMode | (DPDenormModeDefault << 2);
13977 return DAG.getTargetConstant(Val: Mode, DL: SDLoc(), VT: MVT::i32);
13978}
13979
13980SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const {
13981 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
13982 return FastLowered;
13983
13984 // The selection matcher assumes anything with a chain selecting to a
13985 // mayRaiseFPException machine instruction. Since we're introducing a chain
13986 // here, we need to explicitly report nofpexcept for the regular fdiv
13987 // lowering.
13988 SDNodeFlags Flags = Op->getFlags();
13989 Flags.setNoFPExcept(true);
13990
13991 SDLoc SL(Op);
13992 SDValue LHS = Op.getOperand(i: 0);
13993 SDValue RHS = Op.getOperand(i: 1);
13994
13995 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f32);
13996
13997 SDVTList ScaleVT = DAG.getVTList(VT1: MVT::f32, VT2: MVT::i1);
13998
13999 SDValue DenominatorScaled =
14000 DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, Ops: {RHS, RHS, LHS}, Flags);
14001 SDValue NumeratorScaled =
14002 DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, Ops: {LHS, RHS, LHS}, Flags);
14003
14004 // Denominator is scaled to not be denormal, so using rcp is ok.
14005 SDValue ApproxRcp =
14006 DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: DenominatorScaled, Flags);
14007 SDValue NegDivScale0 =
14008 DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: DenominatorScaled, Flags);
14009
14010 using namespace AMDGPU::Hwreg;
14011 const unsigned Denorm32Reg = HwregEncoding::encode(Values: ID_MODE, Values: 4, Values: 2);
14012 const SDValue BitField = DAG.getTargetConstant(Val: Denorm32Reg, DL: SL, VT: MVT::i32);
14013
14014 const MachineFunction &MF = DAG.getMachineFunction();
14015 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
14016 const DenormalMode DenormMode = Info->getMode().FP32Denormals;
14017
14018 const bool PreservesDenormals = DenormMode == DenormalMode::getIEEE();
14019 const bool HasDynamicDenormals =
14020 (DenormMode.Input == DenormalMode::Dynamic) ||
14021 (DenormMode.Output == DenormalMode::Dynamic);
14022
14023 SDValue SavedDenormMode;
14024
14025 if (!PreservesDenormals) {
14026 // Note we can't use the STRICT_FMA/STRICT_FMUL for the non-strict FDIV
14027 // lowering. The chain dependence is insufficient, and we need glue. We do
14028 // not need the glue variants in a strictfp function.
14029
14030 SDVTList BindParamVTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
14031
14032 SDValue Glue = DAG.getEntryNode();
14033 if (HasDynamicDenormals) {
14034 SDNode *GetReg = DAG.getMachineNode(Opcode: AMDGPU::S_GETREG_B32, dl: SL,
14035 VTs: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Glue),
14036 Ops: {BitField, Glue});
14037 SavedDenormMode = SDValue(GetReg, 0);
14038
14039 Glue = DAG.getMergeValues(
14040 Ops: {DAG.getEntryNode(), SDValue(GetReg, 0), SDValue(GetReg, 1)}, dl: SL);
14041 }
14042
14043 SDNode *EnableDenorm;
14044 if (Subtarget->hasDenormModeInst()) {
14045 const SDValue EnableDenormValue =
14046 getSPDenormModeValue(FP_DENORM_FLUSH_NONE, DAG, Info, ST: Subtarget);
14047
14048 EnableDenorm = DAG.getNode(Opcode: AMDGPUISD::DENORM_MODE, DL: SL, VTList: BindParamVTs, N1: Glue,
14049 N2: EnableDenormValue)
14050 .getNode();
14051 } else {
14052 const SDValue EnableDenormValue =
14053 DAG.getConstant(FP_DENORM_FLUSH_NONE, DL: SL, VT: MVT::i32);
14054 EnableDenorm = DAG.getMachineNode(Opcode: AMDGPU::S_SETREG_B32, dl: SL, VTs: BindParamVTs,
14055 Ops: {EnableDenormValue, BitField, Glue});
14056 }
14057
14058 SDValue Ops[3] = {NegDivScale0, SDValue(EnableDenorm, 0),
14059 SDValue(EnableDenorm, 1)};
14060
14061 NegDivScale0 = DAG.getMergeValues(Ops, dl: SL);
14062 }
14063
14064 SDValue Fma0 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0,
14065 B: ApproxRcp, C: One, GlueChain: NegDivScale0, Flags);
14066
14067 SDValue Fma1 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: Fma0, B: ApproxRcp,
14068 C: ApproxRcp, GlueChain: Fma0, Flags);
14069
14070 SDValue Mul = getFPBinOp(DAG, Opcode: ISD::FMUL, SL, VT: MVT::f32, A: NumeratorScaled, B: Fma1,
14071 GlueChain: Fma1, Flags);
14072
14073 SDValue Fma2 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0, B: Mul,
14074 C: NumeratorScaled, GlueChain: Mul, Flags);
14075
14076 SDValue Fma3 =
14077 getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: Fma2, B: Fma1, C: Mul, GlueChain: Fma2, Flags);
14078
14079 SDValue Fma4 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0, B: Fma3,
14080 C: NumeratorScaled, GlueChain: Fma3, Flags);
14081
14082 if (!PreservesDenormals) {
14083 SDNode *DisableDenorm;
14084 if (!HasDynamicDenormals && Subtarget->hasDenormModeInst()) {
14085 const SDValue DisableDenormValue = getSPDenormModeValue(
14086 FP_DENORM_FLUSH_IN_FLUSH_OUT, DAG, Info, ST: Subtarget);
14087
14088 SDVTList BindParamVTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
14089 DisableDenorm =
14090 DAG.getNode(Opcode: AMDGPUISD::DENORM_MODE, DL: SL, VTList: BindParamVTs,
14091 N1: Fma4.getValue(R: 1), N2: DisableDenormValue, N3: Fma4.getValue(R: 2))
14092 .getNode();
14093 } else {
14094 assert(HasDynamicDenormals == (bool)SavedDenormMode);
14095 const SDValue DisableDenormValue =
14096 HasDynamicDenormals
14097 ? SavedDenormMode
14098 : DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, DL: SL, VT: MVT::i32);
14099
14100 DisableDenorm = DAG.getMachineNode(
14101 Opcode: AMDGPU::S_SETREG_B32, dl: SL, VT: MVT::Other,
14102 Ops: {DisableDenormValue, BitField, Fma4.getValue(R: 1), Fma4.getValue(R: 2)});
14103 }
14104
14105 SDValue OutputChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other,
14106 N1: SDValue(DisableDenorm, 0), N2: DAG.getRoot());
14107 DAG.setRoot(OutputChain);
14108 }
14109
14110 SDValue Scale = NumeratorScaled.getValue(R: 1);
14111 SDValue Fmas = DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL: SL, VT: MVT::f32,
14112 Ops: {Fma4, Fma1, Fma3, Scale}, Flags);
14113
14114 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f32, N1: Fmas, N2: RHS, N3: LHS, Flags);
14115}
14116
14117SDValue SITargetLowering::LowerFDIV64(SDValue Op, SelectionDAG &DAG) const {
14118 if (SDValue FastLowered = lowerFastUnsafeFDIV64(Op, DAG))
14119 return FastLowered;
14120
14121 SDLoc SL(Op);
14122 SDValue X = Op.getOperand(i: 0);
14123 SDValue Y = Op.getOperand(i: 1);
14124
14125 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f64);
14126
14127 SDVTList ScaleVT = DAG.getVTList(VT1: MVT::f64, VT2: MVT::i1);
14128
14129 SDValue DivScale0 = DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, N1: Y, N2: Y, N3: X);
14130
14131 SDValue NegDivScale0 = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f64, Operand: DivScale0);
14132
14133 SDValue Rcp = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f64, Operand: DivScale0);
14134
14135 SDValue Fma0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Rcp, N3: One);
14136
14137 SDValue Fma1 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: Rcp, N2: Fma0, N3: Rcp);
14138
14139 SDValue Fma2 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Fma1, N3: One);
14140
14141 SDValue DivScale1 = DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, N1: X, N2: Y, N3: X);
14142
14143 SDValue Fma3 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: Fma1, N2: Fma2, N3: Fma1);
14144 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f64, N1: DivScale1, N2: Fma3);
14145
14146 SDValue Fma4 =
14147 DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Mul, N3: DivScale1);
14148
14149 SDValue Scale;
14150
14151 if (!Subtarget->hasUsableDivScaleConditionOutput()) {
14152 // Workaround a hardware bug on SI where the condition output from div_scale
14153 // is not usable.
14154
14155 const SDValue Hi = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
14156
14157 // Figure out if the scale to use for div_fmas.
14158 SDValue NumBC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: X);
14159 SDValue DenBC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Y);
14160 SDValue Scale0BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: DivScale0);
14161 SDValue Scale1BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: DivScale1);
14162
14163 SDValue NumHi =
14164 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: NumBC, N2: Hi);
14165 SDValue DenHi =
14166 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: DenBC, N2: Hi);
14167
14168 SDValue Scale0Hi =
14169 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Scale0BC, N2: Hi);
14170 SDValue Scale1Hi =
14171 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Scale1BC, N2: Hi);
14172
14173 SDValue CmpDen = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: DenHi, RHS: Scale0Hi, Cond: ISD::SETEQ);
14174 SDValue CmpNum = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: NumHi, RHS: Scale1Hi, Cond: ISD::SETEQ);
14175 Scale = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i1, N1: CmpNum, N2: CmpDen);
14176 } else {
14177 Scale = DivScale1.getValue(R: 1);
14178 }
14179
14180 SDValue Fmas =
14181 DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL: SL, VT: MVT::f64, N1: Fma4, N2: Fma3, N3: Mul, N4: Scale);
14182
14183 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f64, N1: Fmas, N2: Y, N3: X);
14184}
14185
14186SDValue SITargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const {
14187 EVT VT = Op.getValueType();
14188
14189 if (VT == MVT::f32)
14190 return LowerFDIV32(Op, DAG);
14191
14192 if (VT == MVT::f64)
14193 return LowerFDIV64(Op, DAG);
14194
14195 if (VT == MVT::f16 || VT == MVT::bf16)
14196 return LowerFDIV16(Op, DAG);
14197
14198 llvm_unreachable("Unexpected type for fdiv");
14199}
14200
14201SDValue SITargetLowering::LowerFFREXP(SDValue Op, SelectionDAG &DAG) const {
14202 SDLoc dl(Op);
14203 SDValue Val = Op.getOperand(i: 0);
14204 EVT VT = Val.getValueType();
14205 EVT ResultExpVT = Op->getValueType(ResNo: 1);
14206 EVT InstrExpVT = VT == MVT::f16 ? MVT::i16 : MVT::i32;
14207
14208 SDValue Mant = DAG.getNode(
14209 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT,
14210 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_frexp_mant, DL: dl, VT: MVT::i32), N2: Val);
14211
14212 SDValue Exp = DAG.getNode(
14213 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: InstrExpVT,
14214 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_frexp_exp, DL: dl, VT: MVT::i32), N2: Val);
14215
14216 if (Subtarget->hasFractBug()) {
14217 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Val);
14218 SDValue Inf =
14219 DAG.getConstantFP(Val: APFloat::getInf(Sem: VT.getFltSemantics()), DL: dl, VT);
14220
14221 SDValue IsFinite = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: Fabs, RHS: Inf, Cond: ISD::SETOLT);
14222 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: InstrExpVT);
14223 Exp = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: InstrExpVT, N1: IsFinite, N2: Exp, N3: Zero);
14224 Mant = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: IsFinite, N2: Mant, N3: Val);
14225 }
14226
14227 SDValue CastExp = DAG.getSExtOrTrunc(Op: Exp, DL: dl, VT: ResultExpVT);
14228 return DAG.getMergeValues(Ops: {Mant, CastExp}, dl);
14229}
14230
14231SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
14232 SDLoc DL(Op);
14233 StoreSDNode *Store = cast<StoreSDNode>(Val&: Op);
14234 EVT VT = Store->getMemoryVT();
14235
14236 if (VT == MVT::i1) {
14237 return DAG.getTruncStore(
14238 Chain: Store->getChain(), dl: DL,
14239 Val: DAG.getSExtOrTrunc(Op: Store->getValue(), DL, VT: MVT::i32),
14240 Ptr: Store->getBasePtr(), SVT: MVT::i1, MMO: Store->getMemOperand());
14241 }
14242
14243 assert(VT.isVector() &&
14244 Store->getValue().getValueType().getScalarType() == MVT::i32);
14245
14246 unsigned AS = Store->getAddressSpace();
14247 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
14248 AS == AMDGPUAS::FLAT_ADDRESS &&
14249 Store->getAlign().value() < VT.getStoreSize() &&
14250 VT.getSizeInBits() > 32) {
14251 return SplitVectorStore(Op, DAG);
14252 }
14253
14254 MachineFunction &MF = DAG.getMachineFunction();
14255 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
14256 // If there is a possibility that flat instruction access scratch memory
14257 // then we need to use the same legalization rules we use for private.
14258 if (AS == AMDGPUAS::FLAT_ADDRESS &&
14259 !Subtarget->hasMultiDwordFlatScratchAddressing())
14260 AS = addressMayBeAccessedAsPrivate(MMO: Store->getMemOperand(), Info: *MFI)
14261 ? AMDGPUAS::PRIVATE_ADDRESS
14262 : AMDGPUAS::GLOBAL_ADDRESS;
14263
14264 unsigned NumElements = VT.getVectorNumElements();
14265 if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS) {
14266 if (NumElements > 4)
14267 return SplitVectorStore(Op, DAG);
14268 // v3 stores not supported on SI.
14269 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
14270 return SplitVectorStore(Op, DAG);
14271
14272 if (!allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
14273 VT, MMO: *Store->getMemOperand()))
14274 return expandUnalignedStore(ST: Store, DAG);
14275
14276 return SDValue();
14277 }
14278 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
14279 switch (Subtarget->getMaxPrivateElementSize()) {
14280 case 4:
14281 return scalarizeVectorStore(ST: Store, DAG);
14282 case 8:
14283 if (NumElements > 2)
14284 return SplitVectorStore(Op, DAG);
14285 return SDValue();
14286 case 16:
14287 if (NumElements > 4 ||
14288 (NumElements == 3 && !Subtarget->hasFlatScratchEnabled()))
14289 return SplitVectorStore(Op, DAG);
14290 return SDValue();
14291 default:
14292 llvm_unreachable("unsupported private_element_size");
14293 }
14294 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
14295 unsigned Fast = 0;
14296 auto Flags = Store->getMemOperand()->getFlags();
14297 if (allowsMisalignedMemoryAccessesImpl(Size: VT.getSizeInBits(), AddrSpace: AS,
14298 Alignment: Store->getAlign(), Flags, IsFast: &Fast) &&
14299 Fast > 1)
14300 return SDValue();
14301
14302 if (VT.isVector())
14303 return SplitVectorStore(Op, DAG);
14304
14305 return expandUnalignedStore(ST: Store, DAG);
14306 }
14307
14308 // Probably an invalid store. If so we'll end up emitting a selection error.
14309 return SDValue();
14310}
14311
14312// Avoid the full correct expansion for f32 sqrt when promoting from f16.
14313SDValue SITargetLowering::lowerFSQRTF16(SDValue Op, SelectionDAG &DAG) const {
14314 SDLoc SL(Op);
14315 assert(!Subtarget->has16BitInsts());
14316 SDNodeFlags Flags = Op->getFlags();
14317 SDValue Ext =
14318 DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op.getOperand(i: 0), Flags);
14319
14320 SDValue SqrtID = DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL: SL, VT: MVT::i32);
14321 SDValue Sqrt =
14322 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::f32, N1: SqrtID, N2: Ext, Flags);
14323
14324 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::f16, N1: Sqrt,
14325 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
14326}
14327
14328SDValue SITargetLowering::lowerFSQRTF32(SDValue Op, SelectionDAG &DAG) const {
14329 SDLoc DL(Op);
14330 SDNodeFlags Flags = Op->getFlags();
14331 MVT VT = Op.getValueType().getSimpleVT();
14332 const SDValue X = Op.getOperand(i: 0);
14333
14334 if (allowApproxFunc(DAG, Flags)) {
14335 // Instruction is 1ulp but ignores denormals.
14336 return DAG.getNode(
14337 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT,
14338 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL, VT: MVT::i32), N2: X, Flags);
14339 }
14340
14341 SDValue ScaleThreshold = DAG.getConstantFP(Val: 0x1.0p-96f, DL, VT);
14342 SDValue NeedScale = DAG.getSetCC(DL, VT: MVT::i1, LHS: X, RHS: ScaleThreshold, Cond: ISD::SETOLT);
14343
14344 SDValue ScaleUpFactor = DAG.getConstantFP(Val: 0x1.0p+32f, DL, VT);
14345
14346 SDValue ScaledX = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: X, N2: ScaleUpFactor, Flags);
14347
14348 SDValue SqrtX =
14349 DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: NeedScale, N2: ScaledX, N3: X, Flags);
14350
14351 SDValue SqrtS;
14352 if (needsDenormHandlingF32(DAG, Src: X, Flags)) {
14353 SDValue SqrtID =
14354 DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL, VT: MVT::i32);
14355 SqrtS = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT, N1: SqrtID, N2: SqrtX, Flags);
14356
14357 SDValue SqrtSAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: SqrtS);
14358 SDValue SqrtSNextDownInt =
14359 DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SqrtSAsInt,
14360 N2: DAG.getAllOnesConstant(DL, VT: MVT::i32));
14361 SDValue SqrtSNextDown = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: SqrtSNextDownInt);
14362
14363 SDValue NegSqrtSNextDown =
14364 DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtSNextDown, Flags);
14365
14366 SDValue SqrtVP =
14367 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtSNextDown, N2: SqrtS, N3: SqrtX, Flags);
14368
14369 SDValue SqrtSNextUpInt = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SqrtSAsInt,
14370 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
14371 SDValue SqrtSNextUp = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: SqrtSNextUpInt);
14372
14373 SDValue NegSqrtSNextUp = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtSNextUp, Flags);
14374 SDValue SqrtVS =
14375 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtSNextUp, N2: SqrtS, N3: SqrtX, Flags);
14376
14377 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL, VT);
14378 SDValue SqrtVPLE0 = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtVP, RHS: Zero, Cond: ISD::SETOLE);
14379
14380 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: SqrtVPLE0, N2: SqrtSNextDown, N3: SqrtS,
14381 Flags);
14382
14383 SDValue SqrtVPVSGT0 = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtVS, RHS: Zero, Cond: ISD::SETOGT);
14384 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: SqrtVPVSGT0, N2: SqrtSNextUp, N3: SqrtS,
14385 Flags);
14386 } else {
14387 SDValue SqrtR = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: SqrtX, Flags);
14388
14389 SqrtS = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtX, N2: SqrtR, Flags);
14390
14391 SDValue Half = DAG.getConstantFP(Val: 0.5f, DL, VT);
14392 SDValue SqrtH = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtR, N2: Half, Flags);
14393 SDValue NegSqrtH = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtH, Flags);
14394
14395 SDValue SqrtE = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtH, N2: SqrtS, N3: Half, Flags);
14396 SqrtH = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtH, N2: SqrtE, N3: SqrtH, Flags);
14397 SqrtS = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtS, N2: SqrtE, N3: SqrtS, Flags);
14398
14399 SDValue NegSqrtS = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtS, Flags);
14400 SDValue SqrtD =
14401 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtS, N2: SqrtS, N3: SqrtX, Flags);
14402 SqrtS = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtD, N2: SqrtH, N3: SqrtS, Flags);
14403 }
14404
14405 SDValue ScaleDownFactor = DAG.getConstantFP(Val: 0x1.0p-16f, DL, VT);
14406
14407 SDValue ScaledDown =
14408 DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtS, N2: ScaleDownFactor, Flags);
14409
14410 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: NeedScale, N2: ScaledDown, N3: SqrtS, Flags);
14411 SDValue IsZeroOrInf =
14412 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, N1: SqrtX,
14413 N2: DAG.getTargetConstant(Val: fcZero | fcPosInf, DL, VT: MVT::i32));
14414
14415 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsZeroOrInf, N2: SqrtX, N3: SqrtS, Flags);
14416}
14417
14418SDValue SITargetLowering::lowerFSQRTF64(SDValue Op, SelectionDAG &DAG) const {
14419 // For double type, the SQRT and RSQ instructions don't have required
14420 // precision, we apply Goldschmidt's algorithm to improve the result:
14421 //
14422 // y0 = rsq(x)
14423 // g0 = x * y0
14424 // h0 = 0.5 * y0
14425 //
14426 // r0 = 0.5 - h0 * g0
14427 // g1 = g0 * r0 + g0
14428 // h1 = h0 * r0 + h0
14429 //
14430 // r1 = 0.5 - h1 * g1 => d0 = x - g1 * g1
14431 // g2 = g1 * r1 + g1 g2 = d0 * h1 + g1
14432 // h2 = h1 * r1 + h1
14433 //
14434 // r2 = 0.5 - h2 * g2 => d1 = x - g2 * g2
14435 // g3 = g2 * r2 + g2 g3 = d1 * h1 + g2
14436 //
14437 // sqrt(x) = g3
14438
14439 SDNodeFlags Flags = Op->getFlags();
14440
14441 SDLoc DL(Op);
14442
14443 SDValue X = Op.getOperand(i: 0);
14444 SDValue ZeroInt = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
14445
14446 SDValue SqrtX = X;
14447 SDValue Scaling;
14448 if (!Flags.hasApproximateFuncs()) {
14449 SDValue ScaleConstant = DAG.getConstantFP(Val: 0x1.0p-767, DL, VT: MVT::f64);
14450 Scaling = DAG.getSetCC(DL, VT: MVT::i1, LHS: X, RHS: ScaleConstant, Cond: ISD::SETOLT);
14451
14452 // Scale up input if it is too small.
14453 SDValue ScaleUpFactor = DAG.getConstant(Val: 256, DL, VT: MVT::i32);
14454 SDValue ScaleUp =
14455 DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::i32, N1: Scaling, N2: ScaleUpFactor, N3: ZeroInt);
14456 SqrtX = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: X, N2: ScaleUp, Flags);
14457 }
14458
14459 SDValue SqrtY = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT: MVT::f64, Operand: SqrtX);
14460
14461 SDValue SqrtS0 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: SqrtX, N2: SqrtY);
14462
14463 SDValue Half = DAG.getConstantFP(Val: 0.5, DL, VT: MVT::f64);
14464 SDValue SqrtH0 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: SqrtY, N2: Half);
14465
14466 SDValue NegSqrtH0 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtH0);
14467 SDValue SqrtR0 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtH0, N2: SqrtS0, N3: Half);
14468
14469 SDValue SqrtH1 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtH0, N2: SqrtR0, N3: SqrtH0);
14470
14471 SDValue SqrtS1 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtS0, N2: SqrtR0, N3: SqrtS0);
14472
14473 SDValue NegSqrtS1 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtS1);
14474 SDValue SqrtD0 =
14475 DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtS1, N2: SqrtS1, N3: SqrtX);
14476
14477 SDValue SqrtS2 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtD0, N2: SqrtH1, N3: SqrtS1);
14478
14479 SDValue SqrtRet = SqrtS2;
14480 if (!Flags.hasApproximateFuncs()) {
14481 SDValue NegSqrtS2 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtS2);
14482 SDValue SqrtD1 =
14483 DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtS2, N2: SqrtS2, N3: SqrtX);
14484
14485 SqrtRet = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtD1, N2: SqrtH1, N3: SqrtS2);
14486
14487 SDValue ScaleDownFactor = DAG.getSignedConstant(Val: -128, DL, VT: MVT::i32);
14488 SDValue ScaleDown = DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::i32, N1: Scaling,
14489 N2: ScaleDownFactor, N3: ZeroInt);
14490 SqrtRet = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: SqrtRet, N2: ScaleDown, Flags);
14491 }
14492
14493 // TODO: Check for DAZ and expand to subnormals
14494
14495 SDValue IsZeroOrInf;
14496 if (Flags.hasNoInfs()) {
14497 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: MVT::f64);
14498 IsZeroOrInf = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtX, RHS: Zero, Cond: ISD::SETOEQ);
14499 } else {
14500 IsZeroOrInf =
14501 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, N1: SqrtX,
14502 N2: DAG.getTargetConstant(Val: fcZero | fcPosInf, DL, VT: MVT::i32));
14503 }
14504
14505 // If x is +INF, +0, or -0, use its original value
14506 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::f64, N1: IsZeroOrInf, N2: SqrtX, N3: SqrtRet,
14507 Flags);
14508}
14509
14510SDValue SITargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
14511 SDLoc DL(Op);
14512 EVT VT = Op.getValueType();
14513 SDValue Arg = Op.getOperand(i: 0);
14514 SDValue TrigVal;
14515
14516 // Propagate fast-math flags so that the multiply we introduce can be folded
14517 // if Arg is already the result of a multiply by constant.
14518 auto Flags = Op->getFlags();
14519
14520 // AMDGPUISD nodes of vector type must be unrolled here since
14521 // they will not be expanded elsewhere.
14522 auto UnrollIfVec = [&DAG](SDValue V) -> SDValue {
14523 if (!V.getValueType().isVector())
14524 return V;
14525
14526 return DAG.UnrollVectorOp(N: cast<SDNode>(Val&: V));
14527 };
14528
14529 SDValue OneOver2Pi = DAG.getConstantFP(Val: 0.5 * numbers::inv_pi, DL, VT);
14530
14531 if (Subtarget->hasTrigReducedRange()) {
14532 SDValue MulVal = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Arg, N2: OneOver2Pi, Flags);
14533 TrigVal = UnrollIfVec(DAG.getNode(Opcode: AMDGPUISD::FRACT, DL, VT, Operand: MulVal, Flags));
14534 } else {
14535 TrigVal = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Arg, N2: OneOver2Pi, Flags);
14536 }
14537
14538 switch (Op.getOpcode()) {
14539 case ISD::FCOS:
14540 TrigVal = DAG.getNode(Opcode: AMDGPUISD::COS_HW, DL: SDLoc(Op), VT, Operand: TrigVal, Flags);
14541 break;
14542 case ISD::FSIN:
14543 TrigVal = DAG.getNode(Opcode: AMDGPUISD::SIN_HW, DL: SDLoc(Op), VT, Operand: TrigVal, Flags);
14544 break;
14545 default:
14546 llvm_unreachable("Wrong trig opcode");
14547 }
14548
14549 return UnrollIfVec(TrigVal);
14550}
14551
14552SDValue SITargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op,
14553 SelectionDAG &DAG) const {
14554 AtomicSDNode *AtomicNode = cast<AtomicSDNode>(Val&: Op);
14555 assert(AtomicNode->isCompareAndSwap());
14556 unsigned AS = AtomicNode->getAddressSpace();
14557
14558 // No custom lowering required for local address space
14559 if (!AMDGPU::isFlatGlobalAddrSpace(AS))
14560 return Op;
14561
14562 // Non-local address space requires custom lowering for atomic compare
14563 // and swap; cmp and swap should be in a v2i32 or v2i64 in case of _X2
14564 SDLoc DL(Op);
14565 SDValue ChainIn = Op.getOperand(i: 0);
14566 SDValue Addr = Op.getOperand(i: 1);
14567 SDValue Old = Op.getOperand(i: 2);
14568 SDValue New = Op.getOperand(i: 3);
14569 EVT VT = Op.getValueType();
14570 MVT SimpleVT = VT.getSimpleVT();
14571 MVT VecType = MVT::getVectorVT(VT: SimpleVT, NumElements: 2);
14572
14573 SDValue NewOld = DAG.getBuildVector(VT: VecType, DL, Ops: {New, Old});
14574 SDValue Ops[] = {ChainIn, Addr, NewOld};
14575
14576 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::ATOMIC_CMP_SWAP, dl: DL,
14577 VTList: Op->getVTList(), Ops, MemVT: VT,
14578 MMO: AtomicNode->getMemOperand());
14579}
14580
14581//===----------------------------------------------------------------------===//
14582// Custom DAG optimizations
14583//===----------------------------------------------------------------------===//
14584
14585SDValue
14586SITargetLowering::performUCharToFloatCombine(SDNode *N,
14587 DAGCombinerInfo &DCI) const {
14588 EVT VT = N->getValueType(ResNo: 0);
14589 EVT ScalarVT = VT.getScalarType();
14590 if (ScalarVT != MVT::f32 && ScalarVT != MVT::f16)
14591 return SDValue();
14592
14593 SelectionDAG &DAG = DCI.DAG;
14594 SDLoc DL(N);
14595
14596 SDValue Src = N->getOperand(Num: 0);
14597 EVT SrcVT = Src.getValueType();
14598
14599 // TODO: We could try to match extracting the higher bytes, which would be
14600 // easier if i8 vectors weren't promoted to i32 vectors, particularly after
14601 // types are legalized. v4i8 -> v4f32 is probably the only case to worry
14602 // about in practice.
14603 if (DCI.isAfterLegalizeDAG() && SrcVT == MVT::i32) {
14604 if (DAG.MaskedValueIsZero(Op: Src, Mask: APInt::getHighBitsSet(numBits: 32, hiBitsSet: 24))) {
14605 SDValue Cvt = DAG.getNode(Opcode: AMDGPUISD::CVT_F32_UBYTE0, DL, VT: MVT::f32, Operand: Src);
14606 DCI.AddToWorklist(N: Cvt.getNode());
14607
14608 // For the f16 case, fold to a cast to f32 and then cast back to f16.
14609 if (ScalarVT != MVT::f32) {
14610 Cvt = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Cvt,
14611 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
14612 }
14613 return Cvt;
14614 }
14615 }
14616
14617 return SDValue();
14618}
14619
14620SDValue SITargetLowering::performFCopySignCombine(SDNode *N,
14621 DAGCombinerInfo &DCI) const {
14622 SDValue MagnitudeOp = N->getOperand(Num: 0);
14623 SDValue SignOp = N->getOperand(Num: 1);
14624
14625 // The generic combine for fcopysign + fp cast is too conservative with
14626 // vectors, and also gets confused by the splitting we will perform here, so
14627 // peek through FP casts.
14628 if (SignOp.getOpcode() == ISD::FP_EXTEND ||
14629 SignOp.getOpcode() == ISD::FP_ROUND)
14630 SignOp = SignOp.getOperand(i: 0);
14631
14632 SelectionDAG &DAG = DCI.DAG;
14633 SDLoc DL(N);
14634 EVT SignVT = SignOp.getValueType();
14635
14636 // f64 fcopysign is really an f32 copysign on the high bits, so replace the
14637 // lower half with a copy.
14638 // fcopysign f64:x, _:y -> x.lo32, (fcopysign (f32 x.hi32), _:y)
14639 EVT MagVT = MagnitudeOp.getValueType();
14640
14641 unsigned NumElts = MagVT.isVector() ? MagVT.getVectorNumElements() : 1;
14642
14643 if (MagVT.getScalarType() == MVT::f64) {
14644 EVT F32VT = MagVT.isVector()
14645 ? EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: 2 * NumElts)
14646 : MVT::v2f32;
14647
14648 SDValue MagAsVector = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: F32VT, Operand: MagnitudeOp);
14649
14650 SmallVector<SDValue, 8> NewElts;
14651 for (unsigned I = 0; I != NumElts; ++I) {
14652 SDValue MagLo =
14653 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: MagAsVector,
14654 N2: DAG.getConstant(Val: 2 * I, DL, VT: MVT::i32));
14655 SDValue MagHi =
14656 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: MagAsVector,
14657 N2: DAG.getConstant(Val: 2 * I + 1, DL, VT: MVT::i32));
14658
14659 SDValue SignOpElt =
14660 MagVT.isVector()
14661 ? DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: SignVT.getScalarType(),
14662 N1: SignOp, N2: DAG.getConstant(Val: I, DL, VT: MVT::i32))
14663 : SignOp;
14664
14665 SDValue HiOp =
14666 DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: MVT::f32, N1: MagHi, N2: SignOpElt);
14667
14668 SDValue Vector =
14669 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MVT::v2f32, N1: MagLo, N2: HiOp);
14670
14671 SDValue NewElt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: Vector);
14672 NewElts.push_back(Elt: NewElt);
14673 }
14674
14675 if (NewElts.size() == 1)
14676 return NewElts[0];
14677
14678 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MagVT, Ops: NewElts);
14679 }
14680
14681 if (SignVT.getScalarType() != MVT::f64)
14682 return SDValue();
14683
14684 // Reduce width of sign operand, we only need the highest bit.
14685 //
14686 // fcopysign f64:x, f64:y ->
14687 // fcopysign f64:x, (extract_vector_elt (bitcast f64:y to v2f32), 1)
14688 // TODO: In some cases it might make sense to go all the way to f16.
14689
14690 EVT F32VT = MagVT.isVector()
14691 ? EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: 2 * NumElts)
14692 : MVT::v2f32;
14693
14694 SDValue SignAsVector = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: F32VT, Operand: SignOp);
14695
14696 SmallVector<SDValue, 8> F32Signs;
14697 for (unsigned I = 0; I != NumElts; ++I) {
14698 // Take sign from odd elements of cast vector
14699 SDValue SignAsF32 =
14700 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: SignAsVector,
14701 N2: DAG.getConstant(Val: 2 * I + 1, DL, VT: MVT::i32));
14702 F32Signs.push_back(Elt: SignAsF32);
14703 }
14704
14705 SDValue NewSign =
14706 NumElts == 1
14707 ? F32Signs.back()
14708 : DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL,
14709 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: NumElts),
14710 Ops: F32Signs);
14711
14712 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
14713 N2: NewSign);
14714}
14715
14716// (shl (add x, c1), c2) -> add (shl x, c2), (shl c1, c2)
14717// (shl (or x, c1), c2) -> add (shl x, c2), (shl c1, c2) iff x and c1 share no
14718// bits
14719
14720// This is a variant of
14721// (mul (add x, c1), c2) -> add (mul x, c2), (mul c1, c2),
14722//
14723// The normal DAG combiner will do this, but only if the add has one use since
14724// that would increase the number of instructions.
14725//
14726// This prevents us from seeing a constant offset that can be folded into a
14727// memory instruction's addressing mode. If we know the resulting add offset of
14728// a pointer can be folded into an addressing offset, we can replace the pointer
14729// operand with the add of new constant offset. This eliminates one of the uses,
14730// and may allow the remaining use to also be simplified.
14731//
14732SDValue SITargetLowering::performSHLPtrCombine(SDNode *N, unsigned AddrSpace,
14733 EVT MemVT,
14734 DAGCombinerInfo &DCI) const {
14735 SDValue N0 = N->getOperand(Num: 0);
14736 SDValue N1 = N->getOperand(Num: 1);
14737
14738 // We only do this to handle cases where it's profitable when there are
14739 // multiple uses of the add, so defer to the standard combine.
14740 if ((!N0->isAnyAdd() && N0.getOpcode() != ISD::OR) || N0->hasOneUse())
14741 return SDValue();
14742
14743 const ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(Val&: N1);
14744 if (!CN1)
14745 return SDValue();
14746
14747 const ConstantSDNode *CAdd = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
14748 if (!CAdd)
14749 return SDValue();
14750
14751 SelectionDAG &DAG = DCI.DAG;
14752
14753 if (N0->getOpcode() == ISD::OR &&
14754 !DAG.haveNoCommonBitsSet(A: N0.getOperand(i: 0), B: N0.getOperand(i: 1)))
14755 return SDValue();
14756
14757 // If the resulting offset is too large, we can't fold it into the
14758 // addressing mode offset.
14759 APInt Offset = CAdd->getAPIntValue() << CN1->getAPIntValue();
14760 Type *Ty = MemVT.getTypeForEVT(Context&: *DCI.DAG.getContext());
14761
14762 AddrMode AM;
14763 AM.HasBaseReg = true;
14764 AM.BaseOffs = Offset.getSExtValue();
14765 if (!isLegalAddressingMode(DL: DCI.DAG.getDataLayout(), AM, Ty, AS: AddrSpace))
14766 return SDValue();
14767
14768 SDLoc SL(N);
14769 EVT VT = N->getValueType(ResNo: 0);
14770
14771 SDValue ShlX = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT, N1: N0.getOperand(i: 0), N2: N1);
14772 SDValue COffset = DAG.getConstant(Val: Offset, DL: SL, VT);
14773
14774 SDNodeFlags Flags;
14775 Flags.setNoUnsignedWrap(
14776 N->getFlags().hasNoUnsignedWrap() &&
14777 (N0.getOpcode() == ISD::OR || N0->getFlags().hasNoUnsignedWrap()));
14778
14779 // Use ISD::ADD even if the original operation was ISD::PTRADD, since we can't
14780 // be sure that the new left operand is a proper base pointer.
14781 return DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ShlX, N2: COffset, Flags);
14782}
14783
14784/// MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset
14785/// by the chain and intrinsic ID. Theoretically we would also need to check the
14786/// specific intrinsic, but they all place the pointer operand first.
14787static unsigned getBasePtrIndex(const MemSDNode *N) {
14788 switch (N->getOpcode()) {
14789 case ISD::STORE:
14790 case ISD::INTRINSIC_W_CHAIN:
14791 case ISD::INTRINSIC_VOID:
14792 return 2;
14793 default:
14794 return 1;
14795 }
14796}
14797
14798SDValue SITargetLowering::performMemSDNodeCombine(MemSDNode *N,
14799 DAGCombinerInfo &DCI) const {
14800 SelectionDAG &DAG = DCI.DAG;
14801
14802 unsigned PtrIdx = getBasePtrIndex(N);
14803 SDValue Ptr = N->getOperand(Num: PtrIdx);
14804
14805 // TODO: We could also do this for multiplies.
14806 if (Ptr.getOpcode() == ISD::SHL) {
14807 SDValue NewPtr = performSHLPtrCombine(N: Ptr.getNode(), AddrSpace: N->getAddressSpace(),
14808 MemVT: N->getMemoryVT(), DCI);
14809 if (NewPtr) {
14810 SmallVector<SDValue, 8> NewOps(N->ops());
14811
14812 NewOps[PtrIdx] = NewPtr;
14813 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
14814 }
14815 }
14816
14817 return SDValue();
14818}
14819
14820static bool bitOpWithConstantIsReducible(unsigned Opc, uint32_t Val) {
14821 return (Opc == ISD::AND && (Val == 0 || Val == 0xffffffff)) ||
14822 (Opc == ISD::OR && (Val == 0xffffffff || Val == 0)) ||
14823 (Opc == ISD::XOR && Val == 0);
14824}
14825
14826// Break up 64-bit bit operation of a constant into two 32-bit and/or/xor. This
14827// will typically happen anyway for a VALU 64-bit and. This exposes other 32-bit
14828// integer combine opportunities since most 64-bit operations are decomposed
14829// this way. TODO: We won't want this for SALU especially if it is an inline
14830// immediate.
14831SDValue SITargetLowering::splitBinaryBitConstantOp(
14832 DAGCombinerInfo &DCI, const SDLoc &SL, unsigned Opc, SDValue LHS,
14833 const ConstantSDNode *CRHS) const {
14834 uint64_t Val = CRHS->getZExtValue();
14835 uint32_t ValLo = Lo_32(Value: Val);
14836 uint32_t ValHi = Hi_32(Value: Val);
14837 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
14838
14839 if ((bitOpWithConstantIsReducible(Opc, Val: ValLo) ||
14840 bitOpWithConstantIsReducible(Opc, Val: ValHi)) ||
14841 (CRHS->hasOneUse() && !TII->isInlineConstant(Imm: CRHS->getAPIntValue()))) {
14842 // We have 64-bit scalar and/or/xor, but do not have vector forms.
14843 if (Subtarget->has64BitLiterals() && CRHS->hasOneUse() &&
14844 !CRHS->user_begin()->isDivergent())
14845 return SDValue();
14846
14847 // If we need to materialize a 64-bit immediate, it will be split up later
14848 // anyway. Avoid creating the harder to understand 64-bit immediate
14849 // materialization.
14850 return splitBinaryBitConstantOpImpl(DCI, SL, Opc, LHS, ValLo, ValHi);
14851 }
14852
14853 return SDValue();
14854}
14855
14856bool llvm::isBoolSGPR(SDValue V) {
14857 if (V.getValueType() != MVT::i1)
14858 return false;
14859 switch (V.getOpcode()) {
14860 default:
14861 break;
14862 case ISD::SETCC:
14863 case ISD::IS_FPCLASS:
14864 case AMDGPUISD::FP_CLASS:
14865 return true;
14866 case ISD::AND:
14867 case ISD::OR:
14868 case ISD::XOR:
14869 return isBoolSGPR(V: V.getOperand(i: 0)) && isBoolSGPR(V: V.getOperand(i: 1));
14870 case ISD::SADDO:
14871 case ISD::UADDO:
14872 case ISD::SSUBO:
14873 case ISD::USUBO:
14874 case ISD::SMULO:
14875 case ISD::UMULO:
14876 return V.getResNo() == 1;
14877 case ISD::INTRINSIC_WO_CHAIN: {
14878 unsigned IntrinsicID = V.getConstantOperandVal(i: 0);
14879 switch (IntrinsicID) {
14880 case Intrinsic::amdgcn_is_shared:
14881 case Intrinsic::amdgcn_is_private:
14882 return true;
14883 default:
14884 return false;
14885 }
14886
14887 return false;
14888 }
14889 }
14890 return false;
14891}
14892
14893// If a constant has all zeroes or all ones within each byte return it.
14894// Otherwise return 0.
14895static uint32_t getConstantPermuteMask(uint32_t C) {
14896 // 0xff for any zero byte in the mask
14897 uint32_t ZeroByteMask = 0;
14898 if (!(C & 0x000000ff))
14899 ZeroByteMask |= 0x000000ff;
14900 if (!(C & 0x0000ff00))
14901 ZeroByteMask |= 0x0000ff00;
14902 if (!(C & 0x00ff0000))
14903 ZeroByteMask |= 0x00ff0000;
14904 if (!(C & 0xff000000))
14905 ZeroByteMask |= 0xff000000;
14906 uint32_t NonZeroByteMask = ~ZeroByteMask; // 0xff for any non-zero byte
14907 if ((NonZeroByteMask & C) != NonZeroByteMask)
14908 return 0; // Partial bytes selected.
14909 return C;
14910}
14911
14912// Check if a node selects whole bytes from its operand 0 starting at a byte
14913// boundary while masking the rest. Returns select mask as in the v_perm_b32
14914// or -1 if not succeeded.
14915// Note byte select encoding:
14916// value 0-3 selects corresponding source byte;
14917// value 0xc selects zero;
14918// value 0xff selects 0xff.
14919static uint32_t getPermuteMask(SDValue V) {
14920 assert(V.getValueSizeInBits() == 32);
14921
14922 if (V.getNumOperands() != 2)
14923 return ~0;
14924
14925 ConstantSDNode *N1 = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1));
14926 if (!N1)
14927 return ~0;
14928
14929 uint32_t C = N1->getZExtValue();
14930
14931 switch (V.getOpcode()) {
14932 default:
14933 break;
14934 case ISD::AND:
14935 if (uint32_t ConstMask = getConstantPermuteMask(C))
14936 return (0x03020100 & ConstMask) | (0x0c0c0c0c & ~ConstMask);
14937 break;
14938
14939 case ISD::OR:
14940 if (uint32_t ConstMask = getConstantPermuteMask(C))
14941 return (0x03020100 & ~ConstMask) | ConstMask;
14942 break;
14943
14944 case ISD::SHL:
14945 if (C % 8)
14946 return ~0;
14947
14948 return uint32_t((0x030201000c0c0c0cull << C) >> 32);
14949
14950 case ISD::SRL:
14951 if (C % 8)
14952 return ~0;
14953
14954 return uint32_t(0x0c0c0c0c03020100ull >> C);
14955 }
14956
14957 return ~0;
14958}
14959
14960SDValue SITargetLowering::performAndCombine(SDNode *N,
14961 DAGCombinerInfo &DCI) const {
14962 if (DCI.isBeforeLegalize())
14963 return SDValue();
14964
14965 SelectionDAG &DAG = DCI.DAG;
14966 EVT VT = N->getValueType(ResNo: 0);
14967 SDValue LHS = N->getOperand(Num: 0);
14968 SDValue RHS = N->getOperand(Num: 1);
14969
14970 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
14971 if (VT == MVT::i64 && CRHS) {
14972 if (SDValue Split =
14973 splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::AND, LHS, CRHS))
14974 return Split;
14975 }
14976
14977 if (CRHS && VT == MVT::i32) {
14978 // and (srl x, c), mask => shl (bfe x, nb + c, mask >> nb), nb
14979 // nb = number of trailing zeroes in mask
14980 // It can be optimized out using SDWA for GFX8+ in the SDWA peephole pass,
14981 // given that we are selecting 8 or 16 bit fields starting at byte boundary.
14982 uint64_t Mask = CRHS->getZExtValue();
14983 unsigned Bits = llvm::popcount(Value: Mask);
14984 if (getSubtarget()->hasSDWA() && LHS->getOpcode() == ISD::SRL &&
14985 (Bits == 8 || Bits == 16) && isShiftedMask_64(Value: Mask) && !(Mask & 1)) {
14986 if (auto *CShift = dyn_cast<ConstantSDNode>(Val: LHS->getOperand(Num: 1))) {
14987 unsigned Shift = CShift->getZExtValue();
14988 unsigned NB = CRHS->getAPIntValue().countr_zero();
14989 unsigned Offset = NB + Shift;
14990 if ((Offset & (Bits - 1)) == 0) { // Starts at a byte or word boundary.
14991 SDLoc SL(N);
14992 SDValue BFE =
14993 DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT: MVT::i32, N1: LHS->getOperand(Num: 0),
14994 N2: DAG.getConstant(Val: Offset, DL: SL, VT: MVT::i32),
14995 N3: DAG.getConstant(Val: Bits, DL: SL, VT: MVT::i32));
14996 EVT NarrowVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: Bits);
14997 SDValue Ext = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT, N1: BFE,
14998 N2: DAG.getValueType(NarrowVT));
14999 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(LHS), VT, N1: Ext,
15000 N2: DAG.getConstant(Val: NB, DL: SDLoc(CRHS), VT: MVT::i32));
15001 return Shl;
15002 }
15003 }
15004 }
15005
15006 // and (perm x, y, c1), c2 -> perm x, y, permute_mask(c1, c2)
15007 if (LHS.hasOneUse() && LHS.getOpcode() == AMDGPUISD::PERM &&
15008 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2))) {
15009 uint32_t Sel = getConstantPermuteMask(C: Mask);
15010 if (!Sel)
15011 return SDValue();
15012
15013 // Select 0xc for all zero bytes
15014 Sel = (LHS.getConstantOperandVal(i: 2) & Sel) | (~Sel & 0x0c0c0c0c);
15015 SDLoc DL(N);
15016 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15017 N2: LHS.getOperand(i: 1), N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15018 }
15019 }
15020
15021 // (and (fcmp ord x, x), (fcmp une (fabs x), inf)) ->
15022 // fp_class x, ~(s_nan | q_nan | n_infinity | p_infinity)
15023 if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == ISD::SETCC) {
15024 ISD::CondCode LCC = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
15025 ISD::CondCode RCC = cast<CondCodeSDNode>(Val: RHS.getOperand(i: 2))->get();
15026
15027 SDValue X = LHS.getOperand(i: 0);
15028 SDValue Y = RHS.getOperand(i: 0);
15029 if (Y.getOpcode() != ISD::FABS || Y.getOperand(i: 0) != X ||
15030 !isTypeLegal(VT: X.getValueType()))
15031 return SDValue();
15032
15033 if (LCC == ISD::SETO) {
15034 if (X != LHS.getOperand(i: 1))
15035 return SDValue();
15036
15037 if (RCC == ISD::SETUNE) {
15038 const ConstantFPSDNode *C1 =
15039 dyn_cast<ConstantFPSDNode>(Val: RHS.getOperand(i: 1));
15040 if (!C1 || !C1->isInfinity() || C1->isNegative())
15041 return SDValue();
15042
15043 const uint32_t Mask = SIInstrFlags::N_NORMAL |
15044 SIInstrFlags::N_SUBNORMAL | SIInstrFlags::N_ZERO |
15045 SIInstrFlags::P_ZERO | SIInstrFlags::P_SUBNORMAL |
15046 SIInstrFlags::P_NORMAL;
15047
15048 static_assert(
15049 ((~(SIInstrFlags::S_NAN | SIInstrFlags::Q_NAN |
15050 SIInstrFlags::N_INFINITY | SIInstrFlags::P_INFINITY)) &
15051 0x3ff) == Mask,
15052 "mask not equal");
15053
15054 SDLoc DL(N);
15055 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: X,
15056 N2: DAG.getConstant(Val: Mask, DL, VT: MVT::i32));
15057 }
15058 }
15059 }
15060
15061 if (RHS.getOpcode() == ISD::SETCC && LHS.getOpcode() == AMDGPUISD::FP_CLASS)
15062 std::swap(a&: LHS, b&: RHS);
15063
15064 if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == AMDGPUISD::FP_CLASS &&
15065 RHS.hasOneUse()) {
15066 ISD::CondCode LCC = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
15067 // and (fcmp seto), (fp_class x, mask) -> fp_class x, mask & ~(p_nan |
15068 // n_nan) and (fcmp setuo), (fp_class x, mask) -> fp_class x, mask & (p_nan
15069 // | n_nan)
15070 const ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i: 1));
15071 if ((LCC == ISD::SETO || LCC == ISD::SETUO) && Mask &&
15072 (RHS.getOperand(i: 0) == LHS.getOperand(i: 0) &&
15073 LHS.getOperand(i: 0) == LHS.getOperand(i: 1))) {
15074 const unsigned OrdMask = SIInstrFlags::S_NAN | SIInstrFlags::Q_NAN;
15075 unsigned NewMask = LCC == ISD::SETO ? Mask->getZExtValue() & ~OrdMask
15076 : Mask->getZExtValue() & OrdMask;
15077
15078 SDLoc DL(N);
15079 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: RHS.getOperand(i: 0),
15080 N2: DAG.getConstant(Val: NewMask, DL, VT: MVT::i32));
15081 }
15082 }
15083
15084 if (VT == MVT::i32 && (RHS.getOpcode() == ISD::SIGN_EXTEND ||
15085 LHS.getOpcode() == ISD::SIGN_EXTEND)) {
15086 // and x, (sext cc from i1) => select cc, x, 0
15087 if (RHS.getOpcode() != ISD::SIGN_EXTEND)
15088 std::swap(a&: LHS, b&: RHS);
15089 if (isBoolSGPR(V: RHS.getOperand(i: 0)))
15090 return DAG.getSelect(DL: SDLoc(N), VT: MVT::i32, Cond: RHS.getOperand(i: 0), LHS,
15091 RHS: DAG.getConstant(Val: 0, DL: SDLoc(N), VT: MVT::i32));
15092 }
15093
15094 // and (op x, c1), (op y, c2) -> perm x, y, permute_mask(c1, c2)
15095 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
15096 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() &&
15097 TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
15098 uint32_t LHSMask = getPermuteMask(V: LHS);
15099 uint32_t RHSMask = getPermuteMask(V: RHS);
15100 if (LHSMask != ~0u && RHSMask != ~0u) {
15101 // Canonicalize the expression in an attempt to have fewer unique masks
15102 // and therefore fewer registers used to hold the masks.
15103 if (LHSMask > RHSMask) {
15104 std::swap(a&: LHSMask, b&: RHSMask);
15105 std::swap(a&: LHS, b&: RHS);
15106 }
15107
15108 // Select 0xc for each lane used from source operand. Zero has 0xc mask
15109 // set, 0xff have 0xff in the mask, actual lanes are in the 0-3 range.
15110 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15111 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15112
15113 // Check of we need to combine values from two sources within a byte.
15114 if (!(LHSUsedLanes & RHSUsedLanes) &&
15115 // If we select high and lower word keep it for SDWA.
15116 // TODO: teach SDWA to work with v_perm_b32 and remove the check.
15117 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
15118 // Each byte in each mask is either selector mask 0-3, or has higher
15119 // bits set in either of masks, which can be 0xff for 0xff or 0x0c for
15120 // zero. If 0x0c is in either mask it shall always be 0x0c. Otherwise
15121 // mask which is not 0xff wins. By anding both masks we have a correct
15122 // result except that 0x0c shall be corrected to give 0x0c only.
15123 uint32_t Mask = LHSMask & RHSMask;
15124 for (unsigned I = 0; I < 32; I += 8) {
15125 uint32_t ByteSel = 0xff << I;
15126 if ((LHSMask & ByteSel) == 0x0c || (RHSMask & ByteSel) == 0x0c)
15127 Mask &= (0x0c << I) & 0xffffffff;
15128 }
15129
15130 // Add 4 to each active LHS lane. It will not affect any existing 0xff
15131 // or 0x0c.
15132 uint32_t Sel = Mask | (LHSUsedLanes & 0x04040404);
15133 SDLoc DL(N);
15134
15135 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15136 N2: RHS.getOperand(i: 0),
15137 N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15138 }
15139 }
15140 }
15141
15142 return SDValue();
15143}
15144
15145// A key component of v_perm is a mapping between byte position of the src
15146// operands, and the byte position of the dest. To provide such, we need: 1. the
15147// node that provides x byte of the dest of the OR, and 2. the byte of the node
15148// used to provide that x byte. calculateByteProvider finds which node provides
15149// a certain byte of the dest of the OR, and calculateSrcByte takes that node,
15150// and finds an ultimate src and byte position For example: The supported
15151// LoadCombine pattern for vector loads is as follows
15152// t1
15153// or
15154// / \
15155// t2 t3
15156// zext shl
15157// | | \
15158// t4 t5 16
15159// or anyext
15160// / \ |
15161// t6 t7 t8
15162// srl shl or
15163// / | / \ / \
15164// t9 t10 t11 t12 t13 t14
15165// trunc* 8 trunc* 8 and and
15166// | | / | | \
15167// t15 t16 t17 t18 t19 t20
15168// trunc* 255 srl -256
15169// | / \
15170// t15 t15 16
15171//
15172// *In this example, the truncs are from i32->i16
15173//
15174// calculateByteProvider would find t6, t7, t13, and t14 for bytes 0-3
15175// respectively. calculateSrcByte would find (given node) -> ultimate src &
15176// byteposition: t6 -> t15 & 1, t7 -> t16 & 0, t13 -> t15 & 0, t14 -> t15 & 3.
15177// After finding the mapping, we can combine the tree into vperm t15, t16,
15178// 0x05000407
15179
15180// Find the source and byte position from a node.
15181// \p DestByte is the byte position of the dest of the or that the src
15182// ultimately provides. \p SrcIndex is the byte of the src that maps to this
15183// dest of the or byte. \p Depth tracks how many recursive iterations we have
15184// performed.
15185static const std::optional<ByteProvider<SDValue>>
15186calculateSrcByte(const SDValue Op, uint64_t DestByte, uint64_t SrcIndex = 0,
15187 unsigned Depth = 0) {
15188 // We may need to recursively traverse a series of SRLs
15189 if (Depth >= 6)
15190 return std::nullopt;
15191
15192 if (Op.getValueSizeInBits() < 8)
15193 return std::nullopt;
15194
15195 if (Op.getValueType().isVector())
15196 return ByteProvider<SDValue>::getSrc(Val: Op, ByteOffset: DestByte, VectorOffset: SrcIndex);
15197
15198 switch (Op->getOpcode()) {
15199 case ISD::TRUNCATE: {
15200 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex, Depth: Depth + 1);
15201 }
15202
15203 case ISD::ANY_EXTEND:
15204 case ISD::SIGN_EXTEND:
15205 case ISD::ZERO_EXTEND:
15206 case ISD::SIGN_EXTEND_INREG: {
15207 SDValue NarrowOp = Op->getOperand(Num: 0);
15208 auto NarrowVT = NarrowOp.getValueType();
15209 if (Op->getOpcode() == ISD::SIGN_EXTEND_INREG) {
15210 auto *VTSign = cast<VTSDNode>(Val: Op->getOperand(Num: 1));
15211 NarrowVT = VTSign->getVT();
15212 }
15213 if (!NarrowVT.isByteSized())
15214 return std::nullopt;
15215 uint64_t NarrowByteWidth = NarrowVT.getStoreSize();
15216
15217 if (SrcIndex >= NarrowByteWidth)
15218 return std::nullopt;
15219 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex, Depth: Depth + 1);
15220 }
15221
15222 case ISD::SRA:
15223 case ISD::SRL: {
15224 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15225 if (!ShiftOp)
15226 return std::nullopt;
15227
15228 uint64_t BitShift = ShiftOp->getZExtValue();
15229
15230 if (BitShift % 8 != 0)
15231 return std::nullopt;
15232
15233 uint64_t NewSrcIndex = SrcIndex + BitShift / 8;
15234 if (NewSrcIndex >= Op.getScalarValueSizeInBits() / 8)
15235 return std::nullopt;
15236
15237 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex: NewSrcIndex,
15238 Depth: Depth + 1);
15239 }
15240
15241 default: {
15242 return ByteProvider<SDValue>::getSrc(Val: Op, ByteOffset: DestByte, VectorOffset: SrcIndex);
15243 }
15244 }
15245 llvm_unreachable("fully handled switch");
15246}
15247
15248// For a byte position in the result of an Or, traverse the tree and find the
15249// node (and the byte of the node) which ultimately provides this {Or,
15250// BytePosition}. \p Op is the operand we are currently examining. \p Index is
15251// the byte position of the Op that corresponds with the originally requested
15252// byte of the Or \p Depth tracks how many recursive iterations we have
15253// performed. \p StartingIndex is the originally requested byte of the Or
15254static const std::optional<ByteProvider<SDValue>>
15255calculateByteProvider(const SDValue &Op, unsigned Index, unsigned Depth,
15256 unsigned StartingIndex = 0) {
15257 // Finding Src tree of RHS of or typically requires at least 1 additional
15258 // depth
15259 if (Depth > 6)
15260 return std::nullopt;
15261
15262 unsigned BitWidth = Op.getScalarValueSizeInBits();
15263 if (BitWidth % 8 != 0)
15264 return std::nullopt;
15265 if (Index > BitWidth / 8 - 1)
15266 return std::nullopt;
15267
15268 bool IsVec = Op.getValueType().isVector();
15269 switch (Op.getOpcode()) {
15270 case ISD::OR: {
15271 if (IsVec)
15272 return std::nullopt;
15273
15274 auto RHS = calculateByteProvider(Op: Op.getOperand(i: 1), Index, Depth: Depth + 1,
15275 StartingIndex);
15276 if (!RHS)
15277 return std::nullopt;
15278 auto LHS = calculateByteProvider(Op: Op.getOperand(i: 0), Index, Depth: Depth + 1,
15279 StartingIndex);
15280 if (!LHS)
15281 return std::nullopt;
15282 // A well formed Or will have two ByteProviders for each byte, one of which
15283 // is constant zero
15284 if (!LHS->isConstantZero() && !RHS->isConstantZero())
15285 return std::nullopt;
15286 if (!LHS || LHS->isConstantZero())
15287 return RHS;
15288 if (!RHS || RHS->isConstantZero())
15289 return LHS;
15290 return std::nullopt;
15291 }
15292
15293 case ISD::AND: {
15294 if (IsVec)
15295 return std::nullopt;
15296
15297 auto *BitMaskOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15298 if (!BitMaskOp)
15299 return std::nullopt;
15300
15301 uint32_t BitMask = BitMaskOp->getZExtValue();
15302 // Bits we expect for our StartingIndex
15303 uint32_t IndexMask = 0xFF << (Index * 8);
15304
15305 if ((IndexMask & BitMask) != IndexMask) {
15306 // If the result of the and partially provides the byte, then it
15307 // is not well formatted
15308 if (IndexMask & BitMask)
15309 return std::nullopt;
15310 return ByteProvider<SDValue>::getConstantZero();
15311 }
15312
15313 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte: StartingIndex, SrcIndex: Index);
15314 }
15315
15316 case ISD::FSHR: {
15317 if (IsVec)
15318 return std::nullopt;
15319
15320 // fshr(X,Y,Z): (X << (BW - (Z % BW))) | (Y >> (Z % BW))
15321 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 2));
15322 if (!ShiftOp || Op.getValueType().isVector())
15323 return std::nullopt;
15324
15325 uint64_t BitsProvided = Op.getValueSizeInBits();
15326 if (BitsProvided % 8 != 0)
15327 return std::nullopt;
15328
15329 uint64_t BitShift = ShiftOp->getAPIntValue().urem(RHS: BitsProvided);
15330 if (BitShift % 8)
15331 return std::nullopt;
15332
15333 uint64_t ConcatSizeInBytes = BitsProvided / 4;
15334 uint64_t ByteShift = BitShift / 8;
15335
15336 uint64_t NewIndex = (Index + ByteShift) % ConcatSizeInBytes;
15337 uint64_t BytesProvided = BitsProvided / 8;
15338 SDValue NextOp = Op.getOperand(i: NewIndex >= BytesProvided ? 0 : 1);
15339 NewIndex %= BytesProvided;
15340 return calculateByteProvider(Op: NextOp, Index: NewIndex, Depth: Depth + 1, StartingIndex);
15341 }
15342
15343 case ISD::SRA:
15344 case ISD::SRL: {
15345 if (IsVec)
15346 return std::nullopt;
15347
15348 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15349 if (!ShiftOp)
15350 return std::nullopt;
15351
15352 uint64_t BitShift = ShiftOp->getZExtValue();
15353 if (BitShift % 8)
15354 return std::nullopt;
15355
15356 auto BitsProvided = Op.getScalarValueSizeInBits();
15357 if (BitsProvided % 8 != 0)
15358 return std::nullopt;
15359
15360 uint64_t BytesProvided = BitsProvided / 8;
15361 uint64_t ByteShift = BitShift / 8;
15362 if (Index + ByteShift < BytesProvided)
15363 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte: StartingIndex,
15364 SrcIndex: Index + ByteShift);
15365 // SRA's out-of-range bytes are sign bits, not constant zero.
15366 if (Op.getOpcode() == ISD::SRA)
15367 return std::nullopt;
15368 return ByteProvider<SDValue>::getConstantZero();
15369 }
15370
15371 case ISD::SHL: {
15372 if (IsVec)
15373 return std::nullopt;
15374
15375 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15376 if (!ShiftOp)
15377 return std::nullopt;
15378
15379 uint64_t BitShift = ShiftOp->getZExtValue();
15380 if (BitShift % 8 != 0)
15381 return std::nullopt;
15382 uint64_t ByteShift = BitShift / 8;
15383
15384 // If we are shifting by an amount greater than (or equal to)
15385 // the index we are trying to provide, then it provides 0s. If not,
15386 // then this bytes are not definitively 0s, and the corresponding byte
15387 // of interest is Index - ByteShift of the src
15388 return Index < ByteShift
15389 ? ByteProvider<SDValue>::getConstantZero()
15390 : calculateByteProvider(Op: Op.getOperand(i: 0), Index: Index - ByteShift,
15391 Depth: Depth + 1, StartingIndex);
15392 }
15393 case ISD::ANY_EXTEND:
15394 case ISD::SIGN_EXTEND:
15395 case ISD::ZERO_EXTEND:
15396 case ISD::SIGN_EXTEND_INREG:
15397 case ISD::AssertZext:
15398 case ISD::AssertSext: {
15399 if (IsVec)
15400 return std::nullopt;
15401
15402 SDValue NarrowOp = Op->getOperand(Num: 0);
15403 unsigned NarrowBitWidth = NarrowOp.getValueSizeInBits();
15404 if (Op->getOpcode() == ISD::SIGN_EXTEND_INREG ||
15405 Op->getOpcode() == ISD::AssertZext ||
15406 Op->getOpcode() == ISD::AssertSext) {
15407 auto *VTSign = cast<VTSDNode>(Val: Op->getOperand(Num: 1));
15408 NarrowBitWidth = VTSign->getVT().getSizeInBits();
15409 }
15410 if (NarrowBitWidth % 8 != 0)
15411 return std::nullopt;
15412 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
15413
15414 if (Index >= NarrowByteWidth)
15415 return Op.getOpcode() == ISD::ZERO_EXTEND
15416 ? std::optional<ByteProvider<SDValue>>(
15417 ByteProvider<SDValue>::getConstantZero())
15418 : std::nullopt;
15419 return calculateByteProvider(Op: NarrowOp, Index, Depth: Depth + 1, StartingIndex);
15420 }
15421
15422 case ISD::TRUNCATE: {
15423 if (IsVec)
15424 return std::nullopt;
15425
15426 uint64_t NarrowByteWidth = BitWidth / 8;
15427
15428 if (NarrowByteWidth >= Index) {
15429 return calculateByteProvider(Op: Op.getOperand(i: 0), Index, Depth: Depth + 1,
15430 StartingIndex);
15431 }
15432
15433 return std::nullopt;
15434 }
15435
15436 case ISD::CopyFromReg: {
15437 if (BitWidth / 8 > Index)
15438 return calculateSrcByte(Op, DestByte: StartingIndex, SrcIndex: Index);
15439
15440 return std::nullopt;
15441 }
15442
15443 case ISD::LOAD: {
15444 auto *L = cast<LoadSDNode>(Val: Op.getNode());
15445
15446 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
15447 if (NarrowBitWidth % 8 != 0)
15448 return std::nullopt;
15449 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
15450
15451 // If the width of the load does not reach byte we are trying to provide for
15452 // and it is not a ZEXTLOAD, then the load does not provide for the byte in
15453 // question
15454 if (Index >= NarrowByteWidth) {
15455 return L->getExtensionType() == ISD::ZEXTLOAD
15456 ? std::optional<ByteProvider<SDValue>>(
15457 ByteProvider<SDValue>::getConstantZero())
15458 : std::nullopt;
15459 }
15460
15461 if (NarrowByteWidth > Index) {
15462 return calculateSrcByte(Op, DestByte: StartingIndex, SrcIndex: Index);
15463 }
15464
15465 return std::nullopt;
15466 }
15467
15468 case ISD::BSWAP: {
15469 if (IsVec)
15470 return std::nullopt;
15471
15472 return calculateByteProvider(Op: Op->getOperand(Num: 0), Index: BitWidth / 8 - Index - 1,
15473 Depth: Depth + 1, StartingIndex);
15474 }
15475
15476 case ISD::EXTRACT_VECTOR_ELT: {
15477 auto *IdxOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15478 if (!IdxOp)
15479 return std::nullopt;
15480 auto VecIdx = IdxOp->getZExtValue();
15481 auto ScalarSize = Op.getScalarValueSizeInBits();
15482 if (ScalarSize < 32)
15483 Index = ScalarSize == 8 ? VecIdx : VecIdx * 2 + Index;
15484 return calculateSrcByte(Op: ScalarSize >= 32 ? Op : Op.getOperand(i: 0),
15485 DestByte: StartingIndex, SrcIndex: Index);
15486 }
15487
15488 case AMDGPUISD::PERM: {
15489 if (IsVec)
15490 return std::nullopt;
15491
15492 auto *PermMask = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 2));
15493 if (!PermMask)
15494 return std::nullopt;
15495
15496 auto IdxMask =
15497 (PermMask->getZExtValue() & (0xFF << (Index * 8))) >> (Index * 8);
15498 if (IdxMask > 0x07 && IdxMask != 0x0c)
15499 return std::nullopt;
15500
15501 auto NextOp = Op.getOperand(i: IdxMask > 0x03 ? 0 : 1);
15502 auto NextIndex = IdxMask > 0x03 ? IdxMask % 4 : IdxMask;
15503
15504 return IdxMask != 0x0c ? calculateSrcByte(Op: NextOp, DestByte: StartingIndex, SrcIndex: NextIndex)
15505 : ByteProvider<SDValue>(
15506 ByteProvider<SDValue>::getConstantZero());
15507 }
15508
15509 default: {
15510 return std::nullopt;
15511 }
15512 }
15513
15514 llvm_unreachable("fully handled switch");
15515}
15516
15517// Returns true if the Operand is a scalar and is 16 bits
15518static bool isExtendedFrom16Bits(SDValue &Operand) {
15519
15520 switch (Operand.getOpcode()) {
15521 case ISD::ANY_EXTEND:
15522 case ISD::SIGN_EXTEND:
15523 case ISD::ZERO_EXTEND: {
15524 auto OpVT = Operand.getOperand(i: 0).getValueType();
15525 return !OpVT.isVector() && OpVT.getSizeInBits() == 16;
15526 }
15527 case ISD::LOAD: {
15528 LoadSDNode *L = cast<LoadSDNode>(Val: Operand.getNode());
15529 auto ExtType = cast<LoadSDNode>(Val: L)->getExtensionType();
15530 if (ExtType == ISD::ZEXTLOAD || ExtType == ISD::SEXTLOAD ||
15531 ExtType == ISD::EXTLOAD) {
15532 auto MemVT = L->getMemoryVT();
15533 return !MemVT.isVector() && MemVT.getSizeInBits() == 16;
15534 }
15535 return L->getMemoryVT().getSizeInBits() == 16;
15536 }
15537 default:
15538 return false;
15539 }
15540}
15541
15542// Returns true if the mask matches consecutive bytes, and the first byte
15543// begins at a power of 2 byte offset from 0th byte
15544static bool addresses16Bits(int Mask) {
15545 int Low8 = Mask & 0xff;
15546 int Hi8 = (Mask & 0xff00) >> 8;
15547
15548 assert(Low8 < 8 && Hi8 < 8);
15549 // Are the bytes contiguous in the order of increasing addresses.
15550 bool IsConsecutive = (Hi8 - Low8 == 1);
15551 // Is the first byte at location that is aligned for 16 bit instructions.
15552 // A counter example is taking 2 consecutive bytes starting at the 8th bit.
15553 // In this case, we still need code to extract the 16 bit operand, so it
15554 // is better to use i8 v_perm
15555 bool Is16Aligned = !(Low8 % 2);
15556
15557 return IsConsecutive && Is16Aligned;
15558}
15559
15560// Do not lower into v_perm if the operands are actually 16 bit
15561// and the selected bits (based on PermMask) correspond with two
15562// easily addressable 16 bit operands.
15563static bool hasNon16BitAccesses(uint64_t PermMask, SDValue &Op,
15564 SDValue &OtherOp) {
15565 int Low16 = PermMask & 0xffff;
15566 int Hi16 = (PermMask & 0xffff0000) >> 16;
15567
15568 auto TempOp = peekThroughBitcasts(V: Op);
15569 auto TempOtherOp = peekThroughBitcasts(V: OtherOp);
15570
15571 auto OpIs16Bit =
15572 TempOp.getValueSizeInBits() == 16 || isExtendedFrom16Bits(Operand&: TempOp);
15573 if (!OpIs16Bit)
15574 return true;
15575
15576 auto OtherOpIs16Bit = TempOtherOp.getValueSizeInBits() == 16 ||
15577 isExtendedFrom16Bits(Operand&: TempOtherOp);
15578 if (!OtherOpIs16Bit)
15579 return true;
15580
15581 // Do we cleanly address both
15582 return !addresses16Bits(Mask: Low16) || !addresses16Bits(Mask: Hi16);
15583}
15584
15585static SDValue getDWordFromOffset(SelectionDAG &DAG, SDLoc SL, SDValue Src,
15586 unsigned DWordOffset) {
15587 SDValue Ret;
15588
15589 auto TypeSize = Src.getValueSizeInBits().getFixedValue();
15590 // ByteProvider must be at least 8 bits
15591 assert(Src.getValueSizeInBits().isKnownMultipleOf(8));
15592
15593 if (TypeSize <= 32)
15594 return DAG.getBitcastedAnyExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32);
15595
15596 if (Src.getValueType().isVector()) {
15597 auto ScalarTySize = Src.getScalarValueSizeInBits();
15598 auto ScalarTy = Src.getValueType().getScalarType();
15599 if (ScalarTySize == 32) {
15600 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Src,
15601 N2: DAG.getConstant(Val: DWordOffset, DL: SL, VT: MVT::i32));
15602 }
15603 if (ScalarTySize > 32) {
15604 Ret = DAG.getNode(
15605 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ScalarTy, N1: Src,
15606 N2: DAG.getConstant(Val: DWordOffset / (ScalarTySize / 32), DL: SL, VT: MVT::i32));
15607 auto ShiftVal = 32 * (DWordOffset % (ScalarTySize / 32));
15608 if (ShiftVal)
15609 Ret = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: Ret.getValueType(), N1: Ret,
15610 N2: DAG.getConstant(Val: ShiftVal, DL: SL, VT: MVT::i32));
15611 return DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
15612 }
15613
15614 assert(ScalarTySize < 32);
15615 if (TypeSize % 32 == 0) {
15616 assert(DWordOffset < TypeSize / 32);
15617 SDValue Cast = DAG.getBitcast(
15618 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: TypeSize / 32), V: Src);
15619 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Cast,
15620 N2: DAG.getConstant(Val: DWordOffset, DL: SL, VT: MVT::i32));
15621 }
15622
15623 auto NumElements = TypeSize / ScalarTySize;
15624 auto Trunc32Elements = (ScalarTySize * NumElements) / 32;
15625 auto NormalizedTrunc = Trunc32Elements * 32 / ScalarTySize;
15626 auto NumElementsIn32 = 32 / ScalarTySize;
15627 auto NumAvailElements = DWordOffset < Trunc32Elements
15628 ? NumElementsIn32
15629 : NumElements - NormalizedTrunc;
15630
15631 SmallVector<SDValue, 4> VecSrcs;
15632 DAG.ExtractVectorElements(Op: Src, Args&: VecSrcs, Start: DWordOffset * NumElementsIn32,
15633 Count: NumAvailElements);
15634
15635 Ret = DAG.getBuildVector(
15636 VT: MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ScalarTySize), NumElements: NumAvailElements), DL: SL,
15637 Ops: VecSrcs);
15638 return Ret = DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
15639 }
15640
15641 /// Scalar Type
15642 auto ShiftVal = 32 * DWordOffset;
15643 Ret = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: Src.getValueType(), N1: Src,
15644 N2: DAG.getConstant(Val: ShiftVal, DL: SL, VT: MVT::i32));
15645 return DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
15646}
15647
15648static SDValue matchPERM(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15649 SelectionDAG &DAG = DCI.DAG;
15650 [[maybe_unused]] EVT VT = N->getValueType(ResNo: 0);
15651 SmallVector<ByteProvider<SDValue>, 8> PermNodes;
15652
15653 // VT is known to be MVT::i32, so we need to provide 4 bytes.
15654 assert(VT == MVT::i32);
15655 for (int i = 0; i < 4; i++) {
15656 // Find the ByteProvider that provides the ith byte of the result of OR
15657 std::optional<ByteProvider<SDValue>> P =
15658 calculateByteProvider(Op: SDValue(N, 0), Index: i, Depth: 0, /*StartingIndex = */ i);
15659 // TODO support constantZero
15660 if (!P || P->isConstantZero())
15661 return SDValue();
15662
15663 PermNodes.push_back(Elt: *P);
15664 }
15665 if (PermNodes.size() != 4)
15666 return SDValue();
15667
15668 std::pair<unsigned, unsigned> FirstSrc(0, PermNodes[0].SrcOffset / 4);
15669 std::optional<std::pair<unsigned, unsigned>> SecondSrc;
15670 uint64_t PermMask = 0x00000000;
15671 for (size_t i = 0; i < PermNodes.size(); i++) {
15672 auto PermOp = PermNodes[i];
15673 // Since the mask is applied to Src1:Src2, Src1 bytes must be offset
15674 // by sizeof(Src2) = 4
15675 int SrcByteAdjust = 4;
15676
15677 // If the Src uses a byte from a different DWORD, then it corresponds
15678 // with a difference source
15679 if (!PermOp.hasSameSrc(Other: PermNodes[FirstSrc.first]) ||
15680 ((PermOp.SrcOffset / 4) != FirstSrc.second)) {
15681 if (SecondSrc)
15682 if (!PermOp.hasSameSrc(Other: PermNodes[SecondSrc->first]) ||
15683 ((PermOp.SrcOffset / 4) != SecondSrc->second))
15684 return SDValue();
15685
15686 // Set the index of the second distinct Src node
15687 SecondSrc = {i, PermNodes[i].SrcOffset / 4};
15688 assert(!(PermNodes[SecondSrc->first].Src->getValueSizeInBits() % 8));
15689 SrcByteAdjust = 0;
15690 }
15691 assert((PermOp.SrcOffset % 4) + SrcByteAdjust < 8);
15692 assert(!DAG.getDataLayout().isBigEndian());
15693 PermMask |= ((PermOp.SrcOffset % 4) + SrcByteAdjust) << (i * 8);
15694 }
15695 SDLoc DL(N);
15696 SDValue Op = *PermNodes[FirstSrc.first].Src;
15697 Op = getDWordFromOffset(DAG, SL: DL, Src: Op, DWordOffset: FirstSrc.second);
15698 assert(Op.getValueSizeInBits() == 32);
15699
15700 // Check that we are not just extracting the bytes in order from an op
15701 if (!SecondSrc) {
15702 int Low16 = PermMask & 0xffff;
15703 int Hi16 = (PermMask & 0xffff0000) >> 16;
15704
15705 bool WellFormedLow = (Low16 == 0x0504) || (Low16 == 0x0100);
15706 bool WellFormedHi = (Hi16 == 0x0706) || (Hi16 == 0x0302);
15707
15708 // The perm op would really just produce Op. So combine into Op
15709 if (WellFormedLow && WellFormedHi)
15710 return DAG.getBitcast(VT: MVT::getIntegerVT(BitWidth: 32), V: Op);
15711 }
15712
15713 SDValue OtherOp = SecondSrc ? *PermNodes[SecondSrc->first].Src : Op;
15714
15715 if (SecondSrc) {
15716 OtherOp = getDWordFromOffset(DAG, SL: DL, Src: OtherOp, DWordOffset: SecondSrc->second);
15717 assert(OtherOp.getValueSizeInBits() == 32);
15718 }
15719
15720 // Check that we haven't just recreated the same FSHR node.
15721 if (N->getOpcode() == ISD::FSHR &&
15722 (N->getOperand(Num: 0) == Op || N->getOperand(Num: 0) == OtherOp) &&
15723 (N->getOperand(Num: 1) == Op || N->getOperand(Num: 1) == OtherOp))
15724 return SDValue();
15725
15726 if (hasNon16BitAccesses(PermMask, Op, OtherOp)) {
15727
15728 assert(Op.getValueType().isByteSized() &&
15729 OtherOp.getValueType().isByteSized());
15730
15731 // If the ultimate src is less than 32 bits, then we will only be
15732 // using bytes 0: Op.getValueSizeInBytes() - 1 in the or.
15733 // CalculateByteProvider would not have returned Op as source if we
15734 // used a byte that is outside its ValueType. Thus, we are free to
15735 // ANY_EXTEND as the extended bits are dont-cares.
15736 Op = DAG.getBitcastedAnyExtOrTrunc(Op, DL, VT: MVT::i32);
15737 OtherOp = DAG.getBitcastedAnyExtOrTrunc(Op: OtherOp, DL, VT: MVT::i32);
15738
15739 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: Op, N2: OtherOp,
15740 N3: DAG.getConstant(Val: PermMask, DL, VT: MVT::i32));
15741 }
15742 return SDValue();
15743}
15744
15745SDValue SITargetLowering::performOrCombine(SDNode *N,
15746 DAGCombinerInfo &DCI) const {
15747 SelectionDAG &DAG = DCI.DAG;
15748 SDValue LHS = N->getOperand(Num: 0);
15749 SDValue RHS = N->getOperand(Num: 1);
15750
15751 EVT VT = N->getValueType(ResNo: 0);
15752 if (VT == MVT::i1) {
15753 // or (fp_class x, c1), (fp_class x, c2) -> fp_class x, (c1 | c2)
15754 if (LHS.getOpcode() == AMDGPUISD::FP_CLASS &&
15755 RHS.getOpcode() == AMDGPUISD::FP_CLASS) {
15756 SDValue Src = LHS.getOperand(i: 0);
15757 if (Src != RHS.getOperand(i: 0))
15758 return SDValue();
15759
15760 const ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
15761 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i: 1));
15762 if (!CLHS || !CRHS)
15763 return SDValue();
15764
15765 // Only 10 bits are used.
15766 static const uint32_t MaxMask = 0x3ff;
15767
15768 uint32_t NewMask =
15769 (CLHS->getZExtValue() | CRHS->getZExtValue()) & MaxMask;
15770 SDLoc DL(N);
15771 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: Src,
15772 N2: DAG.getConstant(Val: NewMask, DL, VT: MVT::i32));
15773 }
15774
15775 return SDValue();
15776 }
15777
15778 // or (perm x, y, c1), c2 -> perm x, y, permute_mask(c1, c2)
15779 if (isa<ConstantSDNode>(Val: RHS) && LHS.hasOneUse() &&
15780 LHS.getOpcode() == AMDGPUISD::PERM &&
15781 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2))) {
15782 uint32_t Sel = getConstantPermuteMask(C: N->getConstantOperandVal(Num: 1));
15783 if (!Sel)
15784 return SDValue();
15785
15786 Sel |= LHS.getConstantOperandVal(i: 2);
15787 SDLoc DL(N);
15788 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15789 N2: LHS.getOperand(i: 1), N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15790 }
15791
15792 // or (op x, c1), (op y, c2) -> perm x, y, permute_mask(c1, c2)
15793 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
15794 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() &&
15795 TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
15796
15797 // If all the uses of an or need to extract the individual elements, do not
15798 // attempt to lower into v_perm
15799 auto usesCombinedOperand = [](SDNode *OrUse) {
15800 // If we have any non-vectorized use, then it is a candidate for v_perm
15801 if (OrUse->getOpcode() != ISD::BITCAST ||
15802 !OrUse->getValueType(ResNo: 0).isVector())
15803 return true;
15804
15805 // If we have any non-vectorized use, then it is a candidate for v_perm
15806 for (auto *VUser : OrUse->users()) {
15807 if (!VUser->getValueType(ResNo: 0).isVector())
15808 return true;
15809
15810 // If the use of a vector is a store, then combining via a v_perm
15811 // is beneficial.
15812 // TODO -- whitelist more uses
15813 for (auto VectorwiseOp : {ISD::STORE, ISD::CopyToReg, ISD::CopyFromReg})
15814 if (VUser->getOpcode() == VectorwiseOp)
15815 return true;
15816 }
15817 return false;
15818 };
15819
15820 if (!any_of(Range: N->users(), P: usesCombinedOperand))
15821 return SDValue();
15822
15823 uint32_t LHSMask = getPermuteMask(V: LHS);
15824 uint32_t RHSMask = getPermuteMask(V: RHS);
15825
15826 if (LHSMask != ~0u && RHSMask != ~0u) {
15827 // Canonicalize the expression in an attempt to have fewer unique masks
15828 // and therefore fewer registers used to hold the masks.
15829 if (LHSMask > RHSMask) {
15830 std::swap(a&: LHSMask, b&: RHSMask);
15831 std::swap(a&: LHS, b&: RHS);
15832 }
15833
15834 // Select 0xc for each lane used from source operand. Zero has 0xc mask
15835 // set, 0xff have 0xff in the mask, actual lanes are in the 0-3 range.
15836 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15837 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15838
15839 // Check of we need to combine values from two sources within a byte.
15840 if (!(LHSUsedLanes & RHSUsedLanes) &&
15841 // If we select high and lower word keep it for SDWA.
15842 // TODO: teach SDWA to work with v_perm_b32 and remove the check.
15843 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
15844 // Kill zero bytes selected by other mask. Zero value is 0xc.
15845 LHSMask &= ~RHSUsedLanes;
15846 RHSMask &= ~LHSUsedLanes;
15847 // Add 4 to each active LHS lane
15848 LHSMask |= LHSUsedLanes & 0x04040404;
15849 // Combine masks
15850 uint32_t Sel = LHSMask | RHSMask;
15851 SDLoc DL(N);
15852
15853 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15854 N2: RHS.getOperand(i: 0),
15855 N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15856 }
15857 }
15858 if (LHSMask == ~0u || RHSMask == ~0u) {
15859 if (SDValue Perm = matchPERM(N, DCI))
15860 return Perm;
15861 }
15862 }
15863
15864 // Detect identity v2i32 OR and replace with identity source node.
15865 // Specifically an Or that has operands constructed from the same source node
15866 // via extract_vector_elt and build_vector. I.E.
15867 // v2i32 or(
15868 // v2i32 build_vector(
15869 // i32 extract_elt(%IdentitySrc, 0),
15870 // i32 0
15871 // ),
15872 // v2i32 build_vector(
15873 // i32 0,
15874 // i32 extract_elt(%IdentitySrc, 1)
15875 // ) )
15876 // =>
15877 // v2i32 %IdentitySrc
15878
15879 if (VT == MVT::v2i32 && LHS->getOpcode() == ISD::BUILD_VECTOR &&
15880 RHS->getOpcode() == ISD::BUILD_VECTOR) {
15881
15882 ConstantSDNode *LC = dyn_cast<ConstantSDNode>(Val: LHS->getOperand(Num: 1));
15883 ConstantSDNode *RC = dyn_cast<ConstantSDNode>(Val: RHS->getOperand(Num: 0));
15884
15885 // Test for and normalise build vectors.
15886 if (LC && RC && LC->getZExtValue() == 0 && RC->getZExtValue() == 0) {
15887
15888 // Get the extract_vector_element operands.
15889 SDValue LEVE = LHS->getOperand(Num: 0);
15890 SDValue REVE = RHS->getOperand(Num: 1);
15891
15892 if (LEVE->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15893 REVE->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
15894 // Check that different elements from the same vector are
15895 // extracted.
15896 if (LEVE->getOperand(Num: 0) == REVE->getOperand(Num: 0) &&
15897 LEVE->getOperand(Num: 1) != REVE->getOperand(Num: 1)) {
15898 SDValue IdentitySrc = LEVE.getOperand(i: 0);
15899 return IdentitySrc;
15900 }
15901 }
15902 }
15903 }
15904
15905 if (VT != MVT::i64 || DCI.isBeforeLegalizeOps())
15906 return SDValue();
15907
15908 // TODO: This could be a generic combine with a predicate for extracting the
15909 // high half of an integer being free.
15910
15911 // (or i64:x, (zero_extend i32:y)) ->
15912 // i64 (bitcast (v2i32 build_vector (or i32:y, lo_32(x)), hi_32(x)))
15913 if (LHS.getOpcode() == ISD::ZERO_EXTEND &&
15914 RHS.getOpcode() != ISD::ZERO_EXTEND)
15915 std::swap(a&: LHS, b&: RHS);
15916
15917 if (RHS.getOpcode() == ISD::ZERO_EXTEND) {
15918 SDValue ExtSrc = RHS.getOperand(i: 0);
15919 EVT SrcVT = ExtSrc.getValueType();
15920 if (SrcVT == MVT::i32) {
15921 SDLoc SL(N);
15922 auto [LowLHS, HiBits] = split64BitValue(Op: LHS, DAG);
15923 SDValue LowOr = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: LowLHS, N2: ExtSrc);
15924
15925 DCI.AddToWorklist(N: LowOr.getNode());
15926 DCI.AddToWorklist(N: HiBits.getNode());
15927
15928 SDValue Vec =
15929 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: LowOr, N2: HiBits);
15930 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
15931 }
15932 }
15933
15934 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
15935 if (CRHS) {
15936 if (SDValue Split = splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::OR,
15937 LHS: N->getOperand(Num: 0), CRHS))
15938 return Split;
15939 }
15940
15941 return SDValue();
15942}
15943
15944SDValue SITargetLowering::performXorCombine(SDNode *N,
15945 DAGCombinerInfo &DCI) const {
15946 if (SDValue RV = reassociateScalarOps(N, DAG&: DCI.DAG))
15947 return RV;
15948
15949 SDValue LHS = N->getOperand(Num: 0);
15950 SDValue RHS = N->getOperand(Num: 1);
15951
15952 const ConstantSDNode *CRHS = isConstOrConstSplat(N: RHS);
15953 SelectionDAG &DAG = DCI.DAG;
15954
15955 EVT VT = N->getValueType(ResNo: 0);
15956 if (CRHS && VT == MVT::i64) {
15957 if (SDValue Split =
15958 splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::XOR, LHS, CRHS))
15959 return Split;
15960 }
15961
15962 // v2i32 (xor (vselect cc, x, y), K) ->
15963 // (v2i32 svelect cc, (xor x, K), (xor y, K)) This enables the xor to be
15964 // replaced with source modifiers when the select is lowered to CNDMASK.
15965 unsigned Opc = LHS.getOpcode();
15966 if (((Opc == ISD::VSELECT && VT == MVT::v2i32) ||
15967 (Opc == ISD::SELECT && VT == MVT::i64)) &&
15968 CRHS && CRHS->getAPIntValue().isSignMask()) {
15969 SDValue CC = LHS->getOperand(Num: 0);
15970 SDValue TRUE = LHS->getOperand(Num: 1);
15971 SDValue FALSE = LHS->getOperand(Num: 2);
15972 SDValue XTrue = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N), VT, N1: TRUE, N2: RHS);
15973 SDValue XFalse = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N), VT, N1: FALSE, N2: RHS);
15974 SDValue XSelect =
15975 DAG.getNode(Opcode: ISD::VSELECT, DL: SDLoc(N), VT, N1: CC, N2: XTrue, N3: XFalse);
15976 return XSelect;
15977 }
15978
15979 // Make sure to apply the 64-bit constant splitting fold before trying to fold
15980 // fneg-like xors into 64-bit select.
15981 if (LHS.getOpcode() == ISD::SELECT && VT == MVT::i32) {
15982 // This looks like an fneg, try to fold as a source modifier.
15983 if (CRHS && CRHS->getAPIntValue().isSignMask() &&
15984 shouldFoldFNegIntoSrc(FNeg: N, FNegSrc: LHS)) {
15985 // xor (select c, a, b), 0x80000000 ->
15986 // bitcast (select c, (fneg (bitcast a)), (fneg (bitcast b)))
15987 SDLoc DL(N);
15988 SDValue CastLHS =
15989 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: LHS->getOperand(Num: 1));
15990 SDValue CastRHS =
15991 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: LHS->getOperand(Num: 2));
15992 SDValue FNegLHS = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f32, Operand: CastLHS);
15993 SDValue FNegRHS = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f32, Operand: CastRHS);
15994 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::f32,
15995 N1: LHS->getOperand(Num: 0), N2: FNegLHS, N3: FNegRHS);
15996 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: NewSelect);
15997 }
15998 }
15999
16000 return SDValue();
16001}
16002
16003SDValue
16004SITargetLowering::performZeroOrAnyExtendCombine(SDNode *N,
16005 DAGCombinerInfo &DCI) const {
16006 if (!Subtarget->has16BitInsts() ||
16007 DCI.getDAGCombineLevel() < AfterLegalizeTypes)
16008 return SDValue();
16009
16010 EVT VT = N->getValueType(ResNo: 0);
16011 if (VT != MVT::i32)
16012 return SDValue();
16013
16014 SDValue Src = N->getOperand(Num: 0);
16015 if (Src.getValueType() != MVT::i16)
16016 return SDValue();
16017
16018 if (!Src->hasOneUse())
16019 return SDValue();
16020
16021 // TODO: We bail out below if SrcOffset is not in the first dword (>= 4). It's
16022 // possible we're missing out on some combine opportunities, but we'd need to
16023 // weigh the cost of extracting the byte from the upper dwords.
16024
16025 std::optional<ByteProvider<SDValue>> BP0 =
16026 calculateByteProvider(Op: SDValue(N, 0), Index: 0, Depth: 0, StartingIndex: 0);
16027 if (!BP0 || BP0->SrcOffset >= 4 || !BP0->Src)
16028 return SDValue();
16029 SDValue V0 = *BP0->Src;
16030
16031 std::optional<ByteProvider<SDValue>> BP1 =
16032 calculateByteProvider(Op: SDValue(N, 0), Index: 1, Depth: 0, StartingIndex: 1);
16033 if (!BP1 || BP1->SrcOffset >= 4 || !BP1->Src)
16034 return SDValue();
16035
16036 SDValue V1 = *BP1->Src;
16037
16038 if (V0 == V1)
16039 return SDValue();
16040
16041 SelectionDAG &DAG = DCI.DAG;
16042 SDLoc DL(N);
16043 uint32_t PermMask = 0x0c0c0c0c;
16044 if (V0) {
16045 V0 = DAG.getBitcastedAnyExtOrTrunc(Op: V0, DL, VT: MVT::i32);
16046 PermMask = (PermMask & ~0xFF) | (BP0->SrcOffset + 4);
16047 }
16048
16049 if (V1) {
16050 V1 = DAG.getBitcastedAnyExtOrTrunc(Op: V1, DL, VT: MVT::i32);
16051 PermMask = (PermMask & ~(0xFF << 8)) | (BP1->SrcOffset << 8);
16052 }
16053
16054 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: V0, N2: V1,
16055 N3: DAG.getConstant(Val: PermMask, DL, VT: MVT::i32));
16056}
16057
16058SDValue
16059SITargetLowering::performSignExtendInRegCombine(SDNode *N,
16060 DAGCombinerInfo &DCI) const {
16061 SDValue Src = N->getOperand(Num: 0);
16062 auto *VTSign = cast<VTSDNode>(Val: N->getOperand(Num: 1));
16063
16064 // Combine s_buffer_load_u8 or s_buffer_load_u16 with sext and replace them
16065 // with s_buffer_load_i8 and s_buffer_load_i16 respectively.
16066 if (((Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE &&
16067 VTSign->getVT() == MVT::i8) ||
16068 (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_USHORT &&
16069 VTSign->getVT() == MVT::i16))) {
16070 assert(Subtarget->hasScalarSubwordLoads() &&
16071 "s_buffer_load_{u8, i8} are supported "
16072 "in GFX12 (or newer) architectures.");
16073 unsigned Opc = (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE)
16074 ? AMDGPUISD::SBUFFER_LOAD_BYTE
16075 : AMDGPUISD::SBUFFER_LOAD_SHORT;
16076 SDLoc DL(N);
16077 SDVTList ResList =
16078 DCI.DAG.getVTList(VT1: MVT::i32, VT2: Src.getOperand(i: 0).getValueType());
16079 SDValue Ops[] = {
16080 Src.getOperand(i: 0), // Chain
16081 Src.getOperand(i: 1), // source register
16082 Src.getOperand(i: 2), // offset
16083 Src.getOperand(i: 3) // cachePolicy
16084 };
16085 auto *M = cast<MemSDNode>(Val&: Src);
16086 SDValue BufferLoad = DCI.DAG.getMemIntrinsicNode(
16087 Opcode: Opc, dl: DL, VTList: ResList, Ops, MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
16088 return DCI.DAG.getMergeValues(Ops: {BufferLoad, BufferLoad.getValue(R: 1)}, dl: DL);
16089 }
16090 if (((Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE &&
16091 VTSign->getVT() == MVT::i8) ||
16092 (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_USHORT &&
16093 VTSign->getVT() == MVT::i16)) &&
16094 Src.hasOneUse()) {
16095 auto *M = cast<MemSDNode>(Val&: Src);
16096 SDValue Ops[] = {Src.getOperand(i: 0), // Chain
16097 Src.getOperand(i: 1), // rsrc
16098 Src.getOperand(i: 2), // vindex
16099 Src.getOperand(i: 3), // voffset
16100 Src.getOperand(i: 4), // soffset
16101 Src.getOperand(i: 5), // offset
16102 Src.getOperand(i: 6), Src.getOperand(i: 7)};
16103 // replace with BUFFER_LOAD_BYTE/SHORT
16104 SDVTList ResList =
16105 DCI.DAG.getVTList(VT1: MVT::i32, VT2: Src.getOperand(i: 0).getValueType());
16106 unsigned Opc = (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE)
16107 ? AMDGPUISD::BUFFER_LOAD_BYTE
16108 : AMDGPUISD::BUFFER_LOAD_SHORT;
16109 SDValue BufferLoadSignExt = DCI.DAG.getMemIntrinsicNode(
16110 Opcode: Opc, dl: SDLoc(N), VTList: ResList, Ops, MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
16111 return DCI.DAG.getMergeValues(
16112 Ops: {BufferLoadSignExt, BufferLoadSignExt.getValue(R: 1)}, dl: SDLoc(N));
16113 }
16114 return SDValue();
16115}
16116
16117SDValue SITargetLowering::performClassCombine(SDNode *N,
16118 DAGCombinerInfo &DCI) const {
16119 SelectionDAG &DAG = DCI.DAG;
16120 SDValue Mask = N->getOperand(Num: 1);
16121
16122 // fp_class x, 0 -> false
16123 if (isNullConstant(V: Mask))
16124 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT: MVT::i1);
16125
16126 if (N->getOperand(Num: 0).isUndef())
16127 return DAG.getUNDEF(VT: MVT::i1);
16128
16129 return SDValue();
16130}
16131
16132SDValue SITargetLowering::performRcpCombine(SDNode *N,
16133 DAGCombinerInfo &DCI) const {
16134 EVT VT = N->getValueType(ResNo: 0);
16135 SDValue N0 = N->getOperand(Num: 0);
16136
16137 if (N0.isUndef()) {
16138 return DCI.DAG.getConstantFP(Val: APFloat::getQNaN(Sem: VT.getFltSemantics()),
16139 DL: SDLoc(N), VT);
16140 }
16141
16142 // TODO: Could handle f32 + amdgcn.sqrt but probably never reaches here.
16143 if ((VT == MVT::f16 && N0.getOpcode() == ISD::FSQRT) &&
16144 N->getFlags().hasAllowContract() && N0->getFlags().hasAllowContract()) {
16145 return DCI.DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SDLoc(N), VT, Operand: N0.getOperand(i: 0),
16146 Flags: N->getFlags());
16147 }
16148
16149 return AMDGPUTargetLowering::performRcpCombine(N, DCI);
16150}
16151
16152bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
16153 SDNodeFlags UserFlags,
16154 unsigned MaxDepth) const {
16155 EVT VT = Op.getValueType();
16156 assert(VT.isFloatingPoint() &&
16157 "expected a floating-point value to query canonicality of");
16158 return isCanonicalized(DAG, Op, QueryVT: VT.getScalarType(), UserFlags, MaxDepth);
16159}
16160
16161bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
16162 EVT QueryVT, SDNodeFlags UserFlags,
16163 unsigned MaxDepth) const {
16164 assert(QueryVT.isFloatingPoint() && !QueryVT.isVector() &&
16165 "QueryVT must be a floating-point scalar type");
16166 EVT VT = Op.getValueType();
16167 if (VT.isFloatingPoint() && VT.getScalarType() != QueryVT)
16168 return false;
16169
16170 unsigned Opcode = Op.getOpcode();
16171 if (Opcode == ISD::FCANONICALIZE)
16172 return true;
16173
16174 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
16175 const auto &F = CFP->getValueAPF();
16176 if (F.isNaN() && F.isSignaling())
16177 return false;
16178 if (!F.isDenormal())
16179 return true;
16180
16181 DenormalMode Mode =
16182 DAG.getMachineFunction().getDenormalMode(FPType: F.getSemantics());
16183 return Mode == DenormalMode::getIEEE();
16184 }
16185
16186 // If source is a result of another standard FP operation it is already in
16187 // canonical form.
16188 if (MaxDepth == 0)
16189 return false;
16190
16191 switch (Opcode) {
16192 // These will flush denorms if required.
16193 case ISD::FADD:
16194 case ISD::FSUB:
16195 case ISD::FMUL:
16196 case ISD::FCEIL:
16197 case ISD::FFLOOR:
16198 case ISD::FMA:
16199 case ISD::FMAD:
16200 case ISD::FSQRT:
16201 case ISD::FDIV:
16202 case ISD::FREM:
16203 case ISD::FP_ROUND:
16204 case ISD::FP_EXTEND:
16205 case ISD::FP16_TO_FP:
16206 case ISD::FP_TO_FP16:
16207 case ISD::BF16_TO_FP:
16208 case ISD::FP_TO_BF16:
16209 case ISD::FLDEXP:
16210 case AMDGPUISD::FMUL_LEGACY:
16211 case AMDGPUISD::FMAD_FTZ:
16212 case AMDGPUISD::RCP:
16213 case AMDGPUISD::RSQ:
16214 case AMDGPUISD::RSQ_CLAMP:
16215 case AMDGPUISD::RCP_LEGACY:
16216 case AMDGPUISD::RCP_IFLAG:
16217 case AMDGPUISD::LOG:
16218 case AMDGPUISD::EXP:
16219 case AMDGPUISD::DIV_SCALE:
16220 case AMDGPUISD::DIV_FMAS:
16221 case AMDGPUISD::DIV_FIXUP:
16222 case AMDGPUISD::FRACT:
16223 case AMDGPUISD::CVT_PKRTZ_F16_F32:
16224 case AMDGPUISD::CVT_F32_UBYTE0:
16225 case AMDGPUISD::CVT_F32_UBYTE1:
16226 case AMDGPUISD::CVT_F32_UBYTE2:
16227 case AMDGPUISD::CVT_F32_UBYTE3:
16228 case AMDGPUISD::FP_TO_FP16:
16229 case AMDGPUISD::SIN_HW:
16230 case AMDGPUISD::COS_HW:
16231 return true;
16232
16233 // It can/will be lowered or combined as a bit operation.
16234 // Need to check their input recursively to handle.
16235 case ISD::FNEG:
16236 case ISD::FABS:
16237 case ISD::FCOPYSIGN:
16238 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), QueryVT, UserFlags,
16239 MaxDepth: MaxDepth - 1);
16240
16241 case ISD::AND:
16242 if (Op.getValueType() == MVT::i32) {
16243 // Be careful as we only know it is a bitcast floating point type. It
16244 // could be f32, v2f16, we have no way of knowing. Luckily the constant
16245 // value that we optimize for, which comes up in fp32 to bf16 conversions,
16246 // is valid to optimize for all types.
16247 if (auto *RHS = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
16248 if (RHS->getZExtValue() == 0xffff0000) {
16249 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), QueryVT, UserFlags,
16250 MaxDepth: MaxDepth - 1);
16251 }
16252 }
16253 }
16254 break;
16255
16256 case ISD::FSIN:
16257 case ISD::FCOS:
16258 case ISD::FSINCOS:
16259 return Op.getValueType().getScalarType() != MVT::f16;
16260
16261 case ISD::FMINNUM:
16262 case ISD::FMAXNUM:
16263 case ISD::FMINNUM_IEEE:
16264 case ISD::FMAXNUM_IEEE:
16265 case ISD::FMINIMUM:
16266 case ISD::FMAXIMUM:
16267 case ISD::FMINIMUMNUM:
16268 case ISD::FMAXIMUMNUM:
16269 case AMDGPUISD::CLAMP:
16270 case AMDGPUISD::FMED3:
16271 case AMDGPUISD::FMAX3:
16272 case AMDGPUISD::FMIN3:
16273 case AMDGPUISD::FMAXIMUM3:
16274 case AMDGPUISD::FMINIMUM3: {
16275 // FIXME: Shouldn't treat the generic operations different based these.
16276 // However, we aren't really required to flush the result from
16277 // minnum/maxnum..
16278
16279 // snans will be quieted, so we only need to worry about denormals.
16280 if (Subtarget->supportsMinMaxDenormModes() ||
16281 // FIXME: denormalsEnabledForType is broken for dynamic
16282 denormalsEnabledForType(DAG, VT: Op.getValueType()))
16283 return true;
16284
16285 // Flushing may be required.
16286 // In pre-GFX9 targets V_MIN_F32 and others do not flush denorms. For such
16287 // targets need to check their input recursively.
16288
16289 // FIXME: Does this apply with clamp? It's implemented with max.
16290 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
16291 if (!isCanonicalized(DAG, Op: Op.getOperand(i: I), QueryVT, UserFlags,
16292 MaxDepth: MaxDepth - 1))
16293 return false;
16294 }
16295
16296 return true;
16297 }
16298 case ISD::SELECT: {
16299 return isCanonicalized(DAG, Op: Op.getOperand(i: 1), QueryVT, UserFlags,
16300 MaxDepth: MaxDepth - 1) &&
16301 isCanonicalized(DAG, Op: Op.getOperand(i: 2), QueryVT, UserFlags,
16302 MaxDepth: MaxDepth - 1);
16303 }
16304 case ISD::BUILD_VECTOR: {
16305 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
16306 SDValue SrcOp = Op.getOperand(i);
16307 if (!isCanonicalized(DAG, Op: SrcOp, QueryVT, UserFlags, MaxDepth: MaxDepth - 1))
16308 return false;
16309 }
16310
16311 return true;
16312 }
16313 case ISD::EXTRACT_VECTOR_ELT:
16314 case ISD::EXTRACT_SUBVECTOR: {
16315 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), QueryVT, UserFlags,
16316 MaxDepth: MaxDepth - 1);
16317 }
16318 case ISD::INSERT_VECTOR_ELT: {
16319 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), QueryVT, UserFlags,
16320 MaxDepth: MaxDepth - 1) &&
16321 isCanonicalized(DAG, Op: Op.getOperand(i: 1), QueryVT, UserFlags,
16322 MaxDepth: MaxDepth - 1);
16323 }
16324 case ISD::POISON:
16325 return true;
16326 case ISD::UNDEF:
16327 // Could be anything.
16328 return false;
16329
16330 case ISD::BITCAST: {
16331 // Carry QueryVT through the bitcast unchanged. The top-of-function guard
16332 // rejects a source whose FP format differs from the consumed type, so a
16333 // value canonical in one FP format is not assumed canonical in another.
16334 SDValue Src = peekThroughBitcasts(V: Op.getOperand(i: 0));
16335 return isCanonicalized(DAG, Op: Src, QueryVT, UserFlags, MaxDepth: MaxDepth - 1);
16336 }
16337 case ISD::TRUNCATE: {
16338 // Hack round the mess we make when legalizing extract_vector_elt
16339 if (Op.getValueType() == MVT::i16) {
16340 SDValue TruncSrc = Op.getOperand(i: 0);
16341 if (TruncSrc.getValueType() == MVT::i32 &&
16342 TruncSrc.getOpcode() == ISD::BITCAST &&
16343 TruncSrc.getOperand(i: 0).getValueType() == MVT::v2f16) {
16344 return isCanonicalized(DAG, Op: TruncSrc.getOperand(i: 0), QueryVT, UserFlags,
16345 MaxDepth: MaxDepth - 1);
16346 }
16347 }
16348 return false;
16349 }
16350 case ISD::INTRINSIC_WO_CHAIN: {
16351 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
16352 // TODO: Handle more intrinsics
16353 switch (IntrinsicID) {
16354 case Intrinsic::amdgcn_cvt_pkrtz:
16355 case Intrinsic::amdgcn_cubeid:
16356 case Intrinsic::amdgcn_frexp_mant:
16357 case Intrinsic::amdgcn_fdot2:
16358 case Intrinsic::amdgcn_rcp:
16359 case Intrinsic::amdgcn_rsq:
16360 case Intrinsic::amdgcn_rsq_clamp:
16361 case Intrinsic::amdgcn_rcp_legacy:
16362 case Intrinsic::amdgcn_rsq_legacy:
16363 case Intrinsic::amdgcn_trig_preop:
16364 case Intrinsic::amdgcn_tanh:
16365 case Intrinsic::amdgcn_log:
16366 case Intrinsic::amdgcn_exp2:
16367 case Intrinsic::amdgcn_sqrt:
16368 return true;
16369 default:
16370 break;
16371 }
16372
16373 break;
16374 }
16375 default:
16376 break;
16377 }
16378
16379 // FIXME: denormalsEnabledForType is broken for dynamic
16380 return denormalsEnabledForType(DAG, VT: Op.getValueType()) &&
16381 (UserFlags.hasNoNaNs() || DAG.isKnownNeverSNaN(Op));
16382}
16383
16384bool SITargetLowering::isCanonicalized(Register Reg, const MachineFunction &MF,
16385 unsigned MaxDepth) const {
16386 const MachineRegisterInfo &MRI = MF.getRegInfo();
16387 MachineInstr *MI = MRI.getVRegDef(Reg);
16388 unsigned Opcode = MI->getOpcode();
16389
16390 if (Opcode == AMDGPU::G_FCANONICALIZE)
16391 return true;
16392
16393 std::optional<FPValueAndVReg> FCR;
16394 // Constant splat (can be padded with undef) or scalar constant.
16395 if (mi_match(R: Reg, MRI, P: MIPatternMatch::m_GFCstOrSplat(FPValReg&: FCR))) {
16396 if (FCR->Value.isSignaling())
16397 return false;
16398 if (!FCR->Value.isDenormal())
16399 return true;
16400
16401 DenormalMode Mode = MF.getDenormalMode(FPType: FCR->Value.getSemantics());
16402 return Mode == DenormalMode::getIEEE();
16403 }
16404
16405 if (MaxDepth == 0)
16406 return false;
16407
16408 switch (Opcode) {
16409 case AMDGPU::G_FADD:
16410 case AMDGPU::G_FSUB:
16411 case AMDGPU::G_FMUL:
16412 case AMDGPU::G_FCEIL:
16413 case AMDGPU::G_FFLOOR:
16414 case AMDGPU::G_FRINT:
16415 case AMDGPU::G_FNEARBYINT:
16416 case AMDGPU::G_INTRINSIC_FPTRUNC_ROUND:
16417 case AMDGPU::G_INTRINSIC_TRUNC:
16418 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
16419 case AMDGPU::G_FMA:
16420 case AMDGPU::G_FMAD:
16421 case AMDGPU::G_FSQRT:
16422 case AMDGPU::G_FDIV:
16423 case AMDGPU::G_FREM:
16424 case AMDGPU::G_FPOW:
16425 case AMDGPU::G_FPEXT:
16426 case AMDGPU::G_FLOG:
16427 case AMDGPU::G_FLOG2:
16428 case AMDGPU::G_FLOG10:
16429 case AMDGPU::G_FPTRUNC:
16430 case AMDGPU::G_AMDGPU_RCP_IFLAG:
16431 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
16432 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
16433 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
16434 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
16435 return true;
16436 case AMDGPU::G_FNEG:
16437 case AMDGPU::G_FABS:
16438 case AMDGPU::G_FCOPYSIGN:
16439 return isCanonicalized(Reg: MI->getOperand(i: 1).getReg(), MF, MaxDepth: MaxDepth - 1);
16440 case AMDGPU::G_FMINNUM:
16441 case AMDGPU::G_FMAXNUM:
16442 case AMDGPU::G_FMINNUM_IEEE:
16443 case AMDGPU::G_FMAXNUM_IEEE:
16444 case AMDGPU::G_FMINIMUM:
16445 case AMDGPU::G_FMAXIMUM:
16446 case AMDGPU::G_FMINIMUMNUM:
16447 case AMDGPU::G_FMAXIMUMNUM: {
16448 if (Subtarget->supportsMinMaxDenormModes() ||
16449 // FIXME: denormalsEnabledForType is broken for dynamic
16450 denormalsEnabledForType(Ty: MRI.getType(Reg), MF))
16451 return true;
16452
16453 [[fallthrough]];
16454 }
16455 case AMDGPU::G_BUILD_VECTOR:
16456 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands()))
16457 if (!isCanonicalized(Reg: MO.getReg(), MF, MaxDepth: MaxDepth - 1))
16458 return false;
16459 return true;
16460 case AMDGPU::G_INTRINSIC:
16461 case AMDGPU::G_INTRINSIC_CONVERGENT:
16462 switch (cast<GIntrinsic>(Val: MI)->getIntrinsicID()) {
16463 case Intrinsic::amdgcn_fmul_legacy:
16464 case Intrinsic::amdgcn_fmad_ftz:
16465 case Intrinsic::amdgcn_sqrt:
16466 case Intrinsic::amdgcn_fmed3:
16467 case Intrinsic::amdgcn_sin:
16468 case Intrinsic::amdgcn_cos:
16469 case Intrinsic::amdgcn_log:
16470 case Intrinsic::amdgcn_exp2:
16471 case Intrinsic::amdgcn_log_clamp:
16472 case Intrinsic::amdgcn_rcp:
16473 case Intrinsic::amdgcn_rcp_legacy:
16474 case Intrinsic::amdgcn_rsq:
16475 case Intrinsic::amdgcn_rsq_clamp:
16476 case Intrinsic::amdgcn_rsq_legacy:
16477 case Intrinsic::amdgcn_div_scale:
16478 case Intrinsic::amdgcn_div_fmas:
16479 case Intrinsic::amdgcn_div_fixup:
16480 case Intrinsic::amdgcn_fract:
16481 case Intrinsic::amdgcn_cvt_pkrtz:
16482 case Intrinsic::amdgcn_cubeid:
16483 case Intrinsic::amdgcn_cubema:
16484 case Intrinsic::amdgcn_cubesc:
16485 case Intrinsic::amdgcn_cubetc:
16486 case Intrinsic::amdgcn_frexp_mant:
16487 case Intrinsic::amdgcn_fdot2:
16488 case Intrinsic::amdgcn_trig_preop:
16489 case Intrinsic::amdgcn_tanh:
16490 return true;
16491 default:
16492 break;
16493 }
16494
16495 [[fallthrough]];
16496 default:
16497 return false;
16498 }
16499
16500 llvm_unreachable("invalid operation");
16501}
16502
16503// Constant fold canonicalize.
16504SDValue SITargetLowering::getCanonicalConstantFP(SelectionDAG &DAG,
16505 const SDLoc &SL, EVT VT,
16506 const APFloat &C) const {
16507 // Flush denormals to 0 if not enabled.
16508 if (C.isDenormal()) {
16509 DenormalMode Mode =
16510 DAG.getMachineFunction().getDenormalMode(FPType: C.getSemantics());
16511 if (Mode == DenormalMode::getPreserveSign()) {
16512 return DAG.getConstantFP(
16513 Val: APFloat::getZero(Sem: C.getSemantics(), Negative: C.isNegative()), DL: SL, VT);
16514 }
16515
16516 if (Mode != DenormalMode::getIEEE())
16517 return SDValue();
16518 }
16519
16520 if (C.isNaN()) {
16521 if (C.isSignaling()) {
16522 // Quiet a signaling NaN.
16523 return DAG.getConstantFP(Val: C.makeQuiet(), DL: SL, VT);
16524 }
16525 }
16526
16527 // Already canonical.
16528 return DAG.getConstantFP(Val: C, DL: SL, VT);
16529}
16530
16531static bool vectorEltWillFoldAway(SDValue Op) {
16532 return Op.isUndef() || isa<ConstantFPSDNode>(Val: Op);
16533}
16534
16535SDValue
16536SITargetLowering::performFCanonicalizeCombine(SDNode *N,
16537 DAGCombinerInfo &DCI) const {
16538 SelectionDAG &DAG = DCI.DAG;
16539 SDValue N0 = N->getOperand(Num: 0);
16540 EVT VT = N->getValueType(ResNo: 0);
16541
16542 // fcanonicalize undef -> qnan
16543 if (N0.isUndef()) {
16544 APFloat QNaN = APFloat::getQNaN(Sem: VT.getFltSemantics());
16545 return DAG.getConstantFP(Val: QNaN, DL: SDLoc(N), VT);
16546 }
16547
16548 if (ConstantFPSDNode *CFP = isConstOrConstSplatFP(N: N0))
16549 return getCanonicalConstantFP(DAG, SL: SDLoc(N), VT, C: CFP->getValueAPF());
16550
16551 // fcanonicalize (build_vector x, k) -> build_vector (fcanonicalize x),
16552 // (fcanonicalize k)
16553 //
16554 // fcanonicalize (build_vector x, undef) -> build_vector (fcanonicalize x), 0
16555
16556 // TODO: This could be better with wider vectors that will be split to v2f16,
16557 // and to consider uses since there aren't that many packed operations.
16558 if (N0.getOpcode() == ISD::BUILD_VECTOR && N0.getNumOperands() == 2 &&
16559 isTypeLegal(VT)) {
16560 SDLoc SL(N);
16561 SDValue NewElts[2];
16562 SDValue Lo = N0.getOperand(i: 0);
16563 SDValue Hi = N0.getOperand(i: 1);
16564 EVT EltVT = Lo.getValueType();
16565
16566 // Only apply this optimization if scalar canonicalize is legal for the
16567 // element type. Otherwise, scalarizing may require widening the scalar back
16568 // to a vector, adding overhead (e.g., bf16 has no scalar instructions).
16569 if (getOperationAction(Op: ISD::FCANONICALIZE, VT: EltVT) != Legal)
16570 return SDValue();
16571
16572 if (vectorEltWillFoldAway(Op: Lo) || vectorEltWillFoldAway(Op: Hi)) {
16573 for (unsigned I = 0; I != 2; ++I) {
16574 SDValue Op = N0.getOperand(i: I);
16575 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
16576 NewElts[I] =
16577 getCanonicalConstantFP(DAG, SL, VT: EltVT, C: CFP->getValueAPF());
16578 } else if (Op.isUndef()) {
16579 // Handled below based on what the other operand is.
16580 NewElts[I] = Op;
16581 } else {
16582 NewElts[I] = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: SL, VT: EltVT, Operand: Op);
16583 }
16584 }
16585
16586 // If one half is undef, and one is constant, prefer a splat vector.
16587 // Otherwise, convert the undef to 0.0 since that's cheaper to use and may
16588 // be free with a packed operation.
16589 if (NewElts[0].isUndef()) {
16590 NewElts[0] = isa<ConstantFPSDNode>(Val: NewElts[1])
16591 ? NewElts[1]
16592 : DAG.getConstantFP(Val: 0.0f, DL: SL, VT: EltVT);
16593 }
16594
16595 if (NewElts[1].isUndef()) {
16596 NewElts[1] = isa<ConstantFPSDNode>(Val: NewElts[0])
16597 ? NewElts[0]
16598 : DAG.getConstantFP(Val: 0.0f, DL: SL, VT: EltVT);
16599 }
16600
16601 return DAG.getBuildVector(VT, DL: SL, Ops: NewElts);
16602 }
16603 }
16604
16605 return SDValue();
16606}
16607
16608static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc) {
16609 switch (Opc) {
16610 case ISD::FMAXNUM:
16611 case ISD::FMAXNUM_IEEE:
16612 case ISD::FMAXIMUMNUM:
16613 return AMDGPUISD::FMAX3;
16614 case ISD::FMAXIMUM:
16615 return AMDGPUISD::FMAXIMUM3;
16616 case ISD::SMAX:
16617 return AMDGPUISD::SMAX3;
16618 case ISD::UMAX:
16619 return AMDGPUISD::UMAX3;
16620 case ISD::FMINNUM:
16621 case ISD::FMINNUM_IEEE:
16622 case ISD::FMINIMUMNUM:
16623 return AMDGPUISD::FMIN3;
16624 case ISD::FMINIMUM:
16625 return AMDGPUISD::FMINIMUM3;
16626 case ISD::SMIN:
16627 return AMDGPUISD::SMIN3;
16628 case ISD::UMIN:
16629 return AMDGPUISD::UMIN3;
16630 default:
16631 llvm_unreachable("Not a min/max opcode");
16632 }
16633}
16634
16635SDValue SITargetLowering::performIntMed3ImmCombine(SelectionDAG &DAG,
16636 const SDLoc &SL, SDValue Src,
16637 SDValue MinVal,
16638 SDValue MaxVal,
16639 bool Signed) const {
16640
16641 // med3 comes from
16642 // min(max(x, K0), K1), K0 < K1
16643 // max(min(x, K0), K1), K1 < K0
16644 //
16645 // "MinVal" and "MaxVal" respectively refer to the rhs of the
16646 // min/max op.
16647 ConstantSDNode *MinK = dyn_cast<ConstantSDNode>(Val&: MinVal);
16648 ConstantSDNode *MaxK = dyn_cast<ConstantSDNode>(Val&: MaxVal);
16649
16650 if (!MinK || !MaxK)
16651 return SDValue();
16652
16653 if (Signed) {
16654 if (MaxK->getAPIntValue().sge(RHS: MinK->getAPIntValue()))
16655 return SDValue();
16656 } else {
16657 if (MaxK->getAPIntValue().uge(RHS: MinK->getAPIntValue()))
16658 return SDValue();
16659 }
16660
16661 EVT VT = MinK->getValueType(ResNo: 0);
16662 unsigned Med3Opc = Signed ? AMDGPUISD::SMED3 : AMDGPUISD::UMED3;
16663 if (VT == MVT::i32 || (VT == MVT::i16 && Subtarget->hasMed3_16()))
16664 return DAG.getNode(Opcode: Med3Opc, DL: SL, VT, N1: Src, N2: MaxVal, N3: MinVal);
16665
16666 // Note: we could also extend to i32 and use i32 med3 if i16 med3 is
16667 // not available, but this is unlikely to be profitable as constants
16668 // will often need to be materialized & extended, especially on
16669 // pre-GFX10 where VOP3 instructions couldn't take literal operands.
16670 return SDValue();
16671}
16672
16673static ConstantFPSDNode *getSplatConstantFP(SDValue Op) {
16674 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op))
16675 return C;
16676
16677 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: Op)) {
16678 if (ConstantFPSDNode *C = BV->getConstantFPSplatNode())
16679 return C;
16680 }
16681
16682 return nullptr;
16683}
16684
16685SDValue SITargetLowering::performFPMed3ImmCombine(SelectionDAG &DAG,
16686 const SDLoc &SL, SDValue Op0,
16687 SDValue Op1,
16688 bool IsKnownNoNaNs) const {
16689 ConstantFPSDNode *K1 = getSplatConstantFP(Op: Op1);
16690 if (!K1)
16691 return SDValue();
16692
16693 ConstantFPSDNode *K0 = getSplatConstantFP(Op: Op0.getOperand(i: 1));
16694 if (!K0)
16695 return SDValue();
16696
16697 // Ordered >= (although NaN inputs should have folded away by now).
16698 if (K0->getValueAPF() > K1->getValueAPF())
16699 return SDValue();
16700
16701 // med3 with a nan input acts like
16702 // v_min_f32(v_min_f32(S0.f32, S1.f32), S2.f32)
16703 //
16704 // So the result depends on whether the IEEE mode bit is enabled or not with a
16705 // signaling nan input.
16706 // ieee=1
16707 // s0 snan: yields s2
16708 // s1 snan: yields s2
16709 // s2 snan: qnan
16710
16711 // s0 qnan: min(s1, s2)
16712 // s1 qnan: min(s0, s2)
16713 // s2 qnan: min(s0, s1)
16714
16715 // ieee=0
16716 // s0 snan: min(s1, s2)
16717 // s1 snan: min(s0, s2)
16718 // s2 snan: qnan
16719
16720 // s0 qnan: min(s1, s2)
16721 // s1 qnan: min(s0, s2)
16722 // s2 qnan: min(s0, s1)
16723 const MachineFunction &MF = DAG.getMachineFunction();
16724 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
16725
16726 // TODO: Check IEEE bit enabled. We can form fmed3 with IEEE=0 regardless of
16727 // whether the input is a signaling nan if op0 is fmaximum or fmaximumnum. We
16728 // can only form if op0 is fmaxnum_ieee if IEEE=1.
16729 EVT VT = Op0.getValueType();
16730 if (Info->getMode().DX10Clamp) {
16731 // If dx10_clamp is enabled, NaNs clamp to 0.0. This is the same as the
16732 // hardware fmed3 behavior converting to a min.
16733 // FIXME: Should this be allowing -0.0?
16734 if (K1->isOne() && K0->isPosZero())
16735 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Op0.getOperand(i: 0));
16736 }
16737
16738 // med3 for f16 is only available on gfx9+, and not available for v2f16.
16739 if (VT == MVT::f32 || (VT == MVT::f16 && Subtarget->hasMed3_16())) {
16740 // This isn't safe with signaling NaNs because in IEEE mode, min/max on a
16741 // signaling NaN gives a quiet NaN. The quiet NaN input to the min would
16742 // then give the other result, which is different from med3 with a NaN
16743 // input.
16744 SDValue Var = Op0.getOperand(i: 0);
16745 if (!IsKnownNoNaNs && !DAG.isKnownNeverSNaN(Op: Var))
16746 return SDValue();
16747
16748 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
16749
16750 if ((!K0->hasOneUse() || TII->isInlineConstant(Imm: K0->getValueAPF())) &&
16751 (!K1->hasOneUse() || TII->isInlineConstant(Imm: K1->getValueAPF()))) {
16752 return DAG.getNode(Opcode: AMDGPUISD::FMED3, DL: SL, VT: K0->getValueType(ResNo: 0), N1: Var,
16753 N2: SDValue(K0, 0), N3: SDValue(K1, 0));
16754 }
16755 }
16756
16757 return SDValue();
16758}
16759
16760/// \return true if the subtarget supports minimum3 and maximum3 with the given
16761/// base min/max opcode \p Opc for type \p VT.
16762static bool supportsMin3Max3(const GCNSubtarget &Subtarget, unsigned Opc,
16763 EVT VT) {
16764 switch (Opc) {
16765 case ISD::FMINNUM:
16766 case ISD::FMAXNUM:
16767 case ISD::FMINNUM_IEEE:
16768 case ISD::FMAXNUM_IEEE:
16769 case ISD::FMINIMUMNUM:
16770 case ISD::FMAXIMUMNUM:
16771 case AMDGPUISD::FMIN_LEGACY:
16772 case AMDGPUISD::FMAX_LEGACY:
16773 return (VT == MVT::f32) || (VT == MVT::f16 && Subtarget.hasMin3Max3_16()) ||
16774 (VT == MVT::v2f16 && Subtarget.hasMin3Max3PKF16());
16775 case ISD::FMINIMUM:
16776 case ISD::FMAXIMUM:
16777 return (VT == MVT::f32 && Subtarget.hasMinimum3Maximum3F32()) ||
16778 (VT == MVT::f16 && Subtarget.hasMinimum3Maximum3F16()) ||
16779 (VT == MVT::v2f16 && Subtarget.hasMinimum3Maximum3PKF16());
16780 case ISD::SMAX:
16781 case ISD::SMIN:
16782 case ISD::UMAX:
16783 case ISD::UMIN:
16784 return (VT == MVT::i32) || (VT == MVT::i16 && Subtarget.hasMin3Max3_16());
16785 default:
16786 return false;
16787 }
16788
16789 llvm_unreachable("not a min/max opcode");
16790}
16791
16792SDValue SITargetLowering::performMinMaxCombine(SDNode *N,
16793 DAGCombinerInfo &DCI) const {
16794 SelectionDAG &DAG = DCI.DAG;
16795
16796 EVT VT = N->getValueType(ResNo: 0);
16797 unsigned Opc = N->getOpcode();
16798 SDValue Op0 = N->getOperand(Num: 0);
16799 SDValue Op1 = N->getOperand(Num: 1);
16800
16801 // Only do this if the inner op has one use since this will just increases
16802 // register pressure for no benefit.
16803
16804 if (supportsMin3Max3(Subtarget: *Subtarget, Opc, VT)) {
16805 auto IsTreeWithCombinableChildren = [Opc](SDValue Op) {
16806 return (Op.getOperand(i: 0).getOpcode() == Opc &&
16807 Op.getOperand(i: 0).hasOneUse()) ||
16808 (Op.getOperand(i: 1).getOpcode() == Opc &&
16809 Op.getOperand(i: 1).hasOneUse());
16810 };
16811
16812 bool CanTreeCombineApply = Op0.getOpcode() == Opc && Op0.hasOneUse() &&
16813 Op1.getOpcode() == Opc && Op1.hasOneUse();
16814 bool HasCombinableTreeChild =
16815 CanTreeCombineApply && (IsTreeWithCombinableChildren(Op0) ||
16816 IsTreeWithCombinableChildren(Op1));
16817
16818 // Tree reduction: when both operands are the same min/max op, restructure
16819 // to keep a 2-op node on top so higher tree levels can still combine.
16820 //
16821 // max(max(a, b), max(c, d)) -> max(max3(a, b, c), d)
16822 // min(min(a, b), min(c, d)) -> min(min3(a, b, c), d)
16823 //
16824 // Defer when either inner op is a tree node with combinable children.
16825 if (CanTreeCombineApply && !HasCombinableTreeChild) {
16826 SDLoc DL(N);
16827 SDValue Inner =
16828 DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT, N1: Op0.getOperand(i: 0),
16829 N2: Op0.getOperand(i: 1), N3: Op1.getOperand(i: 0));
16830 return DAG.getNode(Opcode: Opc, DL, VT, N1: Inner, N2: Op1.getOperand(i: 1));
16831 }
16832
16833 // max(max(a, b), c) -> max3(a, b, c)
16834 // min(min(a, b), c) -> min3(a, b, c)
16835 // Deferred when Op0 is a tree node with combinable children.
16836 if (Op0.getOpcode() == Opc && Op0.hasOneUse() && !HasCombinableTreeChild) {
16837 SDLoc DL(N);
16838 return DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT: N->getValueType(ResNo: 0),
16839 N1: Op0.getOperand(i: 0), N2: Op0.getOperand(i: 1), N3: Op1);
16840 }
16841
16842 // Try commuted.
16843 // max(a, max(b, c)) -> max3(a, b, c)
16844 // min(a, min(b, c)) -> min3(a, b, c)
16845 // Deferred when Op1 is a tree node with combinable children.
16846 if (Op1.getOpcode() == Opc && Op1.hasOneUse() && !HasCombinableTreeChild) {
16847 SDLoc DL(N);
16848 return DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT: N->getValueType(ResNo: 0),
16849 N1: Op0, N2: Op1.getOperand(i: 0), N3: Op1.getOperand(i: 1));
16850 }
16851 }
16852
16853 // umin(sffbh(x), bitwidth) -> sffbh(x) if x is known to be not 0 or -1.
16854 SDValue FfbhSrc;
16855 uint64_t Clamp = 0;
16856 if (Opc == ISD::UMIN &&
16857 sd_match(N: Op0,
16858 P: m_IntrinsicWOChain<Intrinsic::amdgcn_sffbh>(Opnds: m_Value(N&: FfbhSrc))) &&
16859 sd_match(N: Op1, P: m_ConstInt(V&: Clamp))) {
16860 unsigned BitWidth = FfbhSrc.getValueType().getScalarSizeInBits();
16861 if (Clamp >= BitWidth) {
16862 KnownBits Known = DAG.computeKnownBits(Op: FfbhSrc);
16863 if (Known.isNonZero() && Known.Zero.getBoolValue())
16864 return Op0;
16865 }
16866 }
16867
16868 // min(max(x, K0), K1), K0 < K1 -> med3(x, K0, K1)
16869 // max(min(x, K0), K1), K1 < K0 -> med3(x, K1, K0)
16870 if (Opc == ISD::SMIN && Op0.getOpcode() == ISD::SMAX && Op0.hasOneUse()) {
16871 if (SDValue Med3 = performIntMed3ImmCombine(
16872 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op1, MaxVal: Op0->getOperand(Num: 1), Signed: true))
16873 return Med3;
16874 }
16875 if (Opc == ISD::SMAX && Op0.getOpcode() == ISD::SMIN && Op0.hasOneUse()) {
16876 if (SDValue Med3 = performIntMed3ImmCombine(
16877 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op0->getOperand(Num: 1), MaxVal: Op1, Signed: true))
16878 return Med3;
16879 }
16880
16881 if (Opc == ISD::UMIN && Op0.getOpcode() == ISD::UMAX && Op0.hasOneUse()) {
16882 if (SDValue Med3 = performIntMed3ImmCombine(
16883 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op1, MaxVal: Op0->getOperand(Num: 1), Signed: false))
16884 return Med3;
16885 }
16886 if (Opc == ISD::UMAX && Op0.getOpcode() == ISD::UMIN && Op0.hasOneUse()) {
16887 if (SDValue Med3 = performIntMed3ImmCombine(
16888 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op0->getOperand(Num: 1), MaxVal: Op1, Signed: false))
16889 return Med3;
16890 }
16891
16892 // if !is_snan(x):
16893 // fminnum(fmaxnum(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16894 // fminnum_ieee(fmaxnum_ieee(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16895 // fminnumnum(fmaxnumnum(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16896 // fmin_legacy(fmax_legacy(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16897 if (((Opc == ISD::FMINNUM && Op0.getOpcode() == ISD::FMAXNUM) ||
16898 (Opc == ISD::FMINNUM_IEEE && Op0.getOpcode() == ISD::FMAXNUM_IEEE) ||
16899 (Opc == ISD::FMINIMUMNUM && Op0.getOpcode() == ISD::FMAXIMUMNUM) ||
16900 (Opc == AMDGPUISD::FMIN_LEGACY &&
16901 Op0.getOpcode() == AMDGPUISD::FMAX_LEGACY)) &&
16902 (VT == MVT::f32 || VT == MVT::f64 ||
16903 (VT == MVT::f16 && Subtarget->has16BitInsts()) ||
16904 (VT == MVT::bf16 && Subtarget->hasBF16PackedInsts()) ||
16905 (VT == MVT::v2bf16 && Subtarget->hasBF16PackedInsts()) ||
16906 (VT == MVT::v2f16 && Subtarget->hasVOP3PInsts())) &&
16907 Op0.hasOneUse()) {
16908 if (SDValue Res = performFPMed3ImmCombine(DAG, SL: SDLoc(N), Op0, Op1,
16909 IsKnownNoNaNs: N->getFlags().hasNoNaNs()))
16910 return Res;
16911 }
16912
16913 // Prefer fminnum_ieee over fminimum. For gfx950, minimum/maximum are legal
16914 // for some types, but at a higher cost since it's implemented with a 3
16915 // operand form.
16916 const SDNodeFlags Flags = N->getFlags();
16917 if ((Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM) && Flags.hasNoNaNs() &&
16918 !Subtarget->hasIEEEMinimumMaximumInsts() &&
16919 isOperationLegal(Op: ISD::FMINNUM_IEEE, VT: VT.getScalarType())) {
16920 unsigned NewOpc =
16921 Opc == ISD::FMINIMUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
16922 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(N), VT, N1: Op0, N2: Op1, Flags);
16923 }
16924
16925 return SDValue();
16926}
16927
16928static bool isClampZeroToOne(SDValue A, SDValue B) {
16929 if (ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A)) {
16930 if (ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B)) {
16931 // FIXME: Should this be allowing -0.0?
16932 return (CA->isPosZero() && CB->isOne()) ||
16933 (CA->isOne() && CB->isPosZero());
16934 }
16935 }
16936
16937 return false;
16938}
16939
16940// FIXME: Should only worry about snans for version with chain.
16941SDValue SITargetLowering::performFMed3Combine(SDNode *N,
16942 DAGCombinerInfo &DCI) const {
16943 EVT VT = N->getValueType(ResNo: 0);
16944 // v_med3_f32 and v_max_f32 behave identically wrt denorms, exceptions and
16945 // NaNs. With a NaN input, the order of the operands may change the result.
16946
16947 SelectionDAG &DAG = DCI.DAG;
16948 SDLoc SL(N);
16949
16950 SDValue Src0 = N->getOperand(Num: 0);
16951 SDValue Src1 = N->getOperand(Num: 1);
16952 SDValue Src2 = N->getOperand(Num: 2);
16953
16954 if (isClampZeroToOne(A: Src0, B: Src1)) {
16955 // const_a, const_b, x -> clamp is safe in all cases including signaling
16956 // nans.
16957 // FIXME: Should this be allowing -0.0?
16958 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Src2);
16959 }
16960
16961 const MachineFunction &MF = DAG.getMachineFunction();
16962 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
16963
16964 // FIXME: dx10_clamp behavior assumed in instcombine. Should we really bother
16965 // handling no dx10-clamp?
16966 if (Info->getMode().DX10Clamp) {
16967 // If NaNs is clamped to 0, we are free to reorder the inputs.
16968
16969 if (isa<ConstantFPSDNode>(Val: Src0) && !isa<ConstantFPSDNode>(Val: Src1))
16970 std::swap(a&: Src0, b&: Src1);
16971
16972 if (isa<ConstantFPSDNode>(Val: Src1) && !isa<ConstantFPSDNode>(Val: Src2))
16973 std::swap(a&: Src1, b&: Src2);
16974
16975 if (isa<ConstantFPSDNode>(Val: Src0) && !isa<ConstantFPSDNode>(Val: Src1))
16976 std::swap(a&: Src0, b&: Src1);
16977
16978 if (isClampZeroToOne(A: Src1, B: Src2))
16979 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Src0);
16980 }
16981
16982 return SDValue();
16983}
16984
16985SDValue SITargetLowering::performCvtPkRTZCombine(SDNode *N,
16986 DAGCombinerInfo &DCI) const {
16987 SDValue Src0 = N->getOperand(Num: 0);
16988 SDValue Src1 = N->getOperand(Num: 1);
16989 if (Src0.isUndef() && Src1.isUndef())
16990 return DCI.DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
16991 return SDValue();
16992}
16993
16994// Check if EXTRACT_VECTOR_ELT/INSERT_VECTOR_ELT (<n x e>, var-idx) should be
16995// expanded into a set of cmp/select instructions.
16996bool SITargetLowering::shouldExpandVectorDynExt(unsigned EltSize,
16997 unsigned NumElem,
16998 bool IsDivergentIdx,
16999 const GCNSubtarget *Subtarget) {
17000 if (UseDivergentRegisterIndexing)
17001 return false;
17002
17003 unsigned VecSize = EltSize * NumElem;
17004
17005 // Sub-dword vectors of size 2 dword or less have better implementation.
17006 if (VecSize <= 64 && EltSize < 32)
17007 return false;
17008
17009 // Always expand the rest of sub-dword instructions, otherwise it will be
17010 // lowered via memory.
17011 if (EltSize < 32)
17012 return true;
17013
17014 // Always do this if var-idx is divergent, otherwise it will become a loop.
17015 if (IsDivergentIdx)
17016 return true;
17017
17018 // Large vectors would yield too many compares and v_cndmask_b32 instructions.
17019 unsigned NumInsts = NumElem /* Number of compares */ +
17020 ((EltSize + 31) / 32) * NumElem /* Number of cndmasks */;
17021
17022 // On some architectures (GFX9) movrel is not available and it's better
17023 // to expand.
17024 if (Subtarget->useVGPRIndexMode())
17025 return NumInsts <= 16;
17026
17027 // If movrel is available, use it instead of expanding for vector of 8
17028 // elements.
17029 if (Subtarget->hasMovrel())
17030 return NumInsts <= 15;
17031
17032 return true;
17033}
17034
17035bool SITargetLowering::shouldExpandVectorDynExt(SDNode *N) const {
17036 SDValue Idx = N->getOperand(Num: N->getNumOperands() - 1);
17037 if (isa<ConstantSDNode>(Val: Idx))
17038 return false;
17039
17040 SDValue Vec = N->getOperand(Num: 0);
17041 EVT VecVT = Vec.getValueType();
17042 EVT EltVT = VecVT.getVectorElementType();
17043 unsigned EltSize = EltVT.getSizeInBits();
17044 unsigned NumElem = VecVT.getVectorNumElements();
17045
17046 return SITargetLowering::shouldExpandVectorDynExt(
17047 EltSize, NumElem, IsDivergentIdx: Idx->isDivergent(), Subtarget: getSubtarget());
17048}
17049
17050SDValue
17051SITargetLowering::performExtractVectorEltCombine(SDNode *N,
17052 DAGCombinerInfo &DCI) const {
17053 SDValue Vec = N->getOperand(Num: 0);
17054 SelectionDAG &DAG = DCI.DAG;
17055
17056 EVT VecVT = Vec.getValueType();
17057 EVT VecEltVT = VecVT.getVectorElementType();
17058 EVT ResVT = N->getValueType(ResNo: 0);
17059
17060 unsigned VecSize = VecVT.getSizeInBits();
17061 unsigned VecEltSize = VecEltVT.getSizeInBits();
17062
17063 if ((Vec.getOpcode() == ISD::FNEG || Vec.getOpcode() == ISD::FABS) &&
17064 allUsesHaveSourceMods(N)) {
17065 SDLoc SL(N);
17066 SDValue Idx = N->getOperand(Num: 1);
17067 SDValue Elt =
17068 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT, N1: Vec.getOperand(i: 0), N2: Idx);
17069 return DAG.getNode(Opcode: Vec.getOpcode(), DL: SL, VT: ResVT, Operand: Elt);
17070 }
17071
17072 // (extract_vector_element (and {y0, y1}, (build_vector 0x1f, 0x1f)), index)
17073 // -> (and (extract_vector_element {y0, y1}, index), 0x1f)
17074 // There are optimisations to transform 64-bit shifts into 32-bit shifts
17075 // depending on the shift operand. See e.g. performSraCombine().
17076 // This combine ensures that the optimisation is compatible with v2i32
17077 // legalised AND.
17078 if (VecVT == MVT::v2i32 && Vec->getOpcode() == ISD::AND &&
17079 Vec->getOperand(Num: 1)->getOpcode() == ISD::BUILD_VECTOR) {
17080
17081 const ConstantSDNode *C = isConstOrConstSplat(N: Vec.getOperand(i: 1));
17082 if (!C || C->getZExtValue() != 0x1f)
17083 return SDValue();
17084
17085 SDLoc SL(N);
17086 SDValue AndMask = DAG.getConstant(Val: 0x1f, DL: SL, VT: MVT::i32);
17087 SDValue EVE = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32,
17088 N1: Vec->getOperand(Num: 0), N2: N->getOperand(Num: 1));
17089 SDValue A = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: EVE, N2: AndMask);
17090 DAG.ReplaceAllUsesWith(From: N, To: A.getNode());
17091 }
17092
17093 // ScalarRes = EXTRACT_VECTOR_ELT ((vector-BINOP Vec1, Vec2), Idx)
17094 // =>
17095 // Vec1Elt = EXTRACT_VECTOR_ELT(Vec1, Idx)
17096 // Vec2Elt = EXTRACT_VECTOR_ELT(Vec2, Idx)
17097 // ScalarRes = scalar-BINOP Vec1Elt, Vec2Elt
17098 if (Vec.hasOneUse() && DCI.isBeforeLegalize() && VecEltVT == ResVT) {
17099 SDLoc SL(N);
17100 SDValue Idx = N->getOperand(Num: 1);
17101 unsigned Opc = Vec.getOpcode();
17102
17103 switch (Opc) {
17104 default:
17105 break;
17106 // TODO: Support other binary operations.
17107 case ISD::FADD:
17108 case ISD::FSUB:
17109 case ISD::FMUL:
17110 case ISD::ADD:
17111 case ISD::UMIN:
17112 case ISD::UMAX:
17113 case ISD::SMIN:
17114 case ISD::SMAX:
17115 case ISD::FMAXNUM:
17116 case ISD::FMINNUM:
17117 case ISD::FMAXNUM_IEEE:
17118 case ISD::FMINNUM_IEEE:
17119 case ISD::FMAXIMUM:
17120 case ISD::FMINIMUM: {
17121 SDValue Elt0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT,
17122 N1: Vec.getOperand(i: 0), N2: Idx);
17123 SDValue Elt1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT,
17124 N1: Vec.getOperand(i: 1), N2: Idx);
17125
17126 DCI.AddToWorklist(N: Elt0.getNode());
17127 DCI.AddToWorklist(N: Elt1.getNode());
17128 return DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT, N1: Elt0, N2: Elt1, Flags: Vec->getFlags());
17129 }
17130 }
17131 }
17132
17133 // EXTRACT_VECTOR_ELT (<n x e>, var-idx) => n x select (e, const-idx)
17134 if (shouldExpandVectorDynExt(N)) {
17135 SDLoc SL(N);
17136 SDValue Idx = N->getOperand(Num: 1);
17137 SDValue V;
17138 for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
17139 SDValue IC = DAG.getVectorIdxConstant(Val: I, DL: SL);
17140 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT, N1: Vec, N2: IC);
17141 if (I == 0)
17142 V = Elt;
17143 else
17144 V = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IC, True: Elt, False: V, Cond: ISD::SETEQ);
17145 }
17146 return V;
17147 }
17148
17149 // EXTRACT_VECTOR_ELT (v2i32 bitcast (i64/f64:k), Idx)
17150 // =>
17151 // i32:Lo(k) if Idx == 0, or
17152 // i32:Hi(k) if Idx == 1
17153 auto *Idx = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
17154 if (Vec.getOpcode() == ISD::BITCAST && VecVT == MVT::v2i32 && Idx) {
17155 SDLoc SL(N);
17156 SDValue PeekThrough = Vec.getOperand(i: 0);
17157 auto *KImm = dyn_cast<ConstantSDNode>(Val&: PeekThrough);
17158 if (KImm && KImm->getValueType(ResNo: 0).getSizeInBits() == 64) {
17159 uint64_t KImmValue = KImm->getZExtValue();
17160 return DAG.getConstant(
17161 Val: (KImmValue >> (32 * Idx->getZExtValue())) & 0xffffffff, DL: SL, VT: MVT::i32);
17162 }
17163 auto *KFPImm = dyn_cast<ConstantFPSDNode>(Val&: PeekThrough);
17164 if (KFPImm && KFPImm->getValueType(ResNo: 0).getSizeInBits() == 64) {
17165 uint64_t KFPImmValue =
17166 KFPImm->getValueAPF().bitcastToAPInt().getZExtValue();
17167 return DAG.getConstant(Val: (KFPImmValue >> (32 * Idx->getZExtValue())) &
17168 0xffffffff,
17169 DL: SL, VT: MVT::i32);
17170 }
17171 }
17172
17173 if (!DCI.isBeforeLegalize())
17174 return SDValue();
17175
17176 // Try to turn sub-dword accesses of vectors into accesses of the same 32-bit
17177 // elements. This exposes more load reduction opportunities by replacing
17178 // multiple small extract_vector_elements with a single 32-bit extract.
17179 if (isa<MemSDNode>(Val: Vec) && VecEltSize <= 16 && VecEltVT.isByteSized() &&
17180 VecSize > 32 && VecSize % 32 == 0 && Idx) {
17181 EVT NewVT = getEquivalentMemType(Context&: *DAG.getContext(), VT: VecVT);
17182
17183 unsigned BitIndex = Idx->getZExtValue() * VecEltSize;
17184 unsigned EltIdx = BitIndex / 32;
17185 unsigned LeftoverBitIdx = BitIndex % 32;
17186 SDLoc SL(N);
17187
17188 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: Vec);
17189 DCI.AddToWorklist(N: Cast.getNode());
17190
17191 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Cast,
17192 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
17193 DCI.AddToWorklist(N: Elt.getNode());
17194 SDValue Srl = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: Elt,
17195 N2: DAG.getConstant(Val: LeftoverBitIdx, DL: SL, VT: MVT::i32));
17196 DCI.AddToWorklist(N: Srl.getNode());
17197
17198 EVT VecEltAsIntVT = VecEltVT.changeTypeToInteger();
17199 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: VecEltAsIntVT, Operand: Srl);
17200 DCI.AddToWorklist(N: Trunc.getNode());
17201
17202 if (VecEltVT == ResVT) {
17203 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecEltVT, Operand: Trunc);
17204 }
17205
17206 assert(ResVT.isScalarInteger());
17207 return DAG.getAnyExtOrTrunc(Op: Trunc, DL: SL, VT: ResVT);
17208 }
17209
17210 return SDValue();
17211}
17212
17213SDValue
17214SITargetLowering::performInsertVectorEltCombine(SDNode *N,
17215 DAGCombinerInfo &DCI) const {
17216 SDValue Vec = N->getOperand(Num: 0);
17217 SDValue Idx = N->getOperand(Num: 2);
17218 EVT VecVT = Vec.getValueType();
17219 EVT EltVT = VecVT.getVectorElementType();
17220
17221 // INSERT_VECTOR_ELT (<n x e>, var-idx)
17222 // => BUILD_VECTOR n x select (e, const-idx)
17223 if (!shouldExpandVectorDynExt(N))
17224 return SDValue();
17225
17226 SelectionDAG &DAG = DCI.DAG;
17227 SDLoc SL(N);
17228 SDValue Ins = N->getOperand(Num: 1);
17229 EVT IdxVT = Idx.getValueType();
17230
17231 SmallVector<SDValue, 16> Ops;
17232 for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
17233 SDValue IC = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
17234 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec, N2: IC);
17235 SDValue V = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IC, True: Ins, False: Elt, Cond: ISD::SETEQ);
17236 Ops.push_back(Elt: V);
17237 }
17238
17239 return DAG.getBuildVector(VT: VecVT, DL: SL, Ops);
17240}
17241
17242/// Return the source of an fp_extend from f16 to f32, or a converted FP
17243/// constant.
17244static SDValue strictFPExtFromF16(SelectionDAG &DAG, SDValue Src) {
17245 if (Src.getOpcode() == ISD::FP_EXTEND &&
17246 Src.getOperand(i: 0).getValueType() == MVT::f16) {
17247 return Src.getOperand(i: 0);
17248 }
17249
17250 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Src)) {
17251 APFloat Val = CFP->getValueAPF();
17252 bool LosesInfo = true;
17253 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
17254 if (!LosesInfo)
17255 return DAG.getConstantFP(Val, DL: SDLoc(Src), VT: MVT::f16);
17256 }
17257
17258 return SDValue();
17259}
17260
17261SDValue SITargetLowering::performFPRoundCombine(SDNode *N,
17262 DAGCombinerInfo &DCI) const {
17263 assert(Subtarget->has16BitInsts() && !Subtarget->hasMed3_16() &&
17264 "combine only useful on gfx8");
17265
17266 SDValue TruncSrc = N->getOperand(Num: 0);
17267 EVT VT = N->getValueType(ResNo: 0);
17268 if (VT != MVT::f16)
17269 return SDValue();
17270
17271 if (TruncSrc.getOpcode() != AMDGPUISD::FMED3 ||
17272 TruncSrc.getValueType() != MVT::f32 || !TruncSrc.hasOneUse())
17273 return SDValue();
17274
17275 SelectionDAG &DAG = DCI.DAG;
17276 SDLoc SL(N);
17277
17278 // Optimize f16 fmed3 pattern performed on f32. On gfx8 there is no f16 fmed3,
17279 // and expanding it with min/max saves 1 instruction vs. casting to f32 and
17280 // casting back.
17281
17282 // fptrunc (f32 (fmed3 (fpext f16:a, fpext f16:b, fpext f16:c))) =>
17283 // fmin(fmax(a, b), fmax(fmin(a, b), c))
17284 SDValue A = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 0));
17285 if (!A)
17286 return SDValue();
17287
17288 SDValue B = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 1));
17289 if (!B)
17290 return SDValue();
17291
17292 SDValue C = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 2));
17293 if (!C)
17294 return SDValue();
17295
17296 // This changes signaling nan behavior. If an input is a signaling nan, it
17297 // would have been quieted by the fpext originally. We don't care because
17298 // these are unconstrained ops. If we needed to insert quieting canonicalizes
17299 // we would be worse off than just doing the promotion.
17300 SDValue A1 = DAG.getNode(Opcode: ISD::FMINNUM_IEEE, DL: SL, VT, N1: A, N2: B);
17301 SDValue B1 = DAG.getNode(Opcode: ISD::FMAXNUM_IEEE, DL: SL, VT, N1: A, N2: B);
17302 SDValue C1 = DAG.getNode(Opcode: ISD::FMAXNUM_IEEE, DL: SL, VT, N1: A1, N2: C);
17303 return DAG.getNode(Opcode: ISD::FMINNUM_IEEE, DL: SL, VT, N1: B1, N2: C1);
17304}
17305
17306unsigned SITargetLowering::getFusedOpcode(const SelectionDAG &DAG,
17307 const SDNode *N0,
17308 const SDNode *N1) const {
17309 EVT VT = N0->getValueType(ResNo: 0);
17310
17311 // Only do this if we are not trying to support denormals. v_mad_f32 does not
17312 // support denormals ever.
17313 if (((VT == MVT::f32 &&
17314 denormalModeIsFlushAllF32(MF: DAG.getMachineFunction())) ||
17315 (VT == MVT::f16 && Subtarget->hasMadF16() &&
17316 denormalModeIsFlushAllF64F16(MF: DAG.getMachineFunction()))) &&
17317 isOperationLegal(Op: ISD::FMAD, VT))
17318 return ISD::FMAD;
17319
17320 if (N0->getFlags().hasAllowContract() && N1->getFlags().hasAllowContract() &&
17321 isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
17322 return ISD::FMA;
17323 }
17324
17325 return 0;
17326}
17327
17328// For a reassociatable opcode perform:
17329// op x, (op y, z) -> op (op x, z), y, if x and z are uniform
17330SDValue SITargetLowering::reassociateScalarOps(SDNode *N,
17331 SelectionDAG &DAG) const {
17332 EVT VT = N->getValueType(ResNo: 0);
17333 if (VT != MVT::i32 && VT != MVT::i64)
17334 return SDValue();
17335
17336 if (DAG.isBaseWithConstantOffset(Op: SDValue(N, 0)))
17337 return SDValue();
17338
17339 unsigned Opc = N->getOpcode();
17340 SDValue Op0 = N->getOperand(Num: 0);
17341 SDValue Op1 = N->getOperand(Num: 1);
17342
17343 if (!(Op0->isDivergent() ^ Op1->isDivergent()))
17344 return SDValue();
17345
17346 if (Op0->isDivergent())
17347 std::swap(a&: Op0, b&: Op1);
17348
17349 if (Op1.getOpcode() != Opc || !Op1.hasOneUse())
17350 return SDValue();
17351
17352 SDValue Op2 = Op1.getOperand(i: 1);
17353 Op1 = Op1.getOperand(i: 0);
17354 if (!(Op1->isDivergent() ^ Op2->isDivergent()))
17355 return SDValue();
17356
17357 if (Op1->isDivergent())
17358 std::swap(a&: Op1, b&: Op2);
17359
17360 SDLoc SL(N);
17361 SDValue Add1 = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: Op0, N2: Op1);
17362 return DAG.getNode(Opcode: Opc, DL: SL, VT, N1: Add1, N2: Op2);
17363}
17364
17365static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL, EVT VT,
17366 SDValue N0, SDValue N1, SDValue N2, bool Signed) {
17367 unsigned MadOpc = Signed ? AMDGPUISD::MAD_I64_I32 : AMDGPUISD::MAD_U64_U32;
17368 SDVTList VTs = DAG.getVTList(VT1: MVT::i64, VT2: MVT::i1);
17369 SDValue Mad = DAG.getNode(Opcode: MadOpc, DL: SL, VTList: VTs, N1: N0, N2: N1, N3: N2);
17370 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Mad);
17371}
17372
17373// Fold
17374// y = lshr i64 x, 32
17375// res = add (mul i64 y, Const), x where "Const" is a 64-bit constant
17376// with Const.hi == -1
17377// To
17378// res = mad_u64_u32 y.lo ,Const.lo, x.lo
17379static SDValue tryFoldMADwithSRL(SelectionDAG &DAG, const SDLoc &SL,
17380 SDValue MulLHS, SDValue MulRHS,
17381 SDValue AddRHS) {
17382 if (MulRHS.getOpcode() == ISD::SRL)
17383 std::swap(a&: MulLHS, b&: MulRHS);
17384
17385 if (MulLHS.getValueType() != MVT::i64 || MulLHS.getOpcode() != ISD::SRL)
17386 return SDValue();
17387
17388 ConstantSDNode *ShiftVal = dyn_cast<ConstantSDNode>(Val: MulLHS.getOperand(i: 1));
17389 if (!ShiftVal || ShiftVal->getAsZExtVal() != 32 ||
17390 MulLHS.getOperand(i: 0) != AddRHS)
17391 return SDValue();
17392
17393 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val: MulRHS.getNode());
17394 if (!Const || Hi_32(Value: Const->getZExtValue()) != uint32_t(-1))
17395 return SDValue();
17396
17397 SDValue ConstMul =
17398 DAG.getConstant(Val: Lo_32(Value: Const->getZExtValue()), DL: SL, VT: MVT::i32);
17399 return getMad64_32(DAG, SL, VT: MVT::i64,
17400 N0: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulLHS), N1: ConstMul,
17401 N2: DAG.getZeroExtendInReg(Op: AddRHS, DL: SL, VT: MVT::i32), Signed: false);
17402}
17403
17404// Fold (add (mul x, y), z) --> (mad_[iu]64_[iu]32 x, y, z) plus high
17405// multiplies, if any.
17406//
17407// Full 64-bit multiplies that feed into an addition are lowered here instead
17408// of using the generic expansion. The generic expansion ends up with
17409// a tree of ADD nodes that prevents us from using the "add" part of the
17410// MAD instruction. The expansion produced here results in a chain of ADDs
17411// instead of a tree.
17412SDValue SITargetLowering::tryFoldToMad64_32(SDNode *N,
17413 DAGCombinerInfo &DCI) const {
17414 assert(N->isAnyAdd());
17415
17416 SelectionDAG &DAG = DCI.DAG;
17417 EVT VT = N->getValueType(ResNo: 0);
17418 SDLoc SL(N);
17419 SDValue LHS = N->getOperand(Num: 0);
17420 SDValue RHS = N->getOperand(Num: 1);
17421
17422 if (VT.isVector())
17423 return SDValue();
17424
17425 // S_MUL_HI_[IU]32 was added in gfx9, which allows us to keep the overall
17426 // result in scalar registers for uniform values.
17427 if (!N->isDivergent() && Subtarget->hasSMulHi())
17428 return SDValue();
17429
17430 unsigned NumBits = VT.getScalarSizeInBits();
17431 if (NumBits <= 32 || NumBits > 64)
17432 return SDValue();
17433
17434 if (LHS.getOpcode() != ISD::MUL) {
17435 assert(RHS.getOpcode() == ISD::MUL);
17436 std::swap(a&: LHS, b&: RHS);
17437 }
17438
17439 // Avoid the fold if it would unduly increase the number of multiplies due to
17440 // multiple uses, except on hardware with full-rate multiply-add (which is
17441 // part of full-rate 64-bit ops).
17442 if (!Subtarget->hasFullRate64Ops()) {
17443 unsigned NumUsers = 0;
17444 for (SDNode *User : LHS->users()) {
17445 // There is a use that does not feed into addition, so the multiply can't
17446 // be removed. We prefer MUL + ADD + ADDC over MAD + MUL.
17447 if (!User->isAnyAdd())
17448 return SDValue();
17449
17450 // We prefer 2xMAD over MUL + 2xADD + 2xADDC (code density), and prefer
17451 // MUL + 3xADD + 3xADDC over 3xMAD.
17452 ++NumUsers;
17453 if (NumUsers >= 3)
17454 return SDValue();
17455 }
17456 }
17457
17458 SDValue MulLHS = LHS.getOperand(i: 0);
17459 SDValue MulRHS = LHS.getOperand(i: 1);
17460 SDValue AddRHS = RHS;
17461
17462 if (SDValue FoldedMAD = tryFoldMADwithSRL(DAG, SL, MulLHS, MulRHS, AddRHS))
17463 return FoldedMAD;
17464
17465 // Always check whether operands are small unsigned values, since that
17466 // knowledge is useful in more cases. Check for small signed values only if
17467 // doing so can unlock a shorter code sequence.
17468 bool MulLHSUnsigned32 = numBitsUnsigned(Op: MulLHS, DAG) <= 32;
17469 bool MulRHSUnsigned32 = numBitsUnsigned(Op: MulRHS, DAG) <= 32;
17470
17471 bool MulSignedLo = false;
17472 if (!MulLHSUnsigned32 || !MulRHSUnsigned32) {
17473 MulSignedLo =
17474 numBitsSigned(Op: MulLHS, DAG) <= 32 && numBitsSigned(Op: MulRHS, DAG) <= 32;
17475 }
17476
17477 // The operands and final result all have the same number of bits. If
17478 // operands need to be extended, they can be extended with garbage. The
17479 // resulting garbage in the high bits of the mad_[iu]64_[iu]32 result is
17480 // truncated away in the end.
17481 if (VT != MVT::i64) {
17482 MulLHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: MulLHS);
17483 MulRHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: MulRHS);
17484 AddRHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: AddRHS);
17485 }
17486
17487 // The basic code generated is conceptually straightforward. Pseudo code:
17488 //
17489 // accum = mad_64_32 lhs.lo, rhs.lo, accum
17490 // accum.hi = add (mul lhs.hi, rhs.lo), accum.hi
17491 // accum.hi = add (mul lhs.lo, rhs.hi), accum.hi
17492 //
17493 // The second and third lines are optional, depending on whether the factors
17494 // are {sign,zero}-extended or not.
17495 //
17496 // The actual DAG is noisier than the pseudo code, but only due to
17497 // instructions that disassemble values into low and high parts, and
17498 // assemble the final result.
17499 SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
17500
17501 auto MulLHSLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulLHS);
17502 auto MulRHSLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulRHS);
17503 SDValue Accum =
17504 getMad64_32(DAG, SL, VT: MVT::i64, N0: MulLHSLo, N1: MulRHSLo, N2: AddRHS, Signed: MulSignedLo);
17505
17506 if (!MulSignedLo && (!MulLHSUnsigned32 || !MulRHSUnsigned32)) {
17507 auto [AccumLo, AccumHi] = DAG.SplitScalar(N: Accum, DL: SL, LoVT: MVT::i32, HiVT: MVT::i32);
17508
17509 if (!MulLHSUnsigned32) {
17510 auto MulLHSHi =
17511 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SL, VT: MVT::i32, N1: MulLHS, N2: One);
17512 SDValue MulHi = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: MVT::i32, N1: MulLHSHi, N2: MulRHSLo);
17513 AccumHi = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: MulHi, N2: AccumHi);
17514 }
17515
17516 if (!MulRHSUnsigned32) {
17517 auto MulRHSHi =
17518 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SL, VT: MVT::i32, N1: MulRHS, N2: One);
17519 SDValue MulHi = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: MVT::i32, N1: MulLHSLo, N2: MulRHSHi);
17520 AccumHi = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: MulHi, N2: AccumHi);
17521 }
17522
17523 Accum = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {AccumLo, AccumHi});
17524 Accum = DAG.getBitcast(VT: MVT::i64, V: Accum);
17525 }
17526
17527 if (VT != MVT::i64)
17528 Accum = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Accum);
17529 return Accum;
17530}
17531
17532SDValue
17533SITargetLowering::foldAddSub64WithZeroLowBitsTo32(SDNode *N,
17534 DAGCombinerInfo &DCI) const {
17535 SDValue RHS = N->getOperand(Num: 1);
17536 auto *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
17537 if (!CRHS)
17538 return SDValue();
17539
17540 // TODO: Worth using computeKnownBits? Maybe expensive since it's so
17541 // common.
17542 uint64_t Val = CRHS->getZExtValue();
17543 if (countr_zero(Val) >= 32) {
17544 SelectionDAG &DAG = DCI.DAG;
17545 SDLoc SL(N);
17546 SDValue LHS = N->getOperand(Num: 0);
17547
17548 // Avoid carry machinery if we know the low half of the add does not
17549 // contribute to the final result.
17550 //
17551 // add i64:x, K if computeTrailingZeros(K) >= 32
17552 // => build_pair (add x.hi, K.hi), x.lo
17553
17554 // Breaking the 64-bit add here with this strange constant is unlikely
17555 // to interfere with addressing mode patterns.
17556
17557 SDValue Hi = getHiHalf64(Op: LHS, DAG);
17558 SDValue ConstHi32 = DAG.getConstant(Val: Hi_32(Value: Val), DL: SL, VT: MVT::i32);
17559 unsigned Opcode = N->getOpcode();
17560 if (Opcode == ISD::PTRADD)
17561 Opcode = ISD::ADD;
17562 SDValue AddHi =
17563 DAG.getNode(Opcode, DL: SL, VT: MVT::i32, N1: Hi, N2: ConstHi32, Flags: N->getFlags());
17564
17565 SDValue Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: LHS);
17566 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i64, N1: Lo, N2: AddHi);
17567 }
17568
17569 return SDValue();
17570}
17571
17572// Collect the ultimate src of each of the mul node's operands, and confirm
17573// each operand is 8 bytes.
17574static std::optional<ByteProvider<SDValue>>
17575handleMulOperand(const SDValue &MulOperand) {
17576 auto Byte0 = calculateByteProvider(Op: MulOperand, Index: 0, Depth: 0);
17577 if (!Byte0 || Byte0->isConstantZero()) {
17578 return std::nullopt;
17579 }
17580 auto Byte1 = calculateByteProvider(Op: MulOperand, Index: 1, Depth: 0);
17581 if (Byte1 && !Byte1->isConstantZero()) {
17582 return std::nullopt;
17583 }
17584 return Byte0;
17585}
17586
17587static unsigned addPermMasks(unsigned First, unsigned Second) {
17588 unsigned FirstCs = First & 0x0c0c0c0c;
17589 unsigned SecondCs = Second & 0x0c0c0c0c;
17590 unsigned FirstNoCs = First & ~0x0c0c0c0c;
17591 unsigned SecondNoCs = Second & ~0x0c0c0c0c;
17592
17593 assert((FirstCs & 0xFF) | (SecondCs & 0xFF));
17594 assert((FirstCs & 0xFF00) | (SecondCs & 0xFF00));
17595 assert((FirstCs & 0xFF0000) | (SecondCs & 0xFF0000));
17596 assert((FirstCs & 0xFF000000) | (SecondCs & 0xFF000000));
17597
17598 return (FirstNoCs | SecondNoCs) | (FirstCs & SecondCs);
17599}
17600
17601struct DotSrc {
17602 SDValue SrcOp;
17603 int64_t PermMask;
17604 int64_t DWordOffset;
17605};
17606
17607static void placeSources(ByteProvider<SDValue> &Src0,
17608 ByteProvider<SDValue> &Src1,
17609 SmallVectorImpl<DotSrc> &Src0s,
17610 SmallVectorImpl<DotSrc> &Src1s, int Step) {
17611
17612 assert(Src0.Src.has_value() && Src1.Src.has_value());
17613 // Src0s and Src1s are empty, just place arbitrarily.
17614 if (Step == 0) {
17615 Src0s.push_back(Elt: {.SrcOp: *Src0.Src, .PermMask: ((Src0.SrcOffset % 4) << 24) + 0x0c0c0c,
17616 .DWordOffset: Src0.SrcOffset / 4});
17617 Src1s.push_back(Elt: {.SrcOp: *Src1.Src, .PermMask: ((Src1.SrcOffset % 4) << 24) + 0x0c0c0c,
17618 .DWordOffset: Src1.SrcOffset / 4});
17619 return;
17620 }
17621
17622 for (int BPI = 0; BPI < 2; BPI++) {
17623 std::pair<ByteProvider<SDValue>, ByteProvider<SDValue>> BPP = {Src0, Src1};
17624 if (BPI == 1) {
17625 BPP = {Src1, Src0};
17626 }
17627 unsigned ZeroMask = 0x0c0c0c0c;
17628 unsigned FMask = 0xFF << (8 * (3 - Step));
17629
17630 unsigned FirstMask =
17631 (BPP.first.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
17632 unsigned SecondMask =
17633 (BPP.second.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
17634 // Attempt to find Src vector which contains our SDValue, if so, add our
17635 // perm mask to the existing one. If we are unable to find a match for the
17636 // first SDValue, attempt to find match for the second.
17637 int FirstGroup = -1;
17638 for (int I = 0; I < 2; I++) {
17639 SmallVectorImpl<DotSrc> &Srcs = I == 0 ? Src0s : Src1s;
17640 auto MatchesFirst = [&BPP](DotSrc &IterElt) {
17641 return IterElt.SrcOp == *BPP.first.Src &&
17642 (IterElt.DWordOffset == (BPP.first.SrcOffset / 4));
17643 };
17644
17645 auto *Match = llvm::find_if(Range&: Srcs, P: MatchesFirst);
17646 if (Match != Srcs.end()) {
17647 Match->PermMask = addPermMasks(First: FirstMask, Second: Match->PermMask);
17648 FirstGroup = I;
17649 break;
17650 }
17651 }
17652 if (FirstGroup != -1) {
17653 SmallVectorImpl<DotSrc> &Srcs = FirstGroup == 1 ? Src0s : Src1s;
17654 auto MatchesSecond = [&BPP](DotSrc &IterElt) {
17655 return IterElt.SrcOp == *BPP.second.Src &&
17656 (IterElt.DWordOffset == (BPP.second.SrcOffset / 4));
17657 };
17658 auto *Match = llvm::find_if(Range&: Srcs, P: MatchesSecond);
17659 if (Match != Srcs.end()) {
17660 Match->PermMask = addPermMasks(First: SecondMask, Second: Match->PermMask);
17661 } else
17662 Srcs.push_back(Elt: {.SrcOp: *BPP.second.Src, .PermMask: SecondMask, .DWordOffset: BPP.second.SrcOffset / 4});
17663 return;
17664 }
17665 }
17666
17667 // If we have made it here, then we could not find a match in Src0s or Src1s
17668 // for either Src0 or Src1, so just place them arbitrarily.
17669
17670 unsigned ZeroMask = 0x0c0c0c0c;
17671 unsigned FMask = 0xFF << (8 * (3 - Step));
17672
17673 Src0s.push_back(
17674 Elt: {.SrcOp: *Src0.Src,
17675 .PermMask: ((Src0.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
17676 .DWordOffset: Src0.SrcOffset / 4});
17677 Src1s.push_back(
17678 Elt: {.SrcOp: *Src1.Src,
17679 .PermMask: ((Src1.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
17680 .DWordOffset: Src1.SrcOffset / 4});
17681}
17682
17683static SDValue resolveSources(SelectionDAG &DAG, SDLoc SL,
17684 SmallVectorImpl<DotSrc> &Srcs, bool IsSigned,
17685 bool IsAny) {
17686
17687 // If we just have one source, just permute it accordingly.
17688 if (Srcs.size() == 1) {
17689 auto *Elt = Srcs.begin();
17690 auto EltOp = getDWordFromOffset(DAG, SL, Src: Elt->SrcOp, DWordOffset: Elt->DWordOffset);
17691
17692 // v_perm will produce the original value
17693 if (Elt->PermMask == 0x3020100)
17694 return EltOp;
17695
17696 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: EltOp, N2: EltOp,
17697 N3: DAG.getConstant(Val: Elt->PermMask, DL: SL, VT: MVT::i32));
17698 }
17699
17700 auto *FirstElt = Srcs.begin();
17701 auto *SecondElt = std::next(x: FirstElt);
17702
17703 SmallVector<SDValue, 2> Perms;
17704
17705 // If we have multiple sources in the chain, combine them via perms (using
17706 // calculated perm mask) and Ors.
17707 while (true) {
17708 auto FirstMask = FirstElt->PermMask;
17709 auto SecondMask = SecondElt->PermMask;
17710
17711 unsigned FirstCs = FirstMask & 0x0c0c0c0c;
17712 unsigned FirstPlusFour = FirstMask | 0x04040404;
17713 // 0x0c + 0x04 = 0x10, so anding with 0x0F will produced 0x00 for any
17714 // original 0x0C.
17715 FirstMask = (FirstPlusFour & 0x0F0F0F0F) | FirstCs;
17716
17717 auto PermMask = addPermMasks(First: FirstMask, Second: SecondMask);
17718 auto FirstVal =
17719 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
17720 auto SecondVal =
17721 getDWordFromOffset(DAG, SL, Src: SecondElt->SrcOp, DWordOffset: SecondElt->DWordOffset);
17722
17723 Perms.push_back(Elt: DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: FirstVal,
17724 N2: SecondVal,
17725 N3: DAG.getConstant(Val: PermMask, DL: SL, VT: MVT::i32)));
17726
17727 FirstElt = std::next(x: SecondElt);
17728 if (FirstElt == Srcs.end())
17729 break;
17730
17731 SecondElt = std::next(x: FirstElt);
17732 // If we only have a FirstElt, then just combine that into the cumulative
17733 // source node.
17734 if (SecondElt == Srcs.end()) {
17735 auto EltOp =
17736 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
17737
17738 Perms.push_back(
17739 Elt: DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: EltOp, N2: EltOp,
17740 N3: DAG.getConstant(Val: FirstElt->PermMask, DL: SL, VT: MVT::i32)));
17741 break;
17742 }
17743 }
17744
17745 assert(Perms.size() == 1 || Perms.size() == 2);
17746 return Perms.size() == 2
17747 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Perms[0], N2: Perms[1])
17748 : Perms[0];
17749}
17750
17751static void fixMasks(SmallVectorImpl<DotSrc> &Srcs, unsigned ChainLength) {
17752 for (auto &[EntryVal, EntryMask, EntryOffset] : Srcs) {
17753 EntryMask = EntryMask >> ((4 - ChainLength) * 8);
17754 auto ZeroMask = ChainLength == 2 ? 0x0c0c0000 : 0x0c000000;
17755 EntryMask += ZeroMask;
17756 }
17757}
17758
17759static bool isMul(const SDValue Op) {
17760 auto Opcode = Op.getOpcode();
17761
17762 return (Opcode == ISD::MUL || Opcode == AMDGPUISD::MUL_U24 ||
17763 Opcode == AMDGPUISD::MUL_I24);
17764}
17765
17766static std::optional<bool>
17767checkDot4MulSignedness(const SDValue &N, ByteProvider<SDValue> &Src0,
17768 ByteProvider<SDValue> &Src1, const SDValue &S0Op,
17769 const SDValue &S1Op, const SelectionDAG &DAG) {
17770 // If we both ops are i8s (pre legalize-dag), then the signedness semantics
17771 // of the dot4 is irrelevant.
17772 if (S0Op.getValueSizeInBits() == 8 && S1Op.getValueSizeInBits() == 8)
17773 return false;
17774
17775 auto Known0 = DAG.computeKnownBits(Op: S0Op, Depth: 0);
17776 bool S0IsUnsigned = Known0.countMinLeadingZeros() > 0;
17777 bool S0IsSigned = Known0.countMinLeadingOnes() > 0;
17778 auto Known1 = DAG.computeKnownBits(Op: S1Op, Depth: 0);
17779 bool S1IsUnsigned = Known1.countMinLeadingZeros() > 0;
17780 bool S1IsSigned = Known1.countMinLeadingOnes() > 0;
17781
17782 assert(!(S0IsUnsigned && S0IsSigned));
17783 assert(!(S1IsUnsigned && S1IsSigned));
17784
17785 // There are 9 possible permutations of
17786 // {S0IsUnsigned, S0IsSigned, S1IsUnsigned, S1IsSigned}
17787
17788 // In two permutations, the sign bits are known to be the same for both Ops,
17789 // so simply return Signed / Unsigned corresponding to the MSB
17790
17791 if ((S0IsUnsigned && S1IsUnsigned) || (S0IsSigned && S1IsSigned))
17792 return S0IsSigned;
17793
17794 // In another two permutations, the sign bits are known to be opposite. In
17795 // this case return std::nullopt to indicate a bad match.
17796
17797 if ((S0IsUnsigned && S1IsSigned) || (S0IsSigned && S1IsUnsigned))
17798 return std::nullopt;
17799
17800 // In the remaining five permutations, we don't know the value of the sign
17801 // bit for at least one Op. Since we have a valid ByteProvider, we know that
17802 // the upper bits must be extension bits. Thus, the only ways for the sign
17803 // bit to be unknown is if it was sign extended from unknown value, or if it
17804 // was any extended. In either case, it is correct to use the signed
17805 // version of the signedness semantics of dot4
17806
17807 // In two of such permutations, we known the sign bit is set for
17808 // one op, and the other is unknown. It is okay to used signed version of
17809 // dot4.
17810 if ((S0IsSigned && !(S1IsSigned || S1IsUnsigned)) ||
17811 ((S1IsSigned && !(S0IsSigned || S0IsUnsigned))))
17812 return true;
17813
17814 // In one such permutation, we don't know either of the sign bits. It is okay
17815 // to used the signed version of dot4.
17816 if ((!(S1IsSigned || S1IsUnsigned) && !(S0IsSigned || S0IsUnsigned)))
17817 return true;
17818
17819 // In two of such permutations, we known the sign bit is unset for
17820 // one op, and the other is unknown. Return std::nullopt to indicate a
17821 // bad match.
17822 if ((S0IsUnsigned && !(S1IsSigned || S1IsUnsigned)) ||
17823 ((S1IsUnsigned && !(S0IsSigned || S0IsUnsigned))))
17824 return std::nullopt;
17825
17826 llvm_unreachable("Fully covered condition");
17827}
17828
17829SDValue SITargetLowering::performAddCombine(SDNode *N,
17830 DAGCombinerInfo &DCI) const {
17831 SelectionDAG &DAG = DCI.DAG;
17832 EVT VT = N->getValueType(ResNo: 0);
17833 SDLoc SL(N);
17834 SDValue LHS = N->getOperand(Num: 0);
17835 SDValue RHS = N->getOperand(Num: 1);
17836
17837 if (LHS.getOpcode() == ISD::MUL || RHS.getOpcode() == ISD::MUL) {
17838 if (Subtarget->hasMad64_32()) {
17839 if (SDValue Folded = tryFoldToMad64_32(N, DCI))
17840 return Folded;
17841 }
17842 }
17843
17844 if (SDValue V = reassociateScalarOps(N, DAG)) {
17845 return V;
17846 }
17847
17848 if (VT == MVT::i64) {
17849 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
17850 return Folded;
17851 }
17852
17853 // dot4 produces a 32-bit result, so a wider VT can't be folded.
17854 if (!VT.isVector() && VT.getSizeInBits() <= 32 &&
17855 (isMul(Op: LHS) || isMul(Op: RHS)) && Subtarget->hasDot7Insts() &&
17856 (Subtarget->hasDot1Insts() || Subtarget->hasDot8Insts())) {
17857 SDValue TempNode(N, 0);
17858 std::optional<bool> IsSigned;
17859 SmallVector<DotSrc, 4> Src0s;
17860 SmallVector<DotSrc, 4> Src1s;
17861 SmallVector<SDValue, 4> Src2s;
17862
17863 // Match the v_dot4 tree, while collecting src nodes.
17864 int ChainLength = 0;
17865 for (int I = 0; I < 4; I++) {
17866 auto MulIdx = isMul(Op: LHS) ? 0 : isMul(Op: RHS) ? 1 : -1;
17867 if (MulIdx == -1)
17868 break;
17869 auto Src0 = handleMulOperand(MulOperand: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 0));
17870 if (!Src0)
17871 break;
17872 auto Src1 = handleMulOperand(MulOperand: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 1));
17873 if (!Src1)
17874 break;
17875
17876 auto IterIsSigned = checkDot4MulSignedness(
17877 N: TempNode->getOperand(Num: MulIdx), Src0&: *Src0, Src1&: *Src1,
17878 S0Op: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 0),
17879 S1Op: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 1), DAG);
17880 if (!IterIsSigned)
17881 break;
17882 if (!IsSigned)
17883 IsSigned = *IterIsSigned;
17884 if (*IterIsSigned != *IsSigned)
17885 break;
17886 placeSources(Src0&: *Src0, Src1&: *Src1, Src0s, Src1s, Step: I);
17887 auto AddIdx = 1 - MulIdx;
17888 // Allow the special case where add (add (mul24, 0), mul24) became ->
17889 // add (mul24, mul24).
17890 if (I == 2 && isMul(Op: TempNode->getOperand(Num: AddIdx))) {
17891 Src2s.push_back(Elt: TempNode->getOperand(Num: AddIdx));
17892 auto Src0 =
17893 handleMulOperand(MulOperand: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 0));
17894 if (!Src0)
17895 break;
17896 auto Src1 =
17897 handleMulOperand(MulOperand: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 1));
17898 if (!Src1)
17899 break;
17900 auto IterIsSigned = checkDot4MulSignedness(
17901 N: TempNode->getOperand(Num: AddIdx), Src0&: *Src0, Src1&: *Src1,
17902 S0Op: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 0),
17903 S1Op: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 1), DAG);
17904 if (!IterIsSigned)
17905 break;
17906 assert(IsSigned);
17907 if (*IterIsSigned != *IsSigned)
17908 break;
17909 placeSources(Src0&: *Src0, Src1&: *Src1, Src0s, Src1s, Step: I + 1);
17910 Src2s.push_back(Elt: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
17911 ChainLength = I + 2;
17912 break;
17913 }
17914
17915 TempNode = TempNode->getOperand(Num: AddIdx);
17916 Src2s.push_back(Elt: TempNode);
17917 ChainLength = I + 1;
17918 // The loop body treats TempNode's operands as addends.
17919 if (TempNode.getOpcode() != ISD::ADD)
17920 break;
17921 LHS = TempNode->getOperand(Num: 0);
17922 RHS = TempNode->getOperand(Num: 1);
17923 }
17924
17925 if (ChainLength < 2)
17926 return SDValue();
17927
17928 // Masks were constructed with assumption that we would find a chain of
17929 // length 4. If not, then we need to 0 out the MSB bits (via perm mask of
17930 // 0x0c) so they do not affect dot calculation.
17931 if (ChainLength < 4) {
17932 fixMasks(Srcs&: Src0s, ChainLength);
17933 fixMasks(Srcs&: Src1s, ChainLength);
17934 }
17935
17936 SDValue Src0, Src1;
17937
17938 // If we are just using a single source for both, and have permuted the
17939 // bytes consistently, we can just use the sources without permuting
17940 // (commutation).
17941 bool UseOriginalSrc = false;
17942 if (ChainLength == 4 && Src0s.size() == 1 && Src1s.size() == 1 &&
17943 Src0s.begin()->PermMask == Src1s.begin()->PermMask &&
17944 Src0s.begin()->SrcOp.getValueSizeInBits() >= 32 &&
17945 Src1s.begin()->SrcOp.getValueSizeInBits() >= 32) {
17946 SmallVector<unsigned, 4> SrcBytes;
17947 auto Src0Mask = Src0s.begin()->PermMask;
17948 SrcBytes.push_back(Elt: Src0Mask & 0xFF000000);
17949 bool UniqueEntries = true;
17950 for (auto I = 1; I < 4; I++) {
17951 auto NextByte = Src0Mask & (0xFF << ((3 - I) * 8));
17952
17953 if (is_contained(Range&: SrcBytes, Element: NextByte)) {
17954 UniqueEntries = false;
17955 break;
17956 }
17957 SrcBytes.push_back(Elt: NextByte);
17958 }
17959
17960 if (UniqueEntries) {
17961 UseOriginalSrc = true;
17962
17963 auto *FirstElt = Src0s.begin();
17964 auto FirstEltOp =
17965 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
17966
17967 auto *SecondElt = Src1s.begin();
17968 auto SecondEltOp = getDWordFromOffset(DAG, SL, Src: SecondElt->SrcOp,
17969 DWordOffset: SecondElt->DWordOffset);
17970
17971 Src0 = DAG.getBitcastedAnyExtOrTrunc(Op: FirstEltOp, DL: SL,
17972 VT: MVT::getIntegerVT(BitWidth: 32));
17973 Src1 = DAG.getBitcastedAnyExtOrTrunc(Op: SecondEltOp, DL: SL,
17974 VT: MVT::getIntegerVT(BitWidth: 32));
17975 }
17976 }
17977
17978 if (!UseOriginalSrc) {
17979 Src0 = resolveSources(DAG, SL, Srcs&: Src0s, IsSigned: false, IsAny: true);
17980 Src1 = resolveSources(DAG, SL, Srcs&: Src1s, IsSigned: false, IsAny: true);
17981 }
17982
17983 assert(IsSigned);
17984 SDValue Src2 =
17985 DAG.getExtOrTrunc(IsSigned: *IsSigned, Op: Src2s[ChainLength - 1], DL: SL, VT: MVT::i32);
17986
17987 SDValue IID = DAG.getTargetConstant(Val: *IsSigned ? Intrinsic::amdgcn_sdot4
17988 : Intrinsic::amdgcn_udot4,
17989 DL: SL, VT: MVT::i64);
17990
17991 assert(!VT.isVector());
17992 auto Dot = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32, N1: IID, N2: Src0,
17993 N3: Src1, N4: Src2, N5: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i1));
17994
17995 return DAG.getExtOrTrunc(IsSigned: *IsSigned, Op: Dot, DL: SL, VT);
17996 }
17997
17998 if (VT != MVT::i32 || !DCI.isAfterLegalizeDAG())
17999 return SDValue();
18000
18001 // add x, zext (setcc) => uaddo_carry x, 0, setcc
18002 // add x, sext (setcc) => usubo_carry x, 0, setcc
18003 unsigned Opc = LHS.getOpcode();
18004 if (Opc == ISD::ZERO_EXTEND || Opc == ISD::SIGN_EXTEND ||
18005 Opc == ISD::ANY_EXTEND || Opc == ISD::UADDO_CARRY)
18006 std::swap(a&: RHS, b&: LHS);
18007
18008 Opc = RHS.getOpcode();
18009 switch (Opc) {
18010 default:
18011 break;
18012 case ISD::ZERO_EXTEND:
18013 case ISD::SIGN_EXTEND:
18014 case ISD::ANY_EXTEND: {
18015 auto Cond = RHS.getOperand(i: 0);
18016 // If this won't be a real VOPC output, we would still need to insert an
18017 // extra instruction anyway.
18018 if (!isBoolSGPR(V: Cond))
18019 break;
18020 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1);
18021 SDValue Args[] = {LHS, DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond};
18022 Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::USUBO_CARRY : ISD::UADDO_CARRY;
18023 return DAG.getNode(Opcode: Opc, DL: SL, VTList, Ops: Args);
18024 }
18025 case ISD::UADDO_CARRY: {
18026 // add x, (uaddo_carry y, 0, cc) => uaddo_carry x, y, cc
18027 if (!isNullConstant(V: RHS.getOperand(i: 1)))
18028 break;
18029 SDValue Args[] = {LHS, RHS.getOperand(i: 0), RHS.getOperand(i: 2)};
18030 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: SDLoc(N), VTList: RHS->getVTList(), Ops: Args);
18031 }
18032 }
18033 return SDValue();
18034}
18035
18036SDValue SITargetLowering::performPtrAddCombine(SDNode *N,
18037 DAGCombinerInfo &DCI) const {
18038 SelectionDAG &DAG = DCI.DAG;
18039 SDLoc DL(N);
18040 EVT VT = N->getValueType(ResNo: 0);
18041 SDValue N0 = N->getOperand(Num: 0);
18042 SDValue N1 = N->getOperand(Num: 1);
18043
18044 // The following folds transform PTRADDs into regular arithmetic in cases
18045 // where the PTRADD wouldn't be folded as an immediate offset into memory
18046 // instructions anyway. They are target-specific in that other targets might
18047 // prefer to not lose information about the pointer arithmetic.
18048
18049 // Fold (ptradd x, shl(0 - v, k)) -> sub(x, shl(v, k)).
18050 // Adapted from DAGCombiner::visitADDLikeCommutative.
18051 SDValue V, K;
18052 if (sd_match(N: N1, P: m_Shl(L: m_Neg(V: m_Value(N&: V)), R: m_Value(N&: K)))) {
18053 SDNodeFlags ShlFlags = N1->getFlags();
18054 // If the original shl is NUW and NSW, the first k+1 bits of 0-v are all 0,
18055 // so v is either 0 or the first k+1 bits of v are all 1 -> NSW can be
18056 // preserved.
18057 SDNodeFlags NewShlFlags =
18058 ShlFlags.hasNoUnsignedWrap() && ShlFlags.hasNoSignedWrap()
18059 ? SDNodeFlags::NoSignedWrap
18060 : SDNodeFlags();
18061 SDValue Inner = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: V, N2: K, Flags: NewShlFlags);
18062 DCI.AddToWorklist(N: Inner.getNode());
18063 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: Inner);
18064 }
18065
18066 // Fold into Mad64 if the right-hand side is a MUL. Analogous to a fold in
18067 // performAddCombine.
18068 if (N1.getOpcode() == ISD::MUL) {
18069 if (Subtarget->hasMad64_32()) {
18070 if (SDValue Folded = tryFoldToMad64_32(N, DCI))
18071 return Folded;
18072 }
18073 }
18074
18075 // If the 32 low bits of the constant are all zero, there is nothing to fold
18076 // into an immediate offset, so it's better to eliminate the unnecessary
18077 // addition for the lower 32 bits than to preserve the PTRADD.
18078 // Analogous to a fold in performAddCombine.
18079 if (VT == MVT::i64) {
18080 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
18081 return Folded;
18082 }
18083
18084 if (N1.getOpcode() != ISD::ADD || !N1.hasOneUse())
18085 return SDValue();
18086
18087 SDValue X = N0;
18088 SDValue Y = N1.getOperand(i: 0);
18089 SDValue Z = N1.getOperand(i: 1);
18090 bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(N: Y);
18091 bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(N: Z);
18092
18093 if (!YIsConstant && !ZIsConstant && !X->isDivergent() &&
18094 Y->isDivergent() != Z->isDivergent()) {
18095 // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if x and
18096 // y are uniform and z isn't.
18097 // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if x and
18098 // z are uniform and y isn't.
18099 // The goal is to push uniform operands up in the computation, so that they
18100 // can be handled with scalar operations. We can't use reassociateScalarOps
18101 // for this since it requires two identical commutative operations to
18102 // reassociate.
18103 if (Y->isDivergent())
18104 std::swap(a&: Y, b&: Z);
18105 // If both additions in the original were NUW, reassociation preserves that.
18106 SDNodeFlags ReassocFlags =
18107 (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap;
18108 SDValue UniformInner = DAG.getMemBasePlusOffset(Base: X, Offset: Y, DL, Flags: ReassocFlags);
18109 DCI.AddToWorklist(N: UniformInner.getNode());
18110 return DAG.getMemBasePlusOffset(Base: UniformInner, Offset: Z, DL, Flags: ReassocFlags);
18111 }
18112
18113 return SDValue();
18114}
18115
18116static bool isCtlzOpc(unsigned Opc) {
18117 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_POISON;
18118}
18119
18120SDValue SITargetLowering::performSubCombine(SDNode *N,
18121 DAGCombinerInfo &DCI) const {
18122 SelectionDAG &DAG = DCI.DAG;
18123 EVT VT = N->getValueType(ResNo: 0);
18124
18125 if (VT == MVT::i64) {
18126 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
18127 return Folded;
18128 }
18129
18130 if (VT != MVT::i32)
18131 return SDValue();
18132
18133 SDLoc SL(N);
18134 SDValue LHS = N->getOperand(Num: 0);
18135 SDValue RHS = N->getOperand(Num: 1);
18136
18137 // sub x, zext (setcc) => usubo_carry x, 0, setcc
18138 // sub x, sext (setcc) => uaddo_carry x, 0, setcc
18139 unsigned Opc = RHS.getOpcode();
18140 switch (Opc) {
18141 default:
18142 break;
18143 case ISD::ZERO_EXTEND:
18144 case ISD::SIGN_EXTEND:
18145 case ISD::ANY_EXTEND: {
18146 auto Cond = RHS.getOperand(i: 0);
18147 // If this won't be a real VOPC output, we would still need to insert an
18148 // extra instruction anyway.
18149 if (!isBoolSGPR(V: Cond))
18150 break;
18151 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1);
18152 SDValue Args[] = {LHS, DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond};
18153 Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
18154 return DAG.getNode(Opcode: Opc, DL: SL, VTList, Ops: Args);
18155 }
18156 }
18157
18158 if (LHS.getOpcode() == ISD::USUBO_CARRY) {
18159 // sub (usubo_carry x, 0, cc), y => usubo_carry x, y, cc
18160 if (!isNullConstant(V: LHS.getOperand(i: 1)))
18161 return SDValue();
18162 SDValue Args[] = {LHS.getOperand(i: 0), RHS, LHS.getOperand(i: 2)};
18163 return DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: SDLoc(N), VTList: LHS->getVTList(), Ops: Args);
18164 }
18165
18166 // sub (ctlz (xor x, (sra x, 31))), 1 -> ctls x.
18167 if (isOneConstant(V: RHS) && isCtlzOpc(Opc: LHS.getOpcode())) {
18168 SDValue CtlzSrc = LHS.getOperand(i: 0);
18169 // Check for xor x, (sra x, 31) pattern.
18170 if (CtlzSrc.getOpcode() == ISD::XOR) {
18171 SDValue X = CtlzSrc.getOperand(i: 0);
18172 SDValue SignExt = CtlzSrc.getOperand(i: 1);
18173 // Try both ordering of XOR operands.
18174 if (SignExt.getOpcode() != ISD::SRA)
18175 std::swap(a&: X, b&: SignExt);
18176 if (SignExt.getOpcode() == ISD::SRA && SignExt.getOperand(i: 0) == X) {
18177 ConstantSDNode *ShiftAmt =
18178 dyn_cast<ConstantSDNode>(Val: SignExt.getOperand(i: 1));
18179 unsigned BitWidth = X.getValueType().getScalarSizeInBits();
18180 if (ShiftAmt && ShiftAmt->getZExtValue() == BitWidth - 1)
18181 return DAG.getNode(Opcode: ISD::CTLS, DL: SL, VT, Operand: X);
18182 }
18183 }
18184 }
18185
18186 return SDValue();
18187}
18188
18189SDValue SITargetLowering::performFAddCombine(SDNode *N,
18190 DAGCombinerInfo &DCI) const {
18191 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
18192 return SDValue();
18193
18194 SelectionDAG &DAG = DCI.DAG;
18195 EVT VT = N->getValueType(ResNo: 0);
18196
18197 SDLoc SL(N);
18198 SDValue LHS = N->getOperand(Num: 0);
18199 SDValue RHS = N->getOperand(Num: 1);
18200
18201 // These should really be instruction patterns, but writing patterns with
18202 // source modifiers is a pain.
18203
18204 // fadd (fadd (a, a), b) -> mad 2.0, a, b
18205 if (LHS.getOpcode() == ISD::FADD) {
18206 SDValue A = LHS.getOperand(i: 0);
18207 if (A == LHS.getOperand(i: 1)) {
18208 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: LHS.getNode());
18209 if (FusedOp != 0) {
18210 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
18211 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: RHS);
18212 }
18213 }
18214 }
18215
18216 // fadd (b, fadd (a, a)) -> mad 2.0, a, b
18217 if (RHS.getOpcode() == ISD::FADD) {
18218 SDValue A = RHS.getOperand(i: 0);
18219 if (A == RHS.getOperand(i: 1)) {
18220 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: RHS.getNode());
18221 if (FusedOp != 0) {
18222 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
18223 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: LHS);
18224 }
18225 }
18226 }
18227
18228 return SDValue();
18229}
18230
18231SDValue SITargetLowering::performFSubCombine(SDNode *N,
18232 DAGCombinerInfo &DCI) const {
18233 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
18234 return SDValue();
18235
18236 SelectionDAG &DAG = DCI.DAG;
18237 SDLoc SL(N);
18238 EVT VT = N->getValueType(ResNo: 0);
18239 assert(!VT.isVector());
18240
18241 // Try to get the fneg to fold into the source modifier. This undoes generic
18242 // DAG combines and folds them into the mad.
18243 //
18244 // Only do this if we are not trying to support denormals. v_mad_f32 does
18245 // not support denormals ever.
18246 SDValue LHS = N->getOperand(Num: 0);
18247 SDValue RHS = N->getOperand(Num: 1);
18248 if (LHS.getOpcode() == ISD::FADD) {
18249 // (fsub (fadd a, a), c) -> mad 2.0, a, (fneg c)
18250 SDValue A = LHS.getOperand(i: 0);
18251 if (A == LHS.getOperand(i: 1)) {
18252 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: LHS.getNode());
18253 if (FusedOp != 0) {
18254 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
18255 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
18256
18257 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: NegRHS);
18258 }
18259 }
18260 }
18261
18262 if (RHS.getOpcode() == ISD::FADD) {
18263 // (fsub c, (fadd a, a)) -> mad -2.0, a, c
18264
18265 SDValue A = RHS.getOperand(i: 0);
18266 if (A == RHS.getOperand(i: 1)) {
18267 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: RHS.getNode());
18268 if (FusedOp != 0) {
18269 const SDValue NegTwo = DAG.getConstantFP(Val: -2.0, DL: SL, VT);
18270 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: NegTwo, N3: LHS);
18271 }
18272 }
18273 }
18274
18275 return SDValue();
18276}
18277
18278SDValue SITargetLowering::performFDivCombine(SDNode *N,
18279 DAGCombinerInfo &DCI) const {
18280 SelectionDAG &DAG = DCI.DAG;
18281 SDLoc SL(N);
18282 EVT VT = N->getValueType(ResNo: 0);
18283
18284 if (VT != MVT::f16 && VT != MVT::bf16)
18285 return SDValue();
18286
18287 SDValue LHS = N->getOperand(Num: 0);
18288 SDValue RHS = N->getOperand(Num: 1);
18289
18290 SDNodeFlags Flags = N->getFlags();
18291 SDNodeFlags RHSFlags = RHS->getFlags();
18292 if (!Flags.hasAllowContract() || !RHSFlags.hasAllowContract() ||
18293 !RHS->hasOneUse())
18294 return SDValue();
18295
18296 if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
18297 bool IsNegative = false;
18298 if (CLHS->isOne() || (IsNegative = CLHS->isMinusOne())) {
18299 // fdiv contract 1.0, (sqrt contract x) -> rsq
18300 // fdiv contract -1.0, (sqrt contract x) -> fneg(rsq)
18301 if (RHS.getOpcode() == ISD::FSQRT) {
18302 // TODO: Or in RHS flags, somehow missing from SDNodeFlags
18303 SDValue SqrtOp = RHS.getOperand(i: 0);
18304 SDValue Rsq;
18305 if (isOperationLegal(Op: ISD::FSQRT, VT)) {
18306 // fsqrt legality correlates to rsq availability of the same type.
18307 Rsq = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SL, VT, Operand: SqrtOp, Flags);
18308 } else if (VT == MVT::f16) {
18309 // Targets without 16-bit instructions (gfx6/gfx7) have no f16 rsq,
18310 // but v_rsq_f32 is more than accurate enough for f16. Unlike bf16,
18311 // every f16 value (including denormals) extends to a normal f32, and
18312 // an f16 rsq result is never denormal, so the f32 reciprocal square
18313 // root needs no denormal handling. Compute it in f32 and round back.
18314 SDValue Ext =
18315 DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: SqrtOp, Flags);
18316 SDValue F32Rsq =
18317 DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
18318 Rsq = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: F32Rsq,
18319 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
18320 } else {
18321 // bf16 shares f32's exponent range, so bf16 denormals would extend to
18322 // f32 denormals that v_rsq_f32 does not handle. Leave it expanded.
18323 return SDValue();
18324 }
18325 return IsNegative ? DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Rsq, Flags) : Rsq;
18326 }
18327 }
18328 }
18329
18330 return SDValue();
18331}
18332
18333SDValue SITargetLowering::performFMulCombine(SDNode *N,
18334 DAGCombinerInfo &DCI) const {
18335 SelectionDAG &DAG = DCI.DAG;
18336 EVT VT = N->getValueType(ResNo: 0);
18337 EVT ScalarVT = VT.getScalarType();
18338 EVT IntVT = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
18339
18340 if (!N->isDivergent() && getSubtarget()->hasSALUFloatInsts() &&
18341 (ScalarVT == MVT::f32 || ScalarVT == MVT::f16)) {
18342 // Prefer to use s_mul_f16/f32 instead of v_ldexp_f16/f32.
18343 return SDValue();
18344 }
18345
18346 SDValue LHS = N->getOperand(Num: 0);
18347 SDValue RHS = N->getOperand(Num: 1);
18348
18349 // It is cheaper to realize i32 inline constants as compared against
18350 // materializing f16 or f64 (or even non-inline f32) values,
18351 // possible via ldexp usage, as shown below :
18352 //
18353 // Given : A = 2^a & B = 2^b ; where a and b are integers.
18354 // fmul x, (select y, A, B) -> ldexp( x, (select i32 y, a, b) )
18355 // fmul x, (select y, -A, -B) -> ldexp( (fneg x), (select i32 y, a, b) )
18356 if ((ScalarVT == MVT::f64 || ScalarVT == MVT::f32 || ScalarVT == MVT::f16) &&
18357 (RHS.hasOneUse() && RHS.getOpcode() == ISD::SELECT)) {
18358 const ConstantFPSDNode *TrueNode = isConstOrConstSplatFP(N: RHS.getOperand(i: 1));
18359 if (!TrueNode)
18360 return SDValue();
18361 const ConstantFPSDNode *FalseNode =
18362 isConstOrConstSplatFP(N: RHS.getOperand(i: 2));
18363 if (!FalseNode)
18364 return SDValue();
18365
18366 if (TrueNode->isNegative() != FalseNode->isNegative())
18367 return SDValue();
18368
18369 // For f32, only non-inline constants should be transformed.
18370 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
18371 if (ScalarVT == MVT::f32 &&
18372 TII->isInlineConstant(Imm: TrueNode->getValueAPF()) &&
18373 TII->isInlineConstant(Imm: FalseNode->getValueAPF()))
18374 return SDValue();
18375
18376 int TrueNodeExpVal = TrueNode->getValueAPF().getExactLog2Abs();
18377 if (TrueNodeExpVal == INT_MIN)
18378 return SDValue();
18379 int FalseNodeExpVal = FalseNode->getValueAPF().getExactLog2Abs();
18380 if (FalseNodeExpVal == INT_MIN)
18381 return SDValue();
18382
18383 SDLoc SL(N);
18384 SDValue SelectNode =
18385 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: IntVT, N1: RHS.getOperand(i: 0),
18386 N2: DAG.getSignedConstant(Val: TrueNodeExpVal, DL: SL, VT: IntVT),
18387 N3: DAG.getSignedConstant(Val: FalseNodeExpVal, DL: SL, VT: IntVT));
18388
18389 LHS = TrueNode->isNegative()
18390 ? DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS, Flags: LHS->getFlags())
18391 : LHS;
18392
18393 return DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT, N1: LHS, N2: SelectNode, Flags: N->getFlags());
18394 }
18395
18396 return SDValue();
18397}
18398
18399SDValue SITargetLowering::performFMACombine(SDNode *N,
18400 DAGCombinerInfo &DCI) const {
18401 SelectionDAG &DAG = DCI.DAG;
18402 EVT VT = N->getValueType(ResNo: 0);
18403 SDLoc SL(N);
18404
18405 if (!Subtarget->hasDot10Insts() || VT != MVT::f32)
18406 return SDValue();
18407
18408 // FMA((F32)S0.x, (F32)S1. x, FMA((F32)S0.y, (F32)S1.y, (F32)z)) ->
18409 // FDOT2((V2F16)S0, (V2F16)S1, (F32)z))
18410 SDValue Op1 = N->getOperand(Num: 0);
18411 SDValue Op2 = N->getOperand(Num: 1);
18412 SDValue FMA = N->getOperand(Num: 2);
18413
18414 if (FMA.getOpcode() != ISD::FMA || Op1.getOpcode() != ISD::FP_EXTEND ||
18415 Op2.getOpcode() != ISD::FP_EXTEND)
18416 return SDValue();
18417
18418 // The fdot2 fold (fma_mix -> dot2) is only safe when both instructions agree
18419 // on how f16 subnormal inputs are handled. However, if both FMAs carry afn
18420 // the caller accepts approximate results, so any subnormal flushing
18421 // introduced by dot2 is acceptable regardless of mode.
18422 //
18423 // gfx90a (CDNA2) is the sole exception (dot2UnconditionalFlush): v_dot2c
18424 // unconditionally flushes f16 subnormal inputs to zero regardless of MODE,
18425 // while v_fma_mix_f32 preserves them when ieee=1 (the default compute kernel
18426 // mode). The fold is safe only when f32 denorm = PreserveSign, which implies
18427 // ieee=0 so both flush.
18428 //
18429 // All other GPUs: v_dot2 does NOT flush f16 subnormal inputs. v_fma_mix_f32
18430 // flushes them only when f32 denorm = PreserveSign. The fold is safe only
18431 // when f32 denorm is IEEE (both preserve the subnormal). Dynamic mode is
18432 // also rejected since the runtime value is unknown.
18433 bool AllowInaccuracy = N->getFlags().hasApproximateFuncs() &&
18434 FMA->getFlags().hasApproximateFuncs();
18435 if (!AllowInaccuracy) {
18436 const MachineFunction &MF = DAG.getMachineFunction();
18437 DenormalMode Mode = MF.getDenormalMode(FPType: APFloat::IEEEsingle());
18438 if (Subtarget->dot2UnconditionalFlush()) {
18439 // gfx90a: fold safe only when f32 denorm flushes.
18440 if (Mode != DenormalMode::getPreserveSign())
18441 return SDValue();
18442 } else {
18443 // All other GPUs: fold safe only when f32 denorm is IEEE.
18444 if (Mode != DenormalMode::getIEEE())
18445 return SDValue();
18446 }
18447 }
18448
18449 // fp-contract allows reassociating the fma tree into a dot product.
18450 if (N->getFlags().hasAllowContract() && FMA->getFlags().hasAllowContract()) {
18451 Op1 = Op1.getOperand(i: 0);
18452 Op2 = Op2.getOperand(i: 0);
18453 if (Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
18454 Op2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
18455 return SDValue();
18456
18457 SDValue Vec1 = Op1.getOperand(i: 0);
18458 SDValue Idx1 = Op1.getOperand(i: 1);
18459 SDValue Vec2 = Op2.getOperand(i: 0);
18460
18461 SDValue FMAOp1 = FMA.getOperand(i: 0);
18462 SDValue FMAOp2 = FMA.getOperand(i: 1);
18463 SDValue FMAAcc = FMA.getOperand(i: 2);
18464
18465 if (FMAOp1.getOpcode() != ISD::FP_EXTEND ||
18466 FMAOp2.getOpcode() != ISD::FP_EXTEND)
18467 return SDValue();
18468
18469 FMAOp1 = FMAOp1.getOperand(i: 0);
18470 FMAOp2 = FMAOp2.getOperand(i: 0);
18471 if (FMAOp1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
18472 FMAOp2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
18473 return SDValue();
18474
18475 SDValue Vec3 = FMAOp1.getOperand(i: 0);
18476 SDValue Vec4 = FMAOp2.getOperand(i: 0);
18477 SDValue Idx2 = FMAOp1.getOperand(i: 1);
18478
18479 if (Idx1 != Op2.getOperand(i: 1) || Idx2 != FMAOp2.getOperand(i: 1))
18480 return SDValue();
18481
18482 if (!isa<ConstantSDNode>(Val: Idx1) || !isa<ConstantSDNode>(Val: Idx2) ||
18483 Idx1 == Idx2)
18484 return SDValue();
18485
18486 if (Vec1 == Vec2 || Vec3 == Vec4)
18487 return SDValue();
18488
18489 if (Vec1.getValueType() != MVT::v2f16 || Vec2.getValueType() != MVT::v2f16)
18490 return SDValue();
18491
18492 if ((Vec1 == Vec3 && Vec2 == Vec4) || (Vec1 == Vec4 && Vec2 == Vec3)) {
18493 return DAG.getNode(Opcode: AMDGPUISD::FDOT2, DL: SL, VT: MVT::f32, N1: Vec1, N2: Vec2, N3: FMAAcc,
18494 N4: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i1));
18495 }
18496 }
18497 return SDValue();
18498}
18499
18500// Given a double-precision ordered or unordered comparison, return the
18501// condition code for an equivalent integral comparison of the operands' upper
18502// 32 bits, or `SETCC_INVALID` if not possible.
18503// For simplicity, no simplification occurs if the operands are not both known
18504// to have sign bit zero.
18505//
18506// EQ/NE:
18507// If LHS.lo32 == RHS.lo32:
18508// setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, eq/ne
18509// If LHS.lo32 != RHS.lo32:
18510// setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, false/true
18511// The reduction is not possible if operands may be +0 and -0.
18512// For ordered eq / unordered ne, at most one operand may be NaN.
18513// For unordered eq / ordered ne, neither operand can be NaN.
18514//
18515// LT/GE:
18516// If LHS.lo32 >= RHS.lo32 (unsigned):
18517// setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]lt/ge
18518// If LHS.lo32 < RHS.lo32 (unsigned):
18519// setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]le/gt
18520// The reduction is only supported if both operands are nonnegative.
18521// For ordered lt / unordered ge, the RHS cannot be NaN.
18522// For unordered lt / ordered ge, neither operand can be NaN.
18523//
18524// LE/GT:
18525// If LHS.lo32 > RHS.lo32 (unsigned):
18526// setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]lt/ge
18527// If LHS.lo32 <= RHS.lo32 (unsigned):
18528// setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]le/gt
18529// The reduction is only supported if both operands are nonnegative.
18530// For unordered le / ordered gt, the LHS cannot be NaN.
18531// For ordered le / unordered gt, neither operand can be NaN.
18532static ISD::CondCode tryReduceF64CompareToHiHalf(const ISD::CondCode CC,
18533 const SDValue LHS,
18534 const SDValue RHS,
18535 const SelectionDAG &DAG) {
18536 EVT VT = LHS.getValueType();
18537 assert(VT == MVT::f64 && "Incorrect operand type!");
18538
18539 const KnownBits RHSBits = DAG.computeKnownBits(Op: RHS);
18540 // Bail if RHS sign bit is not known to be zero.
18541 if (!RHSBits.Zero.isSignBitSet())
18542 return ISD::SETCC_INVALID;
18543
18544 const KnownBits RHSKnownLo32 = RHSBits.trunc(BitWidth: 32);
18545 const KnownFPClass RHSFPClass =
18546 KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits: RHSBits);
18547 const bool RHSMaybeNaN = !RHSFPClass.isKnownNeverNaN();
18548
18549 const KnownBits LHSBits = DAG.computeKnownBits(Op: LHS);
18550 const KnownBits LHSKnownLo32 = LHSBits.trunc(BitWidth: 32);
18551 const KnownFPClass LHSFPClass =
18552 KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits: LHSBits);
18553 const bool LHSMaybeNaN = !LHSFPClass.isKnownNeverNaN();
18554
18555 // Bail if LHS sign bit is not known to be zero.
18556 if (!LHSBits.Zero.isSignBitSet())
18557 return ISD::SETCC_INVALID;
18558
18559 switch (CC) {
18560 default:
18561 break;
18562 case ISD::SETEQ:
18563 case ISD::SETOEQ:
18564 case ISD::SETUEQ:
18565 case ISD::SETONE:
18566 case ISD::SETUNE: {
18567 // OEQ should be false if either operand is NaN, so it suffices that at
18568 // least one operand is not NaN.
18569 if (CC == ISD::SETOEQ && LHSMaybeNaN && RHSMaybeNaN)
18570 break;
18571 // UEQ should be true if either operand is NaN, but this cannot be checked
18572 // on underlying bits.
18573 if (CC == ISD::SETUEQ && (LHSMaybeNaN || RHSMaybeNaN))
18574 break;
18575 // ONE should be false if either operand is NaN, but this cannot be
18576 // checked on underlying bits.
18577 if (CC == ISD::SETONE && (LHSMaybeNaN || RHSMaybeNaN))
18578 break;
18579 // UNE should be true if either operand is NaN, so it suffices that they
18580 // are not both NaN.
18581 if (CC == ISD::SETUNE && LHSMaybeNaN && RHSMaybeNaN)
18582 break;
18583
18584 const std::optional<bool> KnownEq =
18585 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18586
18587 if (!KnownEq)
18588 break;
18589
18590 if (*KnownEq)
18591 return (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ)
18592 ? ISD::SETEQ
18593 : ISD::SETNE;
18594
18595 return (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ)
18596 ? ISD::SETFALSE
18597 : ISD::SETTRUE;
18598 }
18599 case ISD::SETLT:
18600 case ISD::SETOLT:
18601 case ISD::SETULT:
18602 case ISD::SETGE:
18603 case ISD::SETOGE:
18604 case ISD::SETUGE: {
18605 // OLT should be false if either operand is NaN.
18606 // Since NaNs have maximum exponent and nonzero mantissa, false positives
18607 // are only possible if the RHS is NaN. (No issue with RHS == +inf since
18608 // the inequality is strict)
18609 if (CC == ISD::SETOLT && RHSMaybeNaN)
18610 break;
18611 // ULT should be true if either operand is NaN, but this cannot be ensured
18612 // with a truncated comparison.
18613 if (CC == ISD::SETULT && (LHSMaybeNaN || RHSMaybeNaN))
18614 break;
18615 // OGE should be false if either operand is NaN, but this cannot be
18616 // ensured with a truncated comparison.
18617 if (CC == ISD::SETOGE && (LHSMaybeNaN || RHSMaybeNaN))
18618 break;
18619 // UGE should be true if either operand is NaN.
18620 // False negatives are only possible if the RHS is NaN.
18621 // (No issue with RHS == +inf since the inequality is inclusive)
18622 if (CC == ISD::SETUGE && RHSMaybeNaN)
18623 break;
18624
18625 const std::optional<bool> KnownUge =
18626 KnownBits::uge(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18627
18628 if (!KnownUge)
18629 break;
18630
18631 if (*KnownUge) {
18632 // LHS.lo32 uge RHS.lo32, so LHS >= RHS iff LHS.hi32 >= RHS.hi32
18633 return (CC == ISD::SETLT || CC == ISD::SETOLT || CC == ISD::SETULT)
18634 ? ISD::SETLT
18635 : ISD::SETGE;
18636 }
18637 // LHS.lo32 ult RHS.lo32, so LHS >= RHS iff LHS.hi32 > RHS.hi32
18638 return (CC == ISD::SETLT || CC == ISD::SETOLT || CC == ISD::SETULT)
18639 ? ISD::SETLE
18640 : ISD::SETGT;
18641 }
18642 case ISD::SETLE:
18643 case ISD::SETOLE:
18644 case ISD::SETULE:
18645 case ISD::SETGT:
18646 case ISD::SETOGT:
18647 case ISD::SETUGT: {
18648 // OLE should be false if either operand is NaN, but this cannot be
18649 // ensured with a truncated comparison.
18650 if (CC == ISD::SETOLE && (LHSMaybeNaN || RHSMaybeNaN))
18651 break;
18652 // ULE should be true if either operand is NaN.
18653 // False negatives are only possible if the LHS is NaN.
18654 // (No issue with LHS == +inf since the inequality is inclusive)
18655 if (CC == ISD::SETULE && LHSMaybeNaN)
18656 break;
18657 // OGT should be false if either operand is NaN.
18658 // False positives are only possible if the LHS is NaN.
18659 // (No issue with LHS == +inf since the inequality is strict)
18660 if (CC == ISD::SETOGT && LHSMaybeNaN)
18661 break;
18662 // UGT should be true if either operand is NaN, but this cannot be ensured
18663 // with a truncated comparison.
18664 if (CC == ISD::SETUGT && (LHSMaybeNaN || RHSMaybeNaN))
18665 break;
18666
18667 const std::optional<bool> KnownUle =
18668 KnownBits::ule(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18669
18670 if (!KnownUle)
18671 break;
18672
18673 if (*KnownUle) {
18674 // LHS.lo32 ule RHS.lo32, so LHS <= RHS iff LHS.hi32 <= RHS.hi32
18675 return (CC == ISD::SETLE || CC == ISD::SETOLE || CC == ISD::SETULE)
18676 ? ISD::SETLE
18677 : ISD::SETGT;
18678 }
18679 // LHS.lo32 ugt RHS.lo32, so LHS <= RHS iff LHS.hi32 < RHS.hi32
18680 return (CC == ISD::SETLE || CC == ISD::SETOLE || CC == ISD::SETULE)
18681 ? ISD::SETLT
18682 : ISD::SETGE;
18683 }
18684 }
18685
18686 return ISD::SETCC_INVALID;
18687}
18688
18689SDValue SITargetLowering::performSetCCCombine(SDNode *N,
18690 DAGCombinerInfo &DCI) const {
18691 SelectionDAG &DAG = DCI.DAG;
18692 SDLoc SL(N);
18693
18694 SDValue LHS = N->getOperand(Num: 0);
18695 SDValue RHS = N->getOperand(Num: 1);
18696 EVT VT = LHS.getValueType();
18697 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
18698
18699 auto *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
18700 if (!CRHS) {
18701 CRHS = dyn_cast<ConstantSDNode>(Val&: LHS);
18702 if (CRHS) {
18703 std::swap(a&: LHS, b&: RHS);
18704 CC = getSetCCSwappedOperands(Operation: CC);
18705 }
18706 }
18707
18708 if (CRHS) {
18709 if (VT == MVT::i32 && LHS.getOpcode() == ISD::SIGN_EXTEND &&
18710 isBoolSGPR(V: LHS.getOperand(i: 0))) {
18711 // setcc (sext from i1 cc), -1, ne|sgt|ult) => not cc => xor cc, -1
18712 // setcc (sext from i1 cc), -1, eq|sle|uge) => cc
18713 // setcc (sext from i1 cc), 0, eq|sge|ule) => not cc => xor cc, -1
18714 // setcc (sext from i1 cc), 0, ne|ugt|slt) => cc
18715 if ((CRHS->isAllOnes() &&
18716 (CC == ISD::SETNE || CC == ISD::SETGT || CC == ISD::SETULT)) ||
18717 (CRHS->isZero() &&
18718 (CC == ISD::SETEQ || CC == ISD::SETGE || CC == ISD::SETULE)))
18719 return DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i1, N1: LHS.getOperand(i: 0),
18720 N2: DAG.getAllOnesConstant(DL: SL, VT: MVT::i1));
18721 if ((CRHS->isAllOnes() &&
18722 (CC == ISD::SETEQ || CC == ISD::SETLE || CC == ISD::SETUGE)) ||
18723 (CRHS->isZero() &&
18724 (CC == ISD::SETNE || CC == ISD::SETUGT || CC == ISD::SETLT)))
18725 return LHS.getOperand(i: 0);
18726 }
18727
18728 const APInt &CRHSVal = CRHS->getAPIntValue();
18729 if ((CC == ISD::SETEQ || CC == ISD::SETNE) &&
18730 LHS.getOpcode() == ISD::SELECT &&
18731 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
18732 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2)) &&
18733 isBoolSGPR(V: LHS.getOperand(i: 0))) {
18734 // Given CT != FT:
18735 // setcc (select cc, CT, CF), CF, eq => xor cc, -1
18736 // setcc (select cc, CT, CF), CF, ne => cc
18737 // setcc (select cc, CT, CF), CT, ne => xor cc, -1
18738 // setcc (select cc, CT, CF), CT, eq => cc
18739 const APInt &CT = LHS.getConstantOperandAPInt(i: 1);
18740 const APInt &CF = LHS.getConstantOperandAPInt(i: 2);
18741
18742 if (CT != CF) {
18743 if ((CF == CRHSVal && CC == ISD::SETEQ) ||
18744 (CT == CRHSVal && CC == ISD::SETNE))
18745 return DAG.getNOT(DL: SL, Val: LHS.getOperand(i: 0), VT: MVT::i1);
18746 if ((CF == CRHSVal && CC == ISD::SETNE) ||
18747 (CT == CRHSVal && CC == ISD::SETEQ))
18748 return LHS.getOperand(i: 0);
18749 }
18750 }
18751 }
18752
18753 // Truncate 64-bit setcc to test only upper 32-bits of its operands in the
18754 // following cases where information about the lower 32-bits of its operands
18755 // is known:
18756 //
18757 // If LHS.lo32 == RHS.lo32:
18758 // setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, eq/ne
18759 // If LHS.lo32 != RHS.lo32:
18760 // setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, false/true
18761 // If LHS.lo32 >= RHS.lo32 (unsigned):
18762 // setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]lt/ge
18763 // If LHS.lo32 > RHS.lo32 (unsigned):
18764 // setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]lt/ge
18765 // If LHS.lo32 <= RHS.lo32 (unsigned):
18766 // setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]le/gt
18767 // If LHS.lo32 < RHS.lo32 (unsigned):
18768 // setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]le/gt
18769 if (VT == MVT::i64) {
18770 const KnownBits LHSKnownLo32 = DAG.computeKnownBits(Op: LHS).trunc(BitWidth: 32);
18771 const KnownBits RHSKnownLo32 = DAG.computeKnownBits(Op: RHS).trunc(BitWidth: 32);
18772
18773 // NewCC is valid iff we can truncate the setcc to only test the upper 32
18774 // bits
18775 ISD::CondCode NewCC = ISD::SETCC_INVALID;
18776
18777 switch (CC) {
18778 default:
18779 break;
18780 case ISD::SETEQ: {
18781 const std::optional<bool> KnownEq =
18782 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18783 if (KnownEq)
18784 NewCC = *KnownEq ? ISD::SETEQ : ISD::SETFALSE;
18785
18786 break;
18787 }
18788 case ISD::SETNE: {
18789 const std::optional<bool> KnownEq =
18790 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18791 if (KnownEq)
18792 NewCC = *KnownEq ? ISD::SETNE : ISD::SETTRUE;
18793
18794 break;
18795 }
18796 case ISD::SETULT:
18797 case ISD::SETUGE:
18798 case ISD::SETLT:
18799 case ISD::SETGE: {
18800 const std::optional<bool> KnownUge =
18801 KnownBits::uge(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18802 if (KnownUge) {
18803 if (*KnownUge) {
18804 // LHS.lo32 uge RHS.lo32, so LHS >= RHS iff LHS.hi32 >= RHS.hi32
18805 NewCC = CC;
18806 } else {
18807 // LHS.lo32 ult RHS.lo32, so LHS >= RHS iff LHS.hi32 > RHS.hi32
18808 NewCC = CC == ISD::SETULT ? ISD::SETULE
18809 : CC == ISD::SETUGE ? ISD::SETUGT
18810 : CC == ISD::SETLT ? ISD::SETLE
18811 : ISD::SETGT;
18812 }
18813 }
18814 break;
18815 }
18816 case ISD::SETULE:
18817 case ISD::SETUGT:
18818 case ISD::SETLE:
18819 case ISD::SETGT: {
18820 const std::optional<bool> KnownUle =
18821 KnownBits::ule(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18822 if (KnownUle) {
18823 if (*KnownUle) {
18824 // LHS.lo32 ule RHS.lo32, so LHS <= RHS iff LHS.hi32 <= RHS.hi32
18825 NewCC = CC;
18826 } else {
18827 // LHS.lo32 ugt RHS.lo32, so LHS <= RHS iff LHS.hi32 < RHS.hi32
18828 NewCC = CC == ISD::SETULE ? ISD::SETULT
18829 : CC == ISD::SETUGT ? ISD::SETUGE
18830 : CC == ISD::SETLE ? ISD::SETLT
18831 : ISD::SETGE;
18832 }
18833 }
18834 break;
18835 }
18836 }
18837
18838 if (NewCC != ISD::SETCC_INVALID)
18839 return DAG.getSetCC(DL: SL, VT: N->getValueType(ResNo: 0), LHS: getHiHalf64(Op: LHS, DAG),
18840 RHS: getHiHalf64(Op: RHS, DAG), Cond: NewCC);
18841 }
18842
18843 // Eliminate setcc by using carryout from add/sub instruction
18844
18845 // LHS = ADD i64 RHS, Z LHSlo = UADDO i32 RHSlo, Zlo
18846 // setcc LHS ult RHS -> LHSHi = UADDO_CARRY i32 RHShi, Zhi
18847 // similarly for subtraction
18848
18849 // LHS = ADD i64 Y, 1 LHSlo = UADDO i32 Ylo, 1
18850 // setcc LHS eq 0 -> LHSHi = UADDO_CARRY i32 Yhi, 0
18851
18852 if (VT == MVT::i64 && ((CC == ISD::SETULT &&
18853 sd_match(N: LHS, P: m_Add(L: m_Specific(N: RHS), R: m_Value()))) ||
18854 (CC == ISD::SETUGT &&
18855 sd_match(N: LHS, P: m_Sub(L: m_Specific(N: RHS), R: m_Value()))) ||
18856 (CC == ISD::SETEQ && CRHS && CRHS->isZero() &&
18857 sd_match(N: LHS, P: m_Add(L: m_Value(), R: m_One()))))) {
18858 bool IsAdd = LHS.getOpcode() == ISD::ADD;
18859
18860 SDValue Op0 = LHS.getOperand(i: 0);
18861 SDValue Op1 = LHS.getOperand(i: 1);
18862
18863 SDValue Op0Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Op0);
18864 SDValue Op1Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Op1);
18865
18866 SDValue Op0Hi = getHiHalf64(Op: Op0, DAG);
18867 SDValue Op1Hi = getHiHalf64(Op: Op1, DAG);
18868
18869 SDValue NodeLo =
18870 DAG.getNode(Opcode: IsAdd ? ISD::UADDO : ISD::USUBO, DL: SL,
18871 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1), Ops: {Op0Lo, Op1Lo});
18872
18873 SDValue CarryInHi = NodeLo.getValue(R: 1);
18874 SDValue NodeHi = DAG.getNode(Opcode: IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY,
18875 DL: SL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1),
18876 Ops: {Op0Hi, Op1Hi, CarryInHi});
18877
18878 SDValue ResultLo = NodeLo.getValue(R: 0);
18879 SDValue ResultHi = NodeHi.getValue(R: 0);
18880
18881 SDValue JoinedResult =
18882 DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {ResultLo, ResultHi});
18883
18884 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: JoinedResult);
18885 SDValue Overflow = NodeHi.getValue(R: 1);
18886 DCI.CombineTo(N: LHS.getNode(), Res: Result);
18887 return Overflow;
18888 }
18889
18890 if (VT != MVT::f32 && VT != MVT::f64 &&
18891 (!Subtarget->has16BitInsts() || VT != MVT::f16))
18892 return SDValue();
18893
18894 // Match isinf/isfinite pattern
18895 // (fcmp oeq (fabs x), inf) -> (fp_class x, (p_infinity | n_infinity))
18896 // (fcmp one (fabs x), inf) -> (fp_class x,
18897 // (p_normal | n_normal | p_subnormal | n_subnormal | p_zero | n_zero)
18898 if ((CC == ISD::SETOEQ || CC == ISD::SETONE) &&
18899 LHS.getOpcode() == ISD::FABS) {
18900 const ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
18901 if (!CRHS)
18902 return SDValue();
18903
18904 const APFloat &APF = CRHS->getValueAPF();
18905 if (APF.isInfinity() && !APF.isNegative()) {
18906 const unsigned IsInfMask =
18907 SIInstrFlags::P_INFINITY | SIInstrFlags::N_INFINITY;
18908 const unsigned IsFiniteMask =
18909 SIInstrFlags::N_ZERO | SIInstrFlags::P_ZERO | SIInstrFlags::N_NORMAL |
18910 SIInstrFlags::P_NORMAL | SIInstrFlags::N_SUBNORMAL |
18911 SIInstrFlags::P_SUBNORMAL;
18912 unsigned Mask = CC == ISD::SETOEQ ? IsInfMask : IsFiniteMask;
18913 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL: SL, VT: MVT::i1, N1: LHS.getOperand(i: 0),
18914 N2: DAG.getConstant(Val: Mask, DL: SL, VT: MVT::i32));
18915 }
18916 }
18917
18918 if (VT == MVT::f64) {
18919 ISD::CondCode HiHalfCC = tryReduceF64CompareToHiHalf(CC, LHS, RHS, DAG);
18920 if (HiHalfCC != ISD::SETCC_INVALID)
18921 return DAG.getSetCC(DL: SL, VT: N->getValueType(ResNo: 0), LHS: getHiHalf64(Op: LHS, DAG),
18922 RHS: getHiHalf64(Op: RHS, DAG), Cond: HiHalfCC);
18923 }
18924
18925 return SDValue();
18926}
18927
18928SDValue
18929SITargetLowering::performCvtF32UByteNCombine(SDNode *N,
18930 DAGCombinerInfo &DCI) const {
18931 SelectionDAG &DAG = DCI.DAG;
18932 SDLoc SL(N);
18933 unsigned Offset = N->getOpcode() - AMDGPUISD::CVT_F32_UBYTE0;
18934
18935 SDValue Src = N->getOperand(Num: 0);
18936 SDValue Shift = N->getOperand(Num: 0);
18937
18938 // TODO: Extend type shouldn't matter (assuming legal types).
18939 if (Shift.getOpcode() == ISD::ZERO_EXTEND)
18940 Shift = Shift.getOperand(i: 0);
18941
18942 if (Shift.getOpcode() == ISD::SRL || Shift.getOpcode() == ISD::SHL) {
18943 // cvt_f32_ubyte1 (shl x, 8) -> cvt_f32_ubyte0 x
18944 // cvt_f32_ubyte3 (shl x, 16) -> cvt_f32_ubyte1 x
18945 // cvt_f32_ubyte0 (srl x, 16) -> cvt_f32_ubyte2 x
18946 // cvt_f32_ubyte1 (srl x, 16) -> cvt_f32_ubyte3 x
18947 // cvt_f32_ubyte0 (srl x, 8) -> cvt_f32_ubyte1 x
18948 if (auto *C = dyn_cast<ConstantSDNode>(Val: Shift.getOperand(i: 1))) {
18949 SDValue Shifted = DAG.getZExtOrTrunc(
18950 Op: Shift.getOperand(i: 0), DL: SDLoc(Shift.getOperand(i: 0)), VT: MVT::i32);
18951
18952 unsigned ShiftOffset = 8 * Offset;
18953 if (Shift.getOpcode() == ISD::SHL)
18954 ShiftOffset -= C->getZExtValue();
18955 else
18956 ShiftOffset += C->getZExtValue();
18957
18958 if (ShiftOffset < 32 && (ShiftOffset % 8) == 0) {
18959 return DAG.getNode(Opcode: AMDGPUISD::CVT_F32_UBYTE0 + ShiftOffset / 8, DL: SL,
18960 VT: MVT::f32, Operand: Shifted);
18961 }
18962 }
18963 }
18964
18965 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
18966 APInt DemandedBits = APInt::getBitsSet(numBits: 32, loBit: 8 * Offset, hiBit: 8 * Offset + 8);
18967 if (TLI.SimplifyDemandedBits(Op: Src, DemandedBits, DCI)) {
18968 // We simplified Src. If this node is not dead, visit it again so it is
18969 // folded properly.
18970 if (N->getOpcode() != ISD::DELETED_NODE)
18971 DCI.AddToWorklist(N);
18972 return SDValue(N, 0);
18973 }
18974
18975 // Handle (or x, (srl y, 8)) pattern when known bits are zero.
18976 if (SDValue DemandedSrc =
18977 TLI.SimplifyMultipleUseDemandedBits(Op: Src, DemandedBits, DAG))
18978 return DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: MVT::f32, Operand: DemandedSrc);
18979
18980 return SDValue();
18981}
18982
18983SDValue SITargetLowering::performClampCombine(SDNode *N,
18984 DAGCombinerInfo &DCI) const {
18985 ConstantFPSDNode *CSrc = dyn_cast<ConstantFPSDNode>(Val: N->getOperand(Num: 0));
18986 if (!CSrc)
18987 return SDValue();
18988
18989 const MachineFunction &MF = DCI.DAG.getMachineFunction();
18990 const APFloat &F = CSrc->getValueAPF();
18991 APFloat Zero = APFloat::getZero(Sem: F.getSemantics());
18992 if (F < Zero ||
18993 (F.isNaN() && MF.getInfo<SIMachineFunctionInfo>()->getMode().DX10Clamp)) {
18994 return DCI.DAG.getConstantFP(Val: Zero, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
18995 }
18996
18997 APFloat One = APFloat::getOne(Sem: F.getSemantics());
18998 if (F > One)
18999 return DCI.DAG.getConstantFP(Val: One, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
19000
19001 return getCanonicalConstantFP(DAG&: DCI.DAG, SL: SDLoc(N), VT: N->getValueType(ResNo: 0), C: F);
19002}
19003
19004// Check if V is the exponent result of a frexp operation. Returns the frexp
19005// input via FrexpInput if matched. We only match the exponent (not mantissa)
19006// because V_FREXP_MANT returns its input for Inf/NaN, not zero.
19007static bool isFrexpExp(SDValue V, SDValue &FrexpInput) {
19008 // ISD::FFREXP returns {mant, exp} - only match if using the exp result
19009 // (result number 1).
19010 if (V.getOpcode() == ISD::FFREXP && V.getResNo() == 1) {
19011 FrexpInput = V.getOperand(i: 0);
19012 return true;
19013 }
19014 if (sd_match(N: V, P: m_IntrinsicWOChain<Intrinsic::amdgcn_frexp_exp>(
19015 Opnds: m_Value(N&: FrexpInput))))
19016 return true;
19017 return false;
19018}
19019
19020SDValue
19021SITargetLowering::performFrexpSelectCombine(SDNode *N,
19022 DAGCombinerInfo &DCI) const {
19023 // This optimization only applies when the hardware handles inf/nan correctly.
19024 if (Subtarget->hasFractBug())
19025 return SDValue();
19026
19027 SDValue Cond = N->getOperand(Num: 0);
19028 SDValue TrueVal = N->getOperand(Num: 1);
19029 SDValue FalseVal = N->getOperand(Num: 2);
19030
19031 // Identify which operand is the frexp result and which is the zero constant.
19032 // Pattern 1: select cond, 0, frexp_result (cond true -> return 0)
19033 // Pattern 2: select cond, frexp_result, 0 (cond false -> return 0)
19034 SDValue FrexpVal;
19035 SDValue ZeroVal;
19036 bool CondSelectsZero; // If true, condition=true selects zero
19037
19038 // Check if FrexpVal comes from ISD::FFREXP (exponent result only) or
19039 // amdgcn_frexp_exp intrinsic.
19040 SDValue FrexpInput;
19041 if (isFrexpExp(V: FalseVal, FrexpInput)) {
19042 FrexpVal = FalseVal;
19043 ZeroVal = TrueVal;
19044 CondSelectsZero = true;
19045 } else if (isFrexpExp(V: TrueVal, FrexpInput)) {
19046 FrexpVal = TrueVal;
19047 ZeroVal = FalseVal;
19048 CondSelectsZero = false;
19049 } else {
19050 return SDValue();
19051 }
19052
19053 // frexp_exp returns integer, so check for integer zero.
19054 if (!isNullConstant(V: ZeroVal))
19055 return SDValue();
19056
19057 // The frexp intrinsics ignore sign, so we can strip sign ops when comparing.
19058 SDValue FrexpInputStripped = peekFPSignOps(Val: FrexpInput);
19059
19060 bool IsNonFiniteTest = false;
19061
19062 // Handle SETCC conditions for inf/nan tests.
19063 // The canonical form of these checks is fcmp + fabs.
19064 if (Cond.getOpcode() == ISD::SETCC) {
19065 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
19066 SDValue CondLHS = Cond.getOperand(i: 0);
19067 SDValue CondRHS = Cond.getOperand(i: 1);
19068
19069 // Check if LHS is fabs(FrexpInput) - required for infinity comparisons.
19070 SDValue FAbsInput;
19071 bool LHSIsFabs = sd_match(N: CondLHS, P: m_FAbs(Op: m_Value(N&: FAbsInput)));
19072 bool LHSMatchesFrexp =
19073 (CondLHS == FrexpInput) ||
19074 (LHSIsFabs && peekFPSignOps(Val: FAbsInput) == FrexpInputStripped) ||
19075 (peekFPSignOps(Val: CondLHS) == FrexpInputStripped);
19076 bool RHSMatchesFrexp = (CondRHS == FrexpInput) ||
19077 (peekFPSignOps(Val: CondRHS) == FrexpInputStripped);
19078
19079 if (CC == ISD::SETUO) {
19080 // fcmp uno x, y - true if either x or y is NaN
19081 // We can only fold if the non-frexp operand is known to never be NaN,
19082 // otherwise the comparison could be true due to the other operand.
19083 // Special case: fcmp uno x, x (same operand) is a valid NaN test.
19084 SelectionDAG &DAG = DCI.DAG;
19085 if (LHSMatchesFrexp &&
19086 (CondLHS == CondRHS || DAG.isKnownNeverNaN(Op: CondRHS)))
19087 IsNonFiniteTest = CondSelectsZero;
19088 else if (RHSMatchesFrexp && DAG.isKnownNeverNaN(Op: CondLHS))
19089 IsNonFiniteTest = CondSelectsZero;
19090 } else if ((CC == ISD::SETOEQ || CC == ISD::SETUEQ) && LHSMatchesFrexp &&
19091 LHSIsFabs &&
19092 sd_match(N: CondRHS,
19093 P: m_SpecificFP(V: APFloat::getInf(
19094 Sem: CondRHS.getValueType().getFltSemantics())))) {
19095 // fcmp oeq/ueq fabs(x), +inf - true if x is inf (or inf/nan for ueq)
19096 IsNonFiniteTest = CondSelectsZero;
19097 } else if ((CC == ISD::SETONE || CC == ISD::SETUNE) && LHSMatchesFrexp &&
19098 LHSIsFabs &&
19099 sd_match(N: CondRHS,
19100 P: m_SpecificFP(V: APFloat::getInf(
19101 Sem: CondRHS.getValueType().getFltSemantics())))) {
19102 // fcmp one/une fabs(x), +inf - true if x is NOT inf
19103 IsNonFiniteTest = !CondSelectsZero;
19104 } else if (CC == ISD::SETO) {
19105 // fcmp ord x, y - true if both are NOT NaN
19106 // We can only fold if the non-frexp operand is known to never be NaN,
19107 // otherwise the comparison could be false due to the other operand.
19108 // Special case: fcmp ord x, x (same operand) is a valid not-NaN test.
19109 SelectionDAG &DAG = DCI.DAG;
19110 if (LHSMatchesFrexp &&
19111 (CondLHS == CondRHS || DAG.isKnownNeverNaN(Op: CondRHS)))
19112 IsNonFiniteTest = !CondSelectsZero;
19113 else if (RHSMatchesFrexp && DAG.isKnownNeverNaN(Op: CondLHS))
19114 IsNonFiniteTest = !CondSelectsZero;
19115 }
19116 }
19117
19118 if (!IsNonFiniteTest)
19119 return SDValue();
19120
19121 // The select can be eliminated - just return the frexp result directly.
19122 return FrexpVal;
19123}
19124
19125SDValue SITargetLowering::performSelectCombine(SDNode *N,
19126 DAGCombinerInfo &DCI) const {
19127
19128 // Try to fold CMP + SELECT patterns with shared constants (both FP and
19129 // integer).
19130 // Detect when CMP and SELECT use the same constant and fold them to avoid
19131 // loading the constant twice. Specifically handles patterns like:
19132 // %cmp = icmp eq i32 %val, 4242
19133 // %sel = select i1 %cmp, i32 4242, i32 %other
19134 // It can be optimized to reuse %val instead of 4242 in select.
19135 SDValue Cond = N->getOperand(Num: 0);
19136 SDValue TrueVal = N->getOperand(Num: 1);
19137 SDValue FalseVal = N->getOperand(Num: 2);
19138
19139 // Check if condition is a comparison.
19140 if (Cond.getOpcode() != ISD::SETCC)
19141 return SDValue();
19142
19143 SDValue LHS = Cond.getOperand(i: 0);
19144 SDValue RHS = Cond.getOperand(i: 1);
19145 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
19146
19147 bool isFloatingPoint = LHS.getValueType().isFloatingPoint();
19148 bool isInteger = LHS.getValueType().isInteger();
19149
19150 // Handle simple floating-point and integer types only.
19151 if (!isFloatingPoint && !isInteger)
19152 return SDValue();
19153
19154 // Bare SETEQ/SETNE is the builder's NaN-impossible downgrade.
19155 bool isEquality = CC == ISD::SETEQ || (isFloatingPoint && CC == ISD::SETOEQ);
19156 bool isNonEquality =
19157 CC == ISD::SETNE || (isFloatingPoint && CC == ISD::SETONE);
19158 if (!isEquality && !isNonEquality)
19159 return SDValue();
19160
19161 SDValue ArgVal, ConstVal;
19162 if ((isFloatingPoint && isa<ConstantFPSDNode>(Val: RHS)) ||
19163 (isInteger && isa<ConstantSDNode>(Val: RHS))) {
19164 ConstVal = RHS;
19165 ArgVal = LHS;
19166 } else if ((isFloatingPoint && isa<ConstantFPSDNode>(Val: LHS)) ||
19167 (isInteger && isa<ConstantSDNode>(Val: LHS))) {
19168 ConstVal = LHS;
19169 ArgVal = RHS;
19170 } else {
19171 return SDValue();
19172 }
19173
19174 // Skip optimization for inlinable immediates.
19175 if (isFloatingPoint) {
19176 const APFloat &Val = cast<ConstantFPSDNode>(Val&: ConstVal)->getValueAPF();
19177 if (!Val.isNormal() || Subtarget->getInstrInfo()->isInlineConstant(Imm: Val))
19178 return SDValue();
19179 } else {
19180 const std::optional<int64_t> Val =
19181 cast<ConstantSDNode>(Val&: ConstVal)->getAPIntValue().trySExtValue();
19182 if (Val && AMDGPU::isInlinableIntLiteral(Literal: *Val))
19183 return SDValue();
19184 }
19185
19186 // For equality and non-equality comparisons, patterns:
19187 // select (setcc x, const), const, y -> select (setcc x, const), x, y
19188 // select (setccinv x, const), y, const -> select (setccinv x, const), y, x
19189 if (!(isEquality && TrueVal == ConstVal) &&
19190 !(isNonEquality && FalseVal == ConstVal))
19191 return SDValue();
19192
19193 // SETONE's false arm is also taken for NaN ArgVal, so require NaN excluded.
19194 if (isFloatingPoint && isNonEquality && FalseVal == ConstVal &&
19195 !Cond->getFlags().hasNoNaNs() && !DCI.DAG.isKnownNeverNaN(Op: ArgVal))
19196 return SDValue();
19197
19198 SDValue SelectLHS = (isEquality && TrueVal == ConstVal) ? ArgVal : TrueVal;
19199 SDValue SelectRHS =
19200 (isNonEquality && FalseVal == ConstVal) ? ArgVal : FalseVal;
19201 return DCI.DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Cond,
19202 N2: SelectLHS, N3: SelectRHS);
19203}
19204
19205SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
19206 DAGCombinerInfo &DCI) const {
19207 switch (N->getOpcode()) {
19208 case ISD::ABS:
19209 if (SDValue Res = promoteUniformUnaryOpToI32(Op: SDValue(N, 0), DCI))
19210 return Res;
19211 break;
19212 case ISD::ADD:
19213 case ISD::SUB:
19214 case ISD::SHL:
19215 case ISD::SRL:
19216 case ISD::SRA:
19217 case ISD::AND:
19218 case ISD::OR:
19219 case ISD::XOR:
19220 case ISD::MUL:
19221 case ISD::SETCC:
19222 case ISD::SELECT:
19223 case ISD::SMIN:
19224 case ISD::SMAX:
19225 case ISD::UMIN:
19226 case ISD::UMAX:
19227 case ISD::USUBSAT:
19228 case ISD::UADDSAT:
19229 if (auto Res = promoteUniformOpToI32(Op: SDValue(N, 0), DCI))
19230 return Res;
19231 break;
19232 default:
19233 break;
19234 }
19235
19236 if (getTargetMachine().getOptLevel() == CodeGenOptLevel::None)
19237 return SDValue();
19238
19239 switch (N->getOpcode()) {
19240 case ISD::ADD:
19241 return performAddCombine(N, DCI);
19242 case ISD::PTRADD:
19243 return performPtrAddCombine(N, DCI);
19244 case ISD::SUB:
19245 return performSubCombine(N, DCI);
19246 case ISD::FADD:
19247 return performFAddCombine(N, DCI);
19248 case ISD::FSUB:
19249 return performFSubCombine(N, DCI);
19250 case ISD::FDIV:
19251 return performFDivCombine(N, DCI);
19252 case ISD::FMUL:
19253 return performFMulCombine(N, DCI);
19254 case ISD::SETCC:
19255 return performSetCCCombine(N, DCI);
19256 case ISD::SELECT:
19257 if (auto Res = performFrexpSelectCombine(N, DCI))
19258 return Res;
19259 if (auto Res = performSelectCombine(N, DCI))
19260 return Res;
19261 break;
19262 case ISD::FMAXNUM:
19263 case ISD::FMINNUM:
19264 case ISD::FMAXNUM_IEEE:
19265 case ISD::FMINNUM_IEEE:
19266 case ISD::FMAXIMUM:
19267 case ISD::FMINIMUM:
19268 case ISD::FMAXIMUMNUM:
19269 case ISD::FMINIMUMNUM:
19270 case ISD::SMAX:
19271 case ISD::SMIN:
19272 case ISD::UMAX:
19273 case ISD::UMIN:
19274 case AMDGPUISD::FMIN_LEGACY:
19275 case AMDGPUISD::FMAX_LEGACY:
19276 return performMinMaxCombine(N, DCI);
19277 case ISD::FMA:
19278 return performFMACombine(N, DCI);
19279 case ISD::AND:
19280 return performAndCombine(N, DCI);
19281 case ISD::OR:
19282 return performOrCombine(N, DCI);
19283 case ISD::FSHR: {
19284 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19285 if (N->getValueType(ResNo: 0) == MVT::i32 && N->isDivergent() &&
19286 TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
19287 return matchPERM(N, DCI);
19288 }
19289 break;
19290 }
19291 case ISD::XOR:
19292 return performXorCombine(N, DCI);
19293 case ISD::ANY_EXTEND:
19294 case ISD::ZERO_EXTEND:
19295 return performZeroOrAnyExtendCombine(N, DCI);
19296 case ISD::SIGN_EXTEND_INREG:
19297 return performSignExtendInRegCombine(N, DCI);
19298 case AMDGPUISD::FP_CLASS:
19299 return performClassCombine(N, DCI);
19300 case ISD::FCANONICALIZE:
19301 return performFCanonicalizeCombine(N, DCI);
19302 case AMDGPUISD::RCP:
19303 return performRcpCombine(N, DCI);
19304 case ISD::FLDEXP:
19305 case AMDGPUISD::FRACT:
19306 case AMDGPUISD::RSQ:
19307 case AMDGPUISD::RCP_LEGACY:
19308 case AMDGPUISD::RCP_IFLAG:
19309 case AMDGPUISD::RSQ_CLAMP: {
19310 // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
19311 SDValue Src = N->getOperand(Num: 0);
19312 if (Src.isUndef())
19313 return Src;
19314 break;
19315 }
19316 case ISD::SINT_TO_FP:
19317 case ISD::UINT_TO_FP:
19318 return performUCharToFloatCombine(N, DCI);
19319 case ISD::FCOPYSIGN:
19320 return performFCopySignCombine(N, DCI);
19321 case AMDGPUISD::CVT_F32_UBYTE0:
19322 case AMDGPUISD::CVT_F32_UBYTE1:
19323 case AMDGPUISD::CVT_F32_UBYTE2:
19324 case AMDGPUISD::CVT_F32_UBYTE3:
19325 return performCvtF32UByteNCombine(N, DCI);
19326 case AMDGPUISD::FMED3:
19327 return performFMed3Combine(N, DCI);
19328 case AMDGPUISD::CVT_PKRTZ_F16_F32:
19329 return performCvtPkRTZCombine(N, DCI);
19330 case AMDGPUISD::CLAMP:
19331 return performClampCombine(N, DCI);
19332 case ISD::SCALAR_TO_VECTOR: {
19333 SelectionDAG &DAG = DCI.DAG;
19334 EVT VT = N->getValueType(ResNo: 0);
19335
19336 // v2i16 (scalar_to_vector i16:x) -> v2i16 (bitcast (any_extend i16:x))
19337 if (VT == MVT::v2i16 || VT == MVT::v2f16 || VT == MVT::v2bf16) {
19338 SDLoc SL(N);
19339 SDValue Src = N->getOperand(Num: 0);
19340 EVT EltVT = Src.getValueType();
19341 if (EltVT != MVT::i16)
19342 Src = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Src);
19343
19344 SDValue Ext = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Src);
19345 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Ext);
19346 }
19347
19348 break;
19349 }
19350 case ISD::EXTRACT_VECTOR_ELT:
19351 return performExtractVectorEltCombine(N, DCI);
19352 case ISD::INSERT_VECTOR_ELT:
19353 return performInsertVectorEltCombine(N, DCI);
19354 case ISD::FP_ROUND:
19355 return performFPRoundCombine(N, DCI);
19356 case ISD::LOAD: {
19357 if (SDValue Widened = widenLoad(Ld: cast<LoadSDNode>(Val: N), DCI))
19358 return Widened;
19359 [[fallthrough]];
19360 }
19361 default: {
19362 if (!DCI.isBeforeLegalize()) {
19363 if (MemSDNode *MemNode = dyn_cast<MemSDNode>(Val: N))
19364 return performMemSDNodeCombine(N: MemNode, DCI);
19365 }
19366
19367 break;
19368 }
19369 }
19370
19371 return AMDGPUTargetLowering::PerformDAGCombine(N, DCI);
19372}
19373
19374/// Helper function for adjustWritemask
19375static unsigned SubIdx2Lane(unsigned Idx) {
19376 switch (Idx) {
19377 default:
19378 return ~0u;
19379 case AMDGPU::sub0:
19380 return 0;
19381 case AMDGPU::sub1:
19382 return 1;
19383 case AMDGPU::sub2:
19384 return 2;
19385 case AMDGPU::sub3:
19386 return 3;
19387 case AMDGPU::sub4:
19388 return 4; // Possible with TFE/LWE
19389 }
19390}
19391
19392/// Adjust the writemask of MIMG, VIMAGE or VSAMPLE instructions
19393SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
19394 SelectionDAG &DAG) const {
19395 unsigned Opcode = Node->getMachineOpcode();
19396
19397 // Subtract 1 because the vdata output is not a MachineSDNode operand.
19398 int D16Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::d16) - 1;
19399 if (D16Idx >= 0 && Node->getConstantOperandVal(Num: D16Idx))
19400 return Node; // not implemented for D16
19401
19402 SDNode *Users[5] = {nullptr};
19403 unsigned Lane = 0;
19404 unsigned DmaskIdx =
19405 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::dmask) - 1;
19406 unsigned OldDmask = Node->getConstantOperandVal(Num: DmaskIdx);
19407 unsigned NewDmask = 0;
19408 unsigned TFEIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::tfe) - 1;
19409 unsigned LWEIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::lwe) - 1;
19410 bool UsesTFC = (int(TFEIdx) >= 0 && Node->getConstantOperandVal(Num: TFEIdx)) ||
19411 (int(LWEIdx) >= 0 && Node->getConstantOperandVal(Num: LWEIdx));
19412 unsigned TFCLane = 0;
19413 bool HasChain = Node->getNumValues() > 1;
19414
19415 if (OldDmask == 0) {
19416 // These are folded out, but on the chance it happens don't assert.
19417 return Node;
19418 }
19419
19420 unsigned OldBitsSet = llvm::popcount(Value: OldDmask);
19421 // Work out which is the TFE/LWE lane if that is enabled.
19422 if (UsesTFC) {
19423 TFCLane = OldBitsSet;
19424 }
19425
19426 // Try to figure out the used register components
19427 for (SDUse &Use : Node->uses()) {
19428
19429 // Don't look at users of the chain.
19430 if (Use.getResNo() != 0)
19431 continue;
19432
19433 SDNode *User = Use.getUser();
19434
19435 // Abort if we can't understand the usage
19436 if (!User->isMachineOpcode() ||
19437 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
19438 return Node;
19439
19440 // Lane means which subreg of %vgpra_vgprb_vgprc_vgprd is used.
19441 // Note that subregs are packed, i.e. Lane==0 is the first bit set
19442 // in OldDmask, so it can be any of X,Y,Z,W; Lane==1 is the second bit
19443 // set, etc.
19444 Lane = SubIdx2Lane(Idx: User->getConstantOperandVal(Num: 1));
19445 if (Lane == ~0u)
19446 return Node;
19447
19448 // Check if the use is for the TFE/LWE generated result at VGPRn+1.
19449 if (UsesTFC && Lane == TFCLane) {
19450 Users[Lane] = User;
19451 } else {
19452 // Set which texture component corresponds to the lane.
19453 unsigned Comp;
19454 for (unsigned i = 0, Dmask = OldDmask; (i <= Lane) && (Dmask != 0); i++) {
19455 Comp = llvm::countr_zero(Val: Dmask);
19456 Dmask &= ~(1 << Comp);
19457 }
19458
19459 // Abort if we have more than one user per component.
19460 if (Users[Lane])
19461 return Node;
19462
19463 Users[Lane] = User;
19464 NewDmask |= 1 << Comp;
19465 }
19466 }
19467
19468 // Don't allow 0 dmask, as hardware assumes one channel enabled.
19469 bool NoChannels = !NewDmask;
19470 if (NoChannels) {
19471 if (!UsesTFC) {
19472 // No uses of the result and not using TFC. Then do nothing.
19473 return Node;
19474 }
19475 // If the original dmask has one channel - then nothing to do
19476 if (OldBitsSet == 1)
19477 return Node;
19478 // Use an arbitrary dmask - required for the instruction to work
19479 NewDmask = 1;
19480 }
19481 // Abort if there's no change
19482 if (NewDmask == OldDmask)
19483 return Node;
19484
19485 unsigned BitsSet = llvm::popcount(Value: NewDmask);
19486
19487 // Check for TFE or LWE - increase the number of channels by one to account
19488 // for the extra return value
19489 // This will need adjustment for D16 if this is also included in
19490 // adjustWriteMask (this function) but at present D16 are excluded.
19491 unsigned NewChannels = BitsSet + UsesTFC;
19492
19493 int NewOpcode =
19494 AMDGPU::getMaskedMIMGOp(Opc: Node->getMachineOpcode(), NewChannels);
19495 assert(NewOpcode != -1 &&
19496 NewOpcode != static_cast<int>(Node->getMachineOpcode()) &&
19497 "failed to find equivalent MIMG op");
19498
19499 // Adjust the writemask in the node
19500 SmallVector<SDValue, 12> Ops;
19501 llvm::append_range(C&: Ops, R: Node->ops().take_front(N: DmaskIdx));
19502 Ops.push_back(Elt: DAG.getTargetConstant(Val: NewDmask, DL: SDLoc(Node), VT: MVT::i32));
19503 llvm::append_range(C&: Ops, R: Node->ops().drop_front(N: DmaskIdx + 1));
19504
19505 MVT SVT = Node->getValueType(ResNo: 0).getVectorElementType().getSimpleVT();
19506
19507 MVT ResultVT = NewChannels == 1
19508 ? SVT
19509 : MVT::getVectorVT(VT: SVT, NumElements: NewChannels == 3 ? 4
19510 : NewChannels == 5 ? 8
19511 : NewChannels);
19512 SDVTList NewVTList =
19513 HasChain ? DAG.getVTList(VT1: ResultVT, VT2: MVT::Other) : DAG.getVTList(VT: ResultVT);
19514
19515 MachineSDNode *NewNode =
19516 DAG.getMachineNode(Opcode: NewOpcode, dl: SDLoc(Node), VTs: NewVTList, Ops);
19517
19518 if (HasChain) {
19519 // Update chain.
19520 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: Node->memoperands());
19521 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: SDValue(NewNode, 1));
19522 }
19523
19524 if (NewChannels == 1) {
19525 assert(Node->hasNUsesOfValue(1, 0));
19526 SDNode *Copy =
19527 DAG.getMachineNode(Opcode: TargetOpcode::COPY, dl: SDLoc(Node),
19528 VT: Users[Lane]->getValueType(ResNo: 0), Op1: SDValue(NewNode, 0));
19529 DAG.ReplaceAllUsesWith(From: Users[Lane], To: Copy);
19530 return nullptr;
19531 }
19532
19533 // Update the users of the node with the new indices
19534 for (unsigned i = 0, Idx = AMDGPU::sub0; i < 5; ++i) {
19535 SDNode *User = Users[i];
19536 if (!User) {
19537 // Handle the special case of NoChannels. We set NewDmask to 1 above, but
19538 // Users[0] is still nullptr because channel 0 doesn't really have a use.
19539 if (i || !NoChannels)
19540 continue;
19541 } else {
19542 SDValue Op = DAG.getTargetConstant(Val: Idx, DL: SDLoc(User), VT: MVT::i32);
19543 SDNode *NewUser = DAG.UpdateNodeOperands(N: User, Op1: SDValue(NewNode, 0), Op2: Op);
19544 if (NewUser != User) {
19545 DAG.ReplaceAllUsesWith(From: SDValue(User, 0), To: SDValue(NewUser, 0));
19546 DAG.RemoveDeadNode(N: User);
19547 }
19548 }
19549
19550 switch (Idx) {
19551 default:
19552 break;
19553 case AMDGPU::sub0:
19554 Idx = AMDGPU::sub1;
19555 break;
19556 case AMDGPU::sub1:
19557 Idx = AMDGPU::sub2;
19558 break;
19559 case AMDGPU::sub2:
19560 Idx = AMDGPU::sub3;
19561 break;
19562 case AMDGPU::sub3:
19563 Idx = AMDGPU::sub4;
19564 break;
19565 }
19566 }
19567
19568 DAG.RemoveDeadNode(N: Node);
19569 return nullptr;
19570}
19571
19572static bool isFrameIndexOp(SDValue Op) {
19573 if (Op.getOpcode() == ISD::AssertZext)
19574 Op = Op.getOperand(i: 0);
19575
19576 return isa<FrameIndexSDNode>(Val: Op);
19577}
19578
19579/// Legalize target independent instructions (e.g. INSERT_SUBREG)
19580/// with frame index operands.
19581/// LLVM assumes that inputs are to these instructions are registers.
19582SDNode *
19583SITargetLowering::legalizeTargetIndependentNode(SDNode *Node,
19584 SelectionDAG &DAG) const {
19585 if (Node->getOpcode() == ISD::CopyToReg) {
19586 RegisterSDNode *DestReg = cast<RegisterSDNode>(Val: Node->getOperand(Num: 1));
19587 SDValue SrcVal = Node->getOperand(Num: 2);
19588
19589 // Insert a copy to a VReg_1 virtual register so LowerI1Copies doesn't have
19590 // to try understanding copies to physical registers.
19591 if (SrcVal.getValueType() == MVT::i1 && DestReg->getReg().isPhysical()) {
19592 SDLoc SL(Node);
19593 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
19594 SDValue VReg = DAG.getRegister(
19595 Reg: MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_1RegClass), VT: MVT::i1);
19596
19597 SDNode *Glued = Node->getGluedNode();
19598 SDValue ToVReg = DAG.getCopyToReg(
19599 Chain: Node->getOperand(Num: 0), dl: SL, Reg: VReg, N: SrcVal,
19600 Glue: SDValue(Glued, Glued ? Glued->getNumValues() - 1 : 0));
19601 SDValue ToResultReg = DAG.getCopyToReg(Chain: ToVReg, dl: SL, Reg: SDValue(DestReg, 0),
19602 N: VReg, Glue: ToVReg.getValue(R: 1));
19603 DAG.ReplaceAllUsesWith(From: Node, To: ToResultReg.getNode());
19604 DAG.RemoveDeadNode(N: Node);
19605 return ToResultReg.getNode();
19606 }
19607 }
19608
19609 SmallVector<SDValue, 8> Ops;
19610 for (unsigned i = 0; i < Node->getNumOperands(); ++i) {
19611 if (!isFrameIndexOp(Op: Node->getOperand(Num: i))) {
19612 Ops.push_back(Elt: Node->getOperand(Num: i));
19613 continue;
19614 }
19615
19616 SDLoc DL(Node);
19617 Ops.push_back(Elt: SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL,
19618 VT: Node->getOperand(Num: i).getValueType(),
19619 Op1: Node->getOperand(Num: i)),
19620 0));
19621 }
19622
19623 return DAG.UpdateNodeOperands(N: Node, Ops);
19624}
19625
19626/// Fold the instructions after selecting them.
19627/// Returns null if users were already updated.
19628SDNode *SITargetLowering::PostISelFolding(MachineSDNode *Node,
19629 SelectionDAG &DAG) const {
19630 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19631 unsigned Opcode = Node->getMachineOpcode();
19632
19633 if (TII->isImage(Opcode) && !TII->get(Opcode).mayStore() &&
19634 !TII->isGather4(Opcode) &&
19635 AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::dmask)) {
19636 return adjustWritemask(Node, DAG);
19637 }
19638
19639 if (Opcode == AMDGPU::INSERT_SUBREG || Opcode == AMDGPU::REG_SEQUENCE) {
19640 legalizeTargetIndependentNode(Node, DAG);
19641 return Node;
19642 }
19643
19644 switch (Opcode) {
19645 case AMDGPU::V_DIV_SCALE_F32_e64:
19646 case AMDGPU::V_DIV_SCALE_F64_e64: {
19647 // Satisfy the operand register constraint when one of the inputs is
19648 // undefined. Ordinarily each undef value will have its own implicit_def of
19649 // a vreg, so force these to use a single register.
19650 SDValue Src0 = Node->getOperand(Num: 1);
19651 SDValue Src1 = Node->getOperand(Num: 3);
19652 SDValue Src2 = Node->getOperand(Num: 5);
19653
19654 if ((Src0.isMachineOpcode() &&
19655 Src0.getMachineOpcode() != AMDGPU::IMPLICIT_DEF) &&
19656 (Src0 == Src1 || Src0 == Src2))
19657 break;
19658
19659 MVT VT = Src0.getValueType().getSimpleVT();
19660 const TargetRegisterClass *RC =
19661 getRegClassFor(VT, isDivergent: Src0.getNode()->isDivergent());
19662
19663 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
19664 SDValue UndefReg = DAG.getRegister(Reg: MRI.createVirtualRegister(RegClass: RC), VT);
19665
19666 SDValue ImpDef = DAG.getCopyToReg(Chain: DAG.getEntryNode(), dl: SDLoc(Node), Reg: UndefReg,
19667 N: Src0, Glue: SDValue());
19668
19669 // src0 must be the same register as src1 or src2, even if the value is
19670 // undefined, so make sure we don't violate this constraint.
19671 if (Src0.isMachineOpcode() &&
19672 Src0.getMachineOpcode() == AMDGPU::IMPLICIT_DEF) {
19673 if (Src1.isMachineOpcode() &&
19674 Src1.getMachineOpcode() != AMDGPU::IMPLICIT_DEF)
19675 Src0 = Src1;
19676 else if (Src2.isMachineOpcode() &&
19677 Src2.getMachineOpcode() != AMDGPU::IMPLICIT_DEF)
19678 Src0 = Src2;
19679 else {
19680 assert(Src1.getMachineOpcode() == AMDGPU::IMPLICIT_DEF);
19681 Src0 = UndefReg;
19682 Src1 = UndefReg;
19683 }
19684 } else
19685 break;
19686
19687 SmallVector<SDValue, 9> Ops(Node->ops());
19688 Ops[1] = Src0;
19689 Ops[3] = Src1;
19690 Ops[5] = Src2;
19691 Ops.push_back(Elt: ImpDef.getValue(R: 1));
19692 return DAG.getMachineNode(Opcode, dl: SDLoc(Node), VTs: Node->getVTList(), Ops);
19693 }
19694 default:
19695 break;
19696 }
19697
19698 return Node;
19699}
19700
19701// Any MIMG instructions that use tfe or lwe require an initialization of the
19702// result register that will be written in the case of a memory access failure.
19703// The required code is also added to tie this init code to the result of the
19704// img instruction.
19705void SITargetLowering::AddMemOpInit(MachineInstr &MI) const {
19706 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19707 const SIRegisterInfo &TRI = TII->getRegisterInfo();
19708 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
19709 MachineBasicBlock &MBB = *MI.getParent();
19710
19711 int DstIdx =
19712 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdata);
19713 unsigned InitIdx = 0;
19714
19715 if (TII->isImage(MI)) {
19716 MachineOperand *TFE = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe);
19717 MachineOperand *LWE = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::lwe);
19718 MachineOperand *D16 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::d16);
19719
19720 if (!TFE && !LWE) // intersect_ray
19721 return;
19722
19723 unsigned TFEVal = TFE ? TFE->getImm() : 0;
19724 unsigned LWEVal = LWE ? LWE->getImm() : 0;
19725 unsigned D16Val = D16 ? D16->getImm() : 0;
19726
19727 if (!TFEVal && !LWEVal)
19728 return;
19729
19730 // At least one of TFE or LWE are non-zero
19731 // We have to insert a suitable initialization of the result value and
19732 // tie this to the dest of the image instruction.
19733
19734 // Calculate which dword we have to initialize to 0.
19735 MachineOperand *MO_Dmask = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dmask);
19736
19737 // check that dmask operand is found.
19738 assert(MO_Dmask && "Expected dmask operand in instruction");
19739
19740 unsigned dmask = MO_Dmask->getImm();
19741 // Determine the number of active lanes taking into account the
19742 // Gather4 special case
19743 unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(Value: dmask);
19744
19745 bool Packed = !Subtarget->hasUnpackedD16VMem();
19746
19747 InitIdx = D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1;
19748
19749 // Abandon attempt if the dst size isn't large enough
19750 // - this is in fact an error but this is picked up elsewhere and
19751 // reported correctly.
19752 const TargetRegisterClass *DstRC = TII->getRegClass(MCID: MI.getDesc(), OpNum: DstIdx);
19753
19754 uint32_t DstSize = TRI.getRegSizeInBits(RC: *DstRC) / 32;
19755 if (DstSize < InitIdx)
19756 return;
19757 } else if (TII->isMUBUF(MI) && AMDGPU::getMUBUFTfe(Opc: MI.getOpcode())) {
19758 const TargetRegisterClass *DstRC = TII->getRegClass(MCID: MI.getDesc(), OpNum: DstIdx);
19759 InitIdx = TRI.getRegSizeInBits(RC: *DstRC) / 32;
19760 } else {
19761 return;
19762 }
19763
19764 const DebugLoc &DL = MI.getDebugLoc();
19765
19766 // Create a register for the initialization value.
19767 Register PrevDst = MRI.cloneVirtualRegister(VReg: MI.getOperand(i: DstIdx).getReg());
19768 unsigned NewDst = 0; // Final initialized value will be in here
19769
19770 // If PRTStrictNull feature is enabled (the default) then initialize
19771 // all the result registers to 0, otherwise just the error indication
19772 // register (VGPRn+1)
19773 unsigned SizeLeft = Subtarget->usePRTStrictNull() ? InitIdx : 1;
19774 unsigned CurrIdx = Subtarget->usePRTStrictNull() ? 0 : (InitIdx - 1);
19775
19776 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: PrevDst);
19777 for (; SizeLeft; SizeLeft--, CurrIdx++) {
19778 NewDst = MRI.createVirtualRegister(RegClass: TII->getOpRegClass(MI, OpNo: DstIdx));
19779 // Initialize dword
19780 Register SubReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
19781 // clang-format off
19782 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: SubReg)
19783 .addImm(Val: 0);
19784 // clang-format on
19785 // Insert into the super-reg
19786 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: NewDst)
19787 .addReg(RegNo: PrevDst)
19788 .addReg(RegNo: SubReg)
19789 .addImm(Val: SIRegisterInfo::getSubRegFromChannel(Channel: CurrIdx));
19790
19791 PrevDst = NewDst;
19792 }
19793
19794 // Add as an implicit operand
19795 MI.addOperand(Op: MachineOperand::CreateReg(Reg: NewDst, isDef: false, isImp: true));
19796
19797 // Tie the just added implicit operand to the dst
19798 MI.tieOperands(DefIdx: DstIdx, UseIdx: MI.getNumOperands() - 1);
19799}
19800
19801/// Assign the register class depending on the number of
19802/// bits set in the writemask
19803void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
19804 SDNode *Node) const {
19805 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19806
19807 MachineFunction *MF = MI.getMF();
19808 MachineRegisterInfo &MRI = MF->getRegInfo();
19809
19810 if (TII->isVOP3(Opcode: MI.getOpcode())) {
19811 // Make sure constant bus requirements are respected.
19812 TII->legalizeOperandsVOP3(MRI, MI);
19813
19814 if (TII->isMAI(MI)) {
19815 // The ordinary src0, src1, src2 were legalized above.
19816 //
19817 // We have to also legalize the appended v_mfma_ld_scale_b32 operands,
19818 // as a separate instruction.
19819 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
19820 Name: AMDGPU::OpName::scale_src0);
19821 if (Src0Idx != -1) {
19822 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
19823 Name: AMDGPU::OpName::scale_src1);
19824 if (TII->usesConstantBus(MRI, MI, OpIdx: Src0Idx) &&
19825 TII->usesConstantBus(MRI, MI, OpIdx: Src1Idx))
19826 TII->legalizeOpWithMove(MI, OpIdx: Src1Idx);
19827 }
19828 }
19829
19830 return;
19831 }
19832
19833 if (TII->isImage(MI))
19834 TII->enforceOperandRCAlignment(MI, OpName: AMDGPU::OpName::vaddr);
19835}
19836
19837static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL,
19838 uint64_t Val) {
19839 SDValue K = DAG.getTargetConstant(Val, DL, VT: MVT::i32);
19840 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: K), 0);
19841}
19842
19843MachineSDNode *SITargetLowering::wrapAddr64Rsrc(SelectionDAG &DAG,
19844 const SDLoc &DL,
19845 SDValue Ptr) const {
19846 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19847
19848 // Build the half of the subregister with the constants before building the
19849 // full 128-bit register. If we are building multiple resource descriptors,
19850 // this will allow CSEing of the 2-component register.
19851 const SDValue Ops0[] = {
19852 DAG.getTargetConstant(Val: AMDGPU::SGPR_64RegClassID, DL, VT: MVT::i32),
19853 buildSMovImm32(DAG, DL, Val: 0),
19854 DAG.getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32),
19855 buildSMovImm32(DAG, DL, Val: TII->getDefaultRsrcDataFormat() >> 32),
19856 DAG.getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32)};
19857
19858 SDValue SubRegHi = SDValue(
19859 DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v2i32, Ops: Ops0), 0);
19860
19861 // Combine the constants and the pointer.
19862 const SDValue Ops1[] = {
19863 DAG.getTargetConstant(Val: AMDGPU::SGPR_128RegClassID, DL, VT: MVT::i32), Ptr,
19864 DAG.getTargetConstant(Val: AMDGPU::sub0_sub1, DL, VT: MVT::i32), SubRegHi,
19865 DAG.getTargetConstant(Val: AMDGPU::sub2_sub3, DL, VT: MVT::i32)};
19866
19867 return DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v4i32, Ops: Ops1);
19868}
19869
19870/// Return a resource descriptor with the 'Add TID' bit enabled
19871/// The TID (Thread ID) is multiplied by the stride value (bits [61:48]
19872/// of the resource descriptor) to create an offset, which is added to
19873/// the resource pointer.
19874MachineSDNode *SITargetLowering::buildRSRC(SelectionDAG &DAG, const SDLoc &DL,
19875 SDValue Ptr, uint32_t RsrcDword1,
19876 uint64_t RsrcDword2And3) const {
19877 SDValue PtrLo = DAG.getTargetExtractSubreg(SRIdx: AMDGPU::sub0, DL, VT: MVT::i32, Operand: Ptr);
19878 SDValue PtrHi = DAG.getTargetExtractSubreg(SRIdx: AMDGPU::sub1, DL, VT: MVT::i32, Operand: Ptr);
19879 if (RsrcDword1) {
19880 PtrHi = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: PtrHi,
19881 N2: DAG.getConstant(Val: RsrcDword1, DL, VT: MVT::i32));
19882 }
19883
19884 SDValue DataLo =
19885 buildSMovImm32(DAG, DL, Val: RsrcDword2And3 & UINT64_C(0xFFFFFFFF));
19886 SDValue DataHi = buildSMovImm32(DAG, DL, Val: RsrcDword2And3 >> 32);
19887
19888 const SDValue Ops[] = {
19889 DAG.getTargetConstant(Val: AMDGPU::SGPR_128RegClassID, DL, VT: MVT::i32),
19890 PtrLo,
19891 DAG.getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32),
19892 PtrHi,
19893 DAG.getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32),
19894 DataLo,
19895 DAG.getTargetConstant(Val: AMDGPU::sub2, DL, VT: MVT::i32),
19896 DataHi,
19897 DAG.getTargetConstant(Val: AMDGPU::sub3, DL, VT: MVT::i32)};
19898
19899 return DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v4i32, Ops);
19900}
19901
19902//===----------------------------------------------------------------------===//
19903// SI Inline Assembly Support
19904//===----------------------------------------------------------------------===//
19905
19906std::pair<unsigned, const TargetRegisterClass *>
19907SITargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI_,
19908 StringRef Constraint,
19909 MVT VT) const {
19910 const SIRegisterInfo *TRI = static_cast<const SIRegisterInfo *>(TRI_);
19911
19912 const TargetRegisterClass *RC = nullptr;
19913 if (Constraint.size() == 1) {
19914 // Check if we cannot determine the bit size of the given value type. This
19915 // can happen, for example, in this situation where we have an empty struct
19916 // (size 0): `call void asm "", "v"({} poison)`-
19917 if (VT == MVT::Other)
19918 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
19919 const unsigned BitWidth = VT.getSizeInBits();
19920 switch (Constraint[0]) {
19921 default:
19922 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
19923 case 's':
19924 case 'r':
19925 switch (BitWidth) {
19926 case 16:
19927 RC = &AMDGPU::SReg_32RegClass;
19928 break;
19929 case 64:
19930 RC = &AMDGPU::SGPR_64RegClass;
19931 break;
19932 default:
19933 RC = SIRegisterInfo::getSGPRClassForBitWidth(BitWidth);
19934 if (!RC)
19935 return std::pair(0U, nullptr);
19936 break;
19937 }
19938 break;
19939 case 'v':
19940 switch (BitWidth) {
19941 case 1:
19942 return std::pair(0U, nullptr);
19943 case 16:
19944 RC = Subtarget->useRealTrue16Insts() ? &AMDGPU::VGPR_16RegClass
19945 : &AMDGPU::VGPR_32_Lo256RegClass;
19946 break;
19947 default:
19948 RC = Subtarget->has1024AddressableVGPRs()
19949 ? TRI->getAlignedLo256VGPRClassForBitWidth(BitWidth)
19950 : TRI->getVGPRClassForBitWidth(BitWidth);
19951 if (!RC)
19952 return std::pair(0U, nullptr);
19953 break;
19954 }
19955 break;
19956 case 'a':
19957 if (!Subtarget->hasMAIInsts())
19958 break;
19959 switch (BitWidth) {
19960 case 1:
19961 return std::pair(0U, nullptr);
19962 case 16:
19963 RC = &AMDGPU::AGPR_32RegClass;
19964 break;
19965 default:
19966 RC = TRI->getAGPRClassForBitWidth(BitWidth);
19967 if (!RC)
19968 return std::pair(0U, nullptr);
19969 break;
19970 }
19971 break;
19972 }
19973 } else if (Constraint == "VA" && Subtarget->hasGFX90AInsts()) {
19974 const unsigned BitWidth = VT.getSizeInBits();
19975 switch (BitWidth) {
19976 case 16:
19977 RC = &AMDGPU::AV_32RegClass;
19978 break;
19979 default:
19980 RC = TRI->getVectorSuperClassForBitWidth(BitWidth);
19981 if (!RC)
19982 return std::pair(0U, nullptr);
19983 break;
19984 }
19985 }
19986
19987 // We actually support i128, i16 and f16 as inline parameters
19988 // even if they are not reported as legal
19989 if (RC && (isTypeLegal(VT) || VT.SimpleTy == MVT::i128 ||
19990 VT.SimpleTy == MVT::i16 || VT.SimpleTy == MVT::f16))
19991 return std::pair(0U, RC);
19992
19993 auto [Kind, Idx, NumRegs] = AMDGPU::parseAsmConstraintPhysReg(Constraint);
19994 if (Kind != '\0') {
19995 if (Kind == 'v') {
19996 RC = &AMDGPU::VGPR_32_Lo256RegClass;
19997 } else if (Kind == 's') {
19998 RC = &AMDGPU::SGPR_32RegClass;
19999 } else if (Kind == 'a') {
20000 RC = &AMDGPU::AGPR_32RegClass;
20001 }
20002
20003 if (RC) {
20004 if (NumRegs > 1) {
20005 if (Idx >= RC->getNumRegs() || Idx + NumRegs - 1 >= RC->getNumRegs())
20006 return std::pair(0U, nullptr);
20007
20008 uint32_t Width = NumRegs * 32;
20009 // Prohibit constraints for register ranges with a width that does not
20010 // match the required type.
20011 if (VT.SimpleTy != MVT::Other && Width != VT.getSizeInBits())
20012 return std::pair(0U, nullptr);
20013
20014 MCRegister Reg = RC->getRegister(i: Idx);
20015 if (SIRegisterInfo::isVGPRClass(RC))
20016 RC = TRI->getVGPRClassForBitWidth(BitWidth: Width);
20017 else if (SIRegisterInfo::isSGPRClass(RC))
20018 RC = TRI->getSGPRClassForBitWidth(BitWidth: Width);
20019 else if (SIRegisterInfo::isAGPRClass(RC))
20020 RC = TRI->getAGPRClassForBitWidth(BitWidth: Width);
20021 if (RC) {
20022 Reg = TRI->getMatchingSuperReg(Reg, SubIdx: AMDGPU::sub0, RC);
20023 if (!Reg) {
20024 // The register class does not contain the requested register,
20025 // e.g., because it is an SGPR pair that would violate alignment
20026 // requirements.
20027 return std::pair(0U, nullptr);
20028 }
20029 return std::pair(Reg, RC);
20030 }
20031 }
20032
20033 // Reject types that do not fit a single 32-bit register: any scalar wider
20034 // than 32 bits, or a vector that is not exactly 32 bits.
20035 if (VT.SimpleTy != MVT::Other &&
20036 (VT.getSizeInBits() > 32 ||
20037 (VT.isVector() && VT.getSizeInBits() != 32)))
20038 return std::pair(0U, nullptr);
20039 if (RC && Idx < RC->getNumRegs())
20040 return std::pair(RC->getRegister(i: Idx), RC);
20041 return std::pair(0U, nullptr);
20042 }
20043 }
20044
20045 auto Ret = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
20046 if (Ret.first)
20047 Ret.second = TRI->getPhysRegBaseClass(Reg: Ret.first);
20048
20049 return Ret;
20050}
20051
20052static bool isImmConstraint(StringRef Constraint) {
20053 if (Constraint.size() == 1) {
20054 switch (Constraint[0]) {
20055 default:
20056 break;
20057 case 'I':
20058 case 'J':
20059 case 'A':
20060 case 'B':
20061 case 'C':
20062 return true;
20063 }
20064 } else if (Constraint == "DA" || Constraint == "DB") {
20065 return true;
20066 }
20067 return false;
20068}
20069
20070SITargetLowering::ConstraintType
20071SITargetLowering::getConstraintType(StringRef Constraint) const {
20072 if (Constraint.size() == 1) {
20073 switch (Constraint[0]) {
20074 default:
20075 break;
20076 case 's':
20077 case 'v':
20078 case 'a':
20079 return C_RegisterClass;
20080 }
20081 } else if (Constraint.size() == 2) {
20082 if (Constraint == "VA")
20083 return C_RegisterClass;
20084 }
20085 if (isImmConstraint(Constraint)) {
20086 return C_Other;
20087 }
20088 return TargetLowering::getConstraintType(Constraint);
20089}
20090
20091static uint64_t clearUnusedBits(uint64_t Val, unsigned Size) {
20092 if (!AMDGPU::isInlinableIntLiteral(Literal: Val)) {
20093 Val = Val & maskTrailingOnes<uint64_t>(N: Size);
20094 }
20095 return Val;
20096}
20097
20098void SITargetLowering::LowerAsmOperandForConstraint(SDValue Op,
20099 StringRef Constraint,
20100 std::vector<SDValue> &Ops,
20101 SelectionDAG &DAG) const {
20102 if (isImmConstraint(Constraint)) {
20103 uint64_t Val;
20104 if (getAsmOperandConstVal(Op, Val) &&
20105 checkAsmConstraintVal(Op, Constraint, Val)) {
20106 Val = clearUnusedBits(Val, Size: Op.getScalarValueSizeInBits());
20107 Ops.push_back(x: DAG.getTargetConstant(Val, DL: SDLoc(Op), VT: MVT::i64));
20108 }
20109 } else {
20110 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
20111 }
20112}
20113
20114bool SITargetLowering::getAsmOperandConstVal(SDValue Op, uint64_t &Val) const {
20115 unsigned Size = Op.getScalarValueSizeInBits();
20116 if (Size > 64)
20117 return false;
20118
20119 if (Size == 16 && !Subtarget->has16BitInsts())
20120 return false;
20121
20122 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
20123 Val = C->getSExtValue();
20124 return true;
20125 }
20126 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
20127 Val = C->getValueAPF().bitcastToAPInt().getSExtValue();
20128 return true;
20129 }
20130 if (BuildVectorSDNode *V = dyn_cast<BuildVectorSDNode>(Val&: Op)) {
20131 if (Size != 16 || Op.getNumOperands() != 2)
20132 return false;
20133 if (Op.getOperand(i: 0).isUndef() || Op.getOperand(i: 1).isUndef())
20134 return false;
20135 if (ConstantSDNode *C = V->getConstantSplatNode()) {
20136 Val = C->getSExtValue();
20137 return true;
20138 }
20139 if (ConstantFPSDNode *C = V->getConstantFPSplatNode()) {
20140 Val = C->getValueAPF().bitcastToAPInt().getSExtValue();
20141 return true;
20142 }
20143 }
20144
20145 return false;
20146}
20147
20148bool SITargetLowering::checkAsmConstraintVal(SDValue Op, StringRef Constraint,
20149 uint64_t Val) const {
20150 if (Constraint.size() == 1) {
20151 switch (Constraint[0]) {
20152 case 'I':
20153 return AMDGPU::isInlinableIntLiteral(Literal: Val);
20154 case 'J':
20155 return isInt<16>(x: Val);
20156 case 'A':
20157 return checkAsmConstraintValA(Op, Val);
20158 case 'B':
20159 return isInt<32>(x: Val);
20160 case 'C':
20161 return isUInt<32>(x: clearUnusedBits(Val, Size: Op.getScalarValueSizeInBits())) ||
20162 AMDGPU::isInlinableIntLiteral(Literal: Val);
20163 default:
20164 break;
20165 }
20166 } else if (Constraint.size() == 2) {
20167 if (Constraint == "DA") {
20168 int64_t HiBits = static_cast<int32_t>(Val >> 32);
20169 int64_t LoBits = static_cast<int32_t>(Val);
20170 return checkAsmConstraintValA(Op, Val: HiBits, MaxSize: 32) &&
20171 checkAsmConstraintValA(Op, Val: LoBits, MaxSize: 32);
20172 }
20173 if (Constraint == "DB") {
20174 return true;
20175 }
20176 }
20177 llvm_unreachable("Invalid asm constraint");
20178}
20179
20180bool SITargetLowering::checkAsmConstraintValA(SDValue Op, uint64_t Val,
20181 unsigned MaxSize) const {
20182 unsigned Size = std::min<unsigned>(a: Op.getScalarValueSizeInBits(), b: MaxSize);
20183 bool HasInv2Pi = Subtarget->hasInv2PiInlineImm();
20184 if (Size == 16) {
20185 MVT VT = Op.getSimpleValueType();
20186 switch (VT.SimpleTy) {
20187 default:
20188 return false;
20189 case MVT::i16:
20190 return AMDGPU::isInlinableLiteralI16(Literal: Val, HasInv2Pi);
20191 case MVT::f16:
20192 return AMDGPU::isInlinableLiteralFP16(Literal: Val, HasInv2Pi);
20193 case MVT::bf16:
20194 return AMDGPU::isInlinableLiteralBF16(Literal: Val, HasInv2Pi);
20195 case MVT::v2i16:
20196 return AMDGPU::getInlineEncodingV2I16(Literal: Val).has_value();
20197 case MVT::v2f16:
20198 return AMDGPU::getInlineEncodingV2F16(Literal: Val).has_value();
20199 case MVT::v2bf16:
20200 return AMDGPU::getInlineEncodingV2BF16(Literal: Val).has_value();
20201 }
20202 }
20203 if ((Size == 32 && AMDGPU::isInlinableLiteral32(Literal: Val, HasInv2Pi)) ||
20204 (Size == 64 && AMDGPU::isInlinableLiteral64(Literal: Val, HasInv2Pi)))
20205 return true;
20206 return false;
20207}
20208
20209static int getAlignedAGPRClassID(unsigned UnalignedClassID) {
20210 switch (UnalignedClassID) {
20211 case AMDGPU::VReg_64RegClassID:
20212 return AMDGPU::VReg_64_Align2RegClassID;
20213 case AMDGPU::VReg_96RegClassID:
20214 return AMDGPU::VReg_96_Align2RegClassID;
20215 case AMDGPU::VReg_128RegClassID:
20216 return AMDGPU::VReg_128_Align2RegClassID;
20217 case AMDGPU::VReg_160RegClassID:
20218 return AMDGPU::VReg_160_Align2RegClassID;
20219 case AMDGPU::VReg_192RegClassID:
20220 return AMDGPU::VReg_192_Align2RegClassID;
20221 case AMDGPU::VReg_224RegClassID:
20222 return AMDGPU::VReg_224_Align2RegClassID;
20223 case AMDGPU::VReg_256RegClassID:
20224 return AMDGPU::VReg_256_Align2RegClassID;
20225 case AMDGPU::VReg_288RegClassID:
20226 return AMDGPU::VReg_288_Align2RegClassID;
20227 case AMDGPU::VReg_320RegClassID:
20228 return AMDGPU::VReg_320_Align2RegClassID;
20229 case AMDGPU::VReg_352RegClassID:
20230 return AMDGPU::VReg_352_Align2RegClassID;
20231 case AMDGPU::VReg_384RegClassID:
20232 return AMDGPU::VReg_384_Align2RegClassID;
20233 case AMDGPU::VReg_512RegClassID:
20234 return AMDGPU::VReg_512_Align2RegClassID;
20235 case AMDGPU::VReg_1024RegClassID:
20236 return AMDGPU::VReg_1024_Align2RegClassID;
20237 case AMDGPU::AReg_64RegClassID:
20238 return AMDGPU::AReg_64_Align2RegClassID;
20239 case AMDGPU::AReg_96RegClassID:
20240 return AMDGPU::AReg_96_Align2RegClassID;
20241 case AMDGPU::AReg_128RegClassID:
20242 return AMDGPU::AReg_128_Align2RegClassID;
20243 case AMDGPU::AReg_160RegClassID:
20244 return AMDGPU::AReg_160_Align2RegClassID;
20245 case AMDGPU::AReg_192RegClassID:
20246 return AMDGPU::AReg_192_Align2RegClassID;
20247 case AMDGPU::AReg_256RegClassID:
20248 return AMDGPU::AReg_256_Align2RegClassID;
20249 case AMDGPU::AReg_512RegClassID:
20250 return AMDGPU::AReg_512_Align2RegClassID;
20251 case AMDGPU::AReg_1024RegClassID:
20252 return AMDGPU::AReg_1024_Align2RegClassID;
20253 default:
20254 return -1;
20255 }
20256}
20257
20258// Figure out which registers should be reserved for stack access. Only after
20259// the function is legalized do we know all of the non-spill stack objects or if
20260// calls are present.
20261void SITargetLowering::finalizeLowering(MachineFunction &MF) const {
20262 MachineRegisterInfo &MRI = MF.getRegInfo();
20263 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
20264 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
20265 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
20266 const SIInstrInfo *TII = ST.getInstrInfo();
20267
20268 if (Info->isEntryFunction()) {
20269 // Callable functions have fixed registers used for stack access.
20270 reservePrivateMemoryRegs(TM: getTargetMachine(), MF, TRI: *TRI, Info&: *Info);
20271 }
20272
20273 // TODO: Move this logic to getReservedRegs()
20274 // Reserve the SGPR(s) to save/restore EXEC for WWM spill/copy handling.
20275 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF);
20276 Register SReg = ST.isWave32()
20277 ? AMDGPU::SGPR_32RegClass.getRegister(i: MaxNumSGPRs - 1)
20278 : TRI->getAlignedHighSGPRForRC(MF, /*Align=*/2,
20279 RC: &AMDGPU::SGPR_64RegClass);
20280 Info->setSGPRForEXECCopy(SReg);
20281
20282 assert(!TRI->isSubRegister(Info->getScratchRSrcReg(),
20283 Info->getStackPtrOffsetReg()));
20284 if (Info->getStackPtrOffsetReg() != AMDGPU::SP_REG)
20285 MRI.replaceRegWith(FromReg: AMDGPU::SP_REG, ToReg: Info->getStackPtrOffsetReg());
20286
20287 // We need to worry about replacing the default register with itself in case
20288 // of MIR testcases missing the MFI.
20289 if (Info->getScratchRSrcReg() != AMDGPU::PRIVATE_RSRC_REG)
20290 MRI.replaceRegWith(FromReg: AMDGPU::PRIVATE_RSRC_REG, ToReg: Info->getScratchRSrcReg());
20291
20292 if (Info->getFrameOffsetReg() != AMDGPU::FP_REG)
20293 MRI.replaceRegWith(FromReg: AMDGPU::FP_REG, ToReg: Info->getFrameOffsetReg());
20294
20295 Info->limitOccupancy(MF);
20296
20297 if (ST.isWave32() && !MF.empty()) {
20298 for (auto &MBB : MF) {
20299 for (auto &MI : MBB) {
20300 TII->fixImplicitOperands(MI);
20301 }
20302 }
20303 }
20304
20305 // FIXME: This is a hack to fixup AGPR classes to use the properly aligned
20306 // classes if required. Ideally the register class constraints would differ
20307 // per-subtarget, but there's no easy way to achieve that right now. This is
20308 // not a problem for VGPRs because the correctly aligned VGPR class is implied
20309 // from using them as the register class for legal types.
20310 if (ST.needsAlignedVGPRs()) {
20311 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
20312 const Register Reg = Register::index2VirtReg(Index: I);
20313 const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg);
20314 if (!RC)
20315 continue;
20316 int NewClassID = getAlignedAGPRClassID(UnalignedClassID: RC->getID());
20317 if (NewClassID != -1)
20318 MRI.setRegClass(Reg, RC: TRI->getRegClass(i: NewClassID));
20319 }
20320 }
20321
20322 TargetLoweringBase::finalizeLowering(MF);
20323}
20324
20325void SITargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
20326 KnownBits &Known,
20327 const APInt &DemandedElts,
20328 const SelectionDAG &DAG,
20329 unsigned Depth) const {
20330 Known.resetAll();
20331 unsigned Opc = Op.getOpcode();
20332 switch (Opc) {
20333 case ISD::INTRINSIC_WO_CHAIN: {
20334 unsigned IID = Op.getConstantOperandVal(i: 0);
20335 switch (IID) {
20336 case Intrinsic::amdgcn_mbcnt_lo:
20337 case Intrinsic::amdgcn_mbcnt_hi: {
20338 const GCNSubtarget &ST =
20339 DAG.getMachineFunction().getSubtarget<GCNSubtarget>();
20340 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
20341 // most 31 + src1.
20342 Known.Zero.setBitsFrom(
20343 IID == Intrinsic::amdgcn_mbcnt_lo ? ST.getWavefrontSizeLog2() : 5);
20344 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
20345 Known = KnownBits::add(LHS: Known, RHS: Known2);
20346 return;
20347 }
20348 }
20349 break;
20350 }
20351 }
20352 return AMDGPUTargetLowering::computeKnownBitsForTargetNode(
20353 Op, Known, DemandedElts, DAG, Depth);
20354}
20355
20356void SITargetLowering::computeKnownBitsForStackObjectPointer(
20357 KnownBits &Known, const MachineFunction &MF, Align Alignment) const {
20358 TargetLowering::computeKnownBitsForStackObjectPointer(Known, MF, Alignment);
20359
20360 // Set the high bits to zero based on the maximum allowed scratch size per
20361 // wave. We can't use vaddr in MUBUF instructions if we don't know the address
20362 // calculation won't overflow, so assume the sign bit is never set.
20363 Known.Zero.setHighBits(getSubtarget()->getKnownHighZeroBitsForFrameIndex());
20364}
20365
20366static void knownBitsForWorkitemID(const GCNSubtarget &ST,
20367 GISelValueTracking &VT, KnownBits &Known,
20368 unsigned Dim) {
20369 unsigned MaxValue =
20370 ST.getMaxWorkitemID(Kernel: VT.getMachineFunction().getFunction(), Dimension: Dim);
20371 Known.Zero.setHighBits(llvm::countl_zero(Val: MaxValue));
20372}
20373
20374static void knownBitsForSBFE(const MachineInstr &MI, GISelValueTracking &VT,
20375 KnownBits &Known, const APInt &DemandedElts,
20376 unsigned BFEWidth, bool SExt, unsigned Depth) {
20377 const MachineRegisterInfo &MRI = VT.getMachineFunction().getRegInfo();
20378 const MachineOperand &Src1 = MI.getOperand(i: 2);
20379
20380 unsigned Src1Cst = 0;
20381 if (Src1.isImm()) {
20382 Src1Cst = Src1.getImm();
20383 } else if (Src1.isReg()) {
20384 auto Cst = getIConstantVRegValWithLookThrough(VReg: Src1.getReg(), MRI);
20385 if (!Cst)
20386 return;
20387 Src1Cst = Cst->Value.getZExtValue();
20388 } else {
20389 return;
20390 }
20391
20392 // Offset is at bits [4:0] for 32 bit, [5:0] for 64 bit.
20393 // Width is always [22:16].
20394 const unsigned Offset =
20395 Src1Cst & maskTrailingOnes<unsigned>(N: (BFEWidth == 32) ? 5 : 6);
20396 const unsigned Width = (Src1Cst >> 16) & maskTrailingOnes<unsigned>(N: 6);
20397
20398 if (Width >= BFEWidth) // Ill-formed.
20399 return;
20400
20401 VT.computeKnownBitsImpl(R: MI.getOperand(i: 1).getReg(), Known, DemandedElts,
20402 Depth: Depth + 1);
20403
20404 Known = Known.extractBits(NumBits: Width, BitPosition: Offset);
20405
20406 if (SExt)
20407 Known = Known.sext(BitWidth: BFEWidth);
20408 else
20409 Known = Known.zext(BitWidth: BFEWidth);
20410}
20411
20412void SITargetLowering::computeKnownBitsForTargetInstr(
20413 GISelValueTracking &VT, Register R, KnownBits &Known,
20414 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
20415 unsigned Depth) const {
20416 Known.resetAll();
20417 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
20418 switch (MI->getOpcode()) {
20419 case AMDGPU::S_BFE_I32:
20420 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 32,
20421 /*SExt=*/true, Depth);
20422 case AMDGPU::S_BFE_U32:
20423 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 32,
20424 /*SExt=*/false, Depth);
20425 case AMDGPU::S_BFE_I64:
20426 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 64,
20427 /*SExt=*/true, Depth);
20428 case AMDGPU::S_BFE_U64:
20429 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 64,
20430 /*SExt=*/false, Depth);
20431 case AMDGPU::G_INTRINSIC:
20432 case AMDGPU::G_INTRINSIC_CONVERGENT: {
20433 Intrinsic::ID IID = cast<GIntrinsic>(Val: MI)->getIntrinsicID();
20434 switch (IID) {
20435 case Intrinsic::amdgcn_workitem_id_x:
20436 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 0);
20437 break;
20438 case Intrinsic::amdgcn_workitem_id_y:
20439 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 1);
20440 break;
20441 case Intrinsic::amdgcn_workitem_id_z:
20442 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 2);
20443 break;
20444 case Intrinsic::amdgcn_mbcnt_lo:
20445 case Intrinsic::amdgcn_mbcnt_hi: {
20446 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
20447 // most 31 + src1.
20448 Known.Zero.setBitsFrom(IID == Intrinsic::amdgcn_mbcnt_lo
20449 ? getSubtarget()->getWavefrontSizeLog2()
20450 : 5);
20451 KnownBits Known2;
20452 VT.computeKnownBitsImpl(R: MI->getOperand(i: 3).getReg(), Known&: Known2, DemandedElts,
20453 Depth: Depth + 1);
20454 Known = KnownBits::add(LHS: Known, RHS: Known2);
20455 break;
20456 }
20457 case Intrinsic::amdgcn_groupstaticsize: {
20458 // We can report everything over the maximum size as 0. We can't report
20459 // based on the actual size because we don't know if it's accurate or not
20460 // at any given point.
20461 Known.Zero.setHighBits(
20462 llvm::countl_zero(Val: getSubtarget()->getAddressableLocalMemorySize()));
20463 break;
20464 }
20465 }
20466 break;
20467 }
20468 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
20469 Known.Zero.setHighBits(24);
20470 break;
20471 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
20472 Known.Zero.setHighBits(16);
20473 break;
20474 case AMDGPU::G_AMDGPU_COPY_SCC_VCC:
20475 // G_AMDGPU_COPY_SCC_VCC converts a uniform boolean in VCC to SGPR s32,
20476 // producing exactly 0 or 1.
20477 Known.Zero.setHighBits(Known.getBitWidth() - 1);
20478 break;
20479 case AMDGPU::G_AMDGPU_SMED3:
20480 case AMDGPU::G_AMDGPU_UMED3: {
20481 auto [Dst, Src0, Src1, Src2] = MI->getFirst4Regs();
20482
20483 KnownBits Known2;
20484 VT.computeKnownBitsImpl(R: Src2, Known&: Known2, DemandedElts, Depth: Depth + 1);
20485 if (Known2.isUnknown())
20486 break;
20487
20488 KnownBits Known1;
20489 VT.computeKnownBitsImpl(R: Src1, Known&: Known1, DemandedElts, Depth: Depth + 1);
20490 if (Known1.isUnknown())
20491 break;
20492
20493 KnownBits Known0;
20494 VT.computeKnownBitsImpl(R: Src0, Known&: Known0, DemandedElts, Depth: Depth + 1);
20495 if (Known0.isUnknown())
20496 break;
20497
20498 // TODO: Handle LeadZero/LeadOne from UMIN/UMAX handling.
20499 Known.Zero = Known0.Zero & Known1.Zero & Known2.Zero;
20500 Known.One = Known0.One & Known1.One & Known2.One;
20501 break;
20502 }
20503 }
20504}
20505
20506Align SITargetLowering::computeKnownAlignForTargetInstr(
20507 GISelValueTracking &VT, Register R, const MachineRegisterInfo &MRI,
20508 unsigned Depth) const {
20509 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
20510 if (auto *GI = dyn_cast<GIntrinsic>(Val: MI)) {
20511 // FIXME: Can this move to generic code? What about the case where the call
20512 // site specifies a lower alignment?
20513 Intrinsic::ID IID = GI->getIntrinsicID();
20514 LLVMContext &Ctx = VT.getMachineFunction().getFunction().getContext();
20515 AttributeList Attrs =
20516 Intrinsic::getAttributes(C&: Ctx, id: IID, FT: Intrinsic::getType(Context&: Ctx, id: IID));
20517 if (MaybeAlign RetAlign = Attrs.getRetAlignment())
20518 return *RetAlign;
20519 }
20520 return Align(1);
20521}
20522
20523Align SITargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
20524 const Align PrefAlign = TargetLowering::getPrefLoopAlignment(ML);
20525 const Align CacheLineAlign = Align(64);
20526
20527 // GFX950: Prevent an 8-byte instruction at loop header from being split by
20528 // the 32-byte instruction fetch window boundary. This avoids a significant
20529 // fetch delay after backward branch. We use 32-byte alignment with max
20530 // padding of 4 bytes (one s_nop), see getMaxPermittedBytesForAlignment().
20531 if (ML && !DisableLoopAlignment &&
20532 getSubtarget()->hasLoopHeadInstSplitSensitivity()) {
20533 const MachineBasicBlock *Header = ML->getHeader();
20534 // Respect user-specified or previously set alignment.
20535 if (Header->getAlignment() != PrefAlign)
20536 return Header->getAlignment();
20537 if (needsFetchWindowAlignment(MBB: *Header))
20538 return Align(32);
20539 }
20540
20541 // Pre-GFX10 target did not benefit from loop alignment
20542 if (!ML || DisableLoopAlignment || !getSubtarget()->hasInstPrefetch() ||
20543 getSubtarget()->hasInstFwdPrefetchBug())
20544 return PrefAlign;
20545
20546 // On GFX10 I$ is 4 x 64 bytes cache lines.
20547 // By default prefetcher keeps one cache line behind and reads two ahead.
20548 // We can modify it with S_INST_PREFETCH for larger loops to have two lines
20549 // behind and one ahead.
20550 // Therefor we can benefit from aligning loop headers if loop fits 192 bytes.
20551 // If loop fits 64 bytes it always spans no more than two cache lines and
20552 // does not need an alignment.
20553 // Else if loop is less or equal 128 bytes we do not need to modify prefetch,
20554 // Else if loop is less or equal 192 bytes we need two lines behind.
20555
20556 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
20557 const MachineBasicBlock *Header = ML->getHeader();
20558 if (Header->getAlignment() != PrefAlign)
20559 return Header->getAlignment(); // Already processed.
20560
20561 unsigned LoopSize = 0;
20562 for (const MachineBasicBlock *MBB : ML->blocks()) {
20563 // If inner loop block is aligned assume in average half of the alignment
20564 // size to be added as nops.
20565 if (MBB != Header)
20566 LoopSize += MBB->getAlignment().value() / 2;
20567
20568 for (const MachineInstr &MI : *MBB) {
20569 LoopSize += TII->getInstSizeInBytes(MI);
20570 if (LoopSize > 192)
20571 return PrefAlign;
20572 }
20573 }
20574
20575 if (LoopSize <= 64)
20576 return PrefAlign;
20577
20578 if (LoopSize <= 128)
20579 return CacheLineAlign;
20580
20581 // If any of parent loops is surrounded by prefetch instructions do not
20582 // insert new for inner loop, which would reset parent's settings.
20583 for (MachineLoop *P = ML->getParentLoop(); P; P = P->getParentLoop()) {
20584 if (MachineBasicBlock *Exit = P->getExitBlock()) {
20585 auto I = Exit->getFirstNonDebugInstr();
20586 if (I != Exit->end() && I->getOpcode() == AMDGPU::S_INST_PREFETCH)
20587 return CacheLineAlign;
20588 }
20589 }
20590
20591 MachineBasicBlock *Pre = ML->getLoopPreheader();
20592 MachineBasicBlock *Exit = ML->getExitBlock();
20593
20594 if (Pre && Exit) {
20595 auto PreTerm = Pre->getFirstTerminator();
20596 if (PreTerm == Pre->begin() ||
20597 std::prev(x: PreTerm)->getOpcode() != AMDGPU::S_INST_PREFETCH)
20598 BuildMI(BB&: *Pre, I: PreTerm, MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_INST_PREFETCH))
20599 .addImm(Val: 1); // prefetch 2 lines behind PC
20600
20601 auto ExitHead = Exit->getFirstNonDebugInstr();
20602 if (ExitHead == Exit->end() ||
20603 ExitHead->getOpcode() != AMDGPU::S_INST_PREFETCH)
20604 BuildMI(BB&: *Exit, I: ExitHead, MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_INST_PREFETCH))
20605 .addImm(Val: 2); // prefetch 1 line behind PC
20606 }
20607
20608 return CacheLineAlign;
20609}
20610
20611unsigned SITargetLowering::getMaxPermittedBytesForAlignment(
20612 MachineBasicBlock *MBB) const {
20613 // GFX950: Limit padding to 4 bytes (one s_nop) for blocks where an 8-byte
20614 // instruction could be split by the 32-byte fetch window boundary.
20615 // See getPrefLoopAlignment() for context.
20616 if (needsFetchWindowAlignment(MBB: *MBB))
20617 return 4;
20618 return TargetLowering::getMaxPermittedBytesForAlignment(MBB);
20619}
20620
20621bool SITargetLowering::needsFetchWindowAlignment(
20622 const MachineBasicBlock &MBB) const {
20623 if (!getSubtarget()->hasLoopHeadInstSplitSensitivity())
20624 return false;
20625 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
20626 for (const MachineInstr &MI : MBB) {
20627 if (MI.isMetaInstruction())
20628 continue;
20629 // Instructions larger than 4 bytes can be split by a 32-byte boundary.
20630 return TII->getInstSizeInBytes(MI) > 4;
20631 }
20632 return false;
20633}
20634
20635[[maybe_unused]]
20636static bool isCopyFromRegOfInlineAsm(const SDNode *N) {
20637 assert(N->getOpcode() == ISD::CopyFromReg);
20638 do {
20639 // Follow the chain until we find an INLINEASM node.
20640 N = N->getOperand(Num: 0).getNode();
20641 if (N->getOpcode() == ISD::INLINEASM || N->getOpcode() == ISD::INLINEASM_BR)
20642 return true;
20643 } while (N->getOpcode() == ISD::CopyFromReg);
20644 return false;
20645}
20646
20647bool SITargetLowering::isSDNodeSourceOfDivergence(const SDNode *N,
20648 FunctionLoweringInfo *FLI,
20649 UniformityInfo *UA) const {
20650 switch (N->getOpcode()) {
20651 case ISD::CopyFromReg: {
20652 const RegisterSDNode *R = cast<RegisterSDNode>(Val: N->getOperand(Num: 1));
20653 const MachineRegisterInfo &MRI = FLI->MF->getRegInfo();
20654 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
20655 Register Reg = R->getReg();
20656
20657 // FIXME: Why does this need to consider isLiveIn?
20658 if (Reg.isPhysical() || MRI.isLiveIn(Reg))
20659 return !TRI->isSGPRReg(MRI, Reg);
20660
20661 if (const Value *V = FLI->getValueFromVirtualReg(Vreg: R->getReg()))
20662 return UA->isDivergentAtDef(V);
20663
20664 assert(Reg == FLI->DemoteRegister || isCopyFromRegOfInlineAsm(N));
20665 return !TRI->isSGPRReg(MRI, Reg);
20666 }
20667 case ISD::LOAD: {
20668 const LoadSDNode *L = cast<LoadSDNode>(Val: N);
20669 unsigned AS = L->getAddressSpace();
20670 // A flat load may access private memory.
20671 return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS;
20672 }
20673 case ISD::CALLSEQ_END:
20674 return true;
20675 case ISD::INTRINSIC_WO_CHAIN:
20676 return AMDGPU::isIntrinsicSourceOfDivergence(IntrID: N->getConstantOperandVal(Num: 0));
20677 case ISD::INTRINSIC_W_CHAIN:
20678 return AMDGPU::isIntrinsicSourceOfDivergence(IntrID: N->getConstantOperandVal(Num: 1));
20679 case AMDGPUISD::ATOMIC_CMP_SWAP:
20680 case AMDGPUISD::BUFFER_ATOMIC_SWAP:
20681 case AMDGPUISD::BUFFER_ATOMIC_ADD:
20682 case AMDGPUISD::BUFFER_ATOMIC_SUB:
20683 case AMDGPUISD::BUFFER_ATOMIC_SMIN:
20684 case AMDGPUISD::BUFFER_ATOMIC_UMIN:
20685 case AMDGPUISD::BUFFER_ATOMIC_SMAX:
20686 case AMDGPUISD::BUFFER_ATOMIC_UMAX:
20687 case AMDGPUISD::BUFFER_ATOMIC_AND:
20688 case AMDGPUISD::BUFFER_ATOMIC_OR:
20689 case AMDGPUISD::BUFFER_ATOMIC_XOR:
20690 case AMDGPUISD::BUFFER_ATOMIC_INC:
20691 case AMDGPUISD::BUFFER_ATOMIC_DEC:
20692 case AMDGPUISD::BUFFER_ATOMIC_CMPSWAP:
20693 case AMDGPUISD::BUFFER_ATOMIC_FADD:
20694 case AMDGPUISD::BUFFER_ATOMIC_FMIN:
20695 case AMDGPUISD::BUFFER_ATOMIC_FMAX:
20696 // Target-specific read-modify-write atomics are sources of divergence.
20697 return true;
20698 default:
20699 if (auto *A = dyn_cast<AtomicSDNode>(Val: N)) {
20700 // Generic read-modify-write atomics are sources of divergence.
20701 return A->readMem() && A->writeMem();
20702 }
20703 return false;
20704 }
20705}
20706
20707bool SITargetLowering::denormalsEnabledForType(const SelectionDAG &DAG,
20708 EVT VT) const {
20709 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
20710 case MVT::f32:
20711 return !denormalModeIsFlushAllF32(MF: DAG.getMachineFunction());
20712 case MVT::f64:
20713 case MVT::f16:
20714 return !denormalModeIsFlushAllF64F16(MF: DAG.getMachineFunction());
20715 default:
20716 return false;
20717 }
20718}
20719
20720bool SITargetLowering::denormalsEnabledForType(
20721 LLT Ty, const MachineFunction &MF) const {
20722 switch (Ty.getScalarSizeInBits()) {
20723 case 32:
20724 return !denormalModeIsFlushAllF32(MF);
20725 case 64:
20726 case 16:
20727 return !denormalModeIsFlushAllF64F16(MF);
20728 default:
20729 return false;
20730 }
20731}
20732
20733bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
20734 const APInt &DemandedElts,
20735 const SelectionDAG &DAG,
20736 bool SNaN,
20737 unsigned Depth) const {
20738 if (Op.getOpcode() == AMDGPUISD::CLAMP) {
20739 const MachineFunction &MF = DAG.getMachineFunction();
20740 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
20741
20742 if (Info->getMode().DX10Clamp)
20743 return true; // Clamped to 0.
20744 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
20745 }
20746
20747 return AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(Op, DemandedElts,
20748 DAG, SNaN, Depth);
20749}
20750
20751// On older subtargets, global FP atomic instructions have a hardcoded FP mode
20752// and do not support FP32 denormals, and only support v2f16/f64 denormals.
20753static bool atomicIgnoresDenormalModeOrFPModeIsFTZ(const AtomicRMWInst *RMW) {
20754 if (RMW->hasMetadata(KindID: LLVMContext::MD_atomic_ignore_denormal_mode))
20755 return true;
20756
20757 const fltSemantics &Flt = RMW->getType()->getScalarType()->getFltSemantics();
20758 auto DenormMode = RMW->getFunction()->getDenormalMode(FPType: Flt);
20759 if (DenormMode == DenormalMode::getPreserveSign())
20760 return true;
20761
20762 // TODO: Remove this.
20763 return RMW->getFunction()
20764 ->getFnAttribute(Kind: "amdgpu-unsafe-fp-atomics")
20765 .getValueAsBool();
20766}
20767
20768static OptimizationRemark emitAtomicRMWLegalRemark(const AtomicRMWInst *RMW) {
20769 LLVMContext &Ctx = RMW->getContext();
20770 StringRef MemScope =
20771 Ctx.getSyncScopeName(Id: RMW->getSyncScopeID()).value_or(u: "system");
20772
20773 return OptimizationRemark(DEBUG_TYPE, "Passed", RMW)
20774 << "Hardware instruction generated for atomic "
20775 << RMW->getOperationName(Op: RMW->getOperation())
20776 << " operation at memory scope " << MemScope;
20777}
20778
20779static bool isV2F16OrV2BF16(Type *Ty) {
20780 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
20781 Type *EltTy = VT->getElementType();
20782 return VT->getNumElements() == 2 &&
20783 (EltTy->isHalfTy() || EltTy->isBFloatTy());
20784 }
20785
20786 return false;
20787}
20788
20789static bool isV2F16(Type *Ty) {
20790 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
20791 return VT && VT->getNumElements() == 2 && VT->getElementType()->isHalfTy();
20792}
20793
20794static bool isV2BF16(Type *Ty) {
20795 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
20796 return VT && VT->getNumElements() == 2 && VT->getElementType()->isBFloatTy();
20797}
20798
20799/// \return true if atomicrmw integer ops work for the type.
20800static bool isAtomicRMWLegalIntTy(Type *Ty) {
20801 if (auto *IT = dyn_cast<IntegerType>(Val: Ty)) {
20802 unsigned BW = IT->getBitWidth();
20803 return BW == 32 || BW == 64;
20804 }
20805
20806 return false;
20807}
20808
20809/// \return true if this atomicrmw xchg type can be selected.
20810static bool isAtomicRMWLegalXChgTy(const AtomicRMWInst *RMW) {
20811 Type *Ty = RMW->getType();
20812 if (isAtomicRMWLegalIntTy(Ty))
20813 return true;
20814
20815 if (PointerType *PT = dyn_cast<PointerType>(Val: Ty)) {
20816 const DataLayout &DL = RMW->getFunction()->getParent()->getDataLayout();
20817 unsigned BW = DL.getPointerSizeInBits(AS: PT->getAddressSpace());
20818 return BW == 32 || BW == 64;
20819 }
20820
20821 if (Ty->isFloatTy() || Ty->isDoubleTy())
20822 return true;
20823
20824 if (FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
20825 return VT->getNumElements() == 2 &&
20826 VT->getElementType()->getPrimitiveSizeInBits() == 16;
20827 }
20828
20829 return false;
20830}
20831
20832/// \returns true if it's valid to emit a native instruction for \p RMW, based
20833/// on the properties of the target memory.
20834static bool globalMemoryFPAtomicIsLegal(const GCNSubtarget &Subtarget,
20835 const AtomicRMWInst *RMW,
20836 bool HasSystemScope) {
20837 // The remote/fine-grained access logic is different from the integer
20838 // atomics. Without AgentScopeFineGrainedRemoteMemoryAtomics support,
20839 // fine-grained access does not work, even for a device local allocation.
20840 //
20841 // With AgentScopeFineGrainedRemoteMemoryAtomics, system scoped device local
20842 // allocations work.
20843 if (HasSystemScope) {
20844 if (Subtarget.hasAgentScopeFineGrainedRemoteMemoryAtomics() &&
20845 RMW->hasMetadata(Kind: "amdgpu.no.remote.memory"))
20846 return true;
20847 if (Subtarget.hasEmulatedSystemScopeAtomics())
20848 return true;
20849 } else if (Subtarget.hasAgentScopeFineGrainedRemoteMemoryAtomics())
20850 return true;
20851
20852 return RMW->hasMetadata(Kind: "amdgpu.no.fine.grained.memory");
20853}
20854
20855/// \return Action to perform on AtomicRMWInsts for integer operations.
20856static TargetLowering::AtomicExpansionKind
20857atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW) {
20858 return isAtomicRMWLegalIntTy(Ty: RMW->getType())
20859 ? TargetLowering::AtomicExpansionKind::None
20860 : TargetLowering::AtomicExpansionKind::CmpXChg;
20861}
20862
20863/// Return if a flat address space atomicrmw can access private memory.
20864static bool flatInstrMayAccessPrivate(const Instruction *I) {
20865 const MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_noalias_addrspace);
20866 return !MD ||
20867 !AMDGPU::hasValueInRangeLikeMetadata(MD: *MD, Val: AMDGPUAS::PRIVATE_ADDRESS);
20868}
20869
20870static TargetLowering::AtomicExpansionKind
20871getPrivateAtomicExpansionKind(const GCNSubtarget &STI) {
20872 // For GAS, lower to flat atomic.
20873 return STI.hasGloballyAddressableScratch()
20874 ? TargetLowering::AtomicExpansionKind::CustomExpand
20875 : TargetLowering::AtomicExpansionKind::NotAtomic;
20876}
20877
20878TargetLowering::AtomicExpansionKind
20879SITargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
20880 unsigned AS = RMW->getPointerAddressSpace();
20881 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
20882 return getPrivateAtomicExpansionKind(STI: *getSubtarget());
20883
20884 // 64-bit flat atomics that dynamically reside in private memory will silently
20885 // be dropped.
20886 //
20887 // Note that we will emit a new copy of the original atomic in the expansion,
20888 // which will be incrementally relegalized.
20889 const DataLayout &DL = RMW->getFunction()->getDataLayout();
20890 if (AS == AMDGPUAS::FLAT_ADDRESS &&
20891 DL.getTypeSizeInBits(Ty: RMW->getType()) == 64 &&
20892 flatInstrMayAccessPrivate(I: RMW))
20893 return AtomicExpansionKind::CustomExpand;
20894
20895 auto ReportUnsafeHWInst = [=](TargetLowering::AtomicExpansionKind Kind) {
20896 OptimizationRemarkEmitter ORE(RMW->getFunction());
20897 ORE.emit(RemarkBuilder: [=]() {
20898 return emitAtomicRMWLegalRemark(RMW) << " due to an unsafe request.";
20899 });
20900 return Kind;
20901 };
20902
20903 auto SSID = RMW->getSyncScopeID();
20904 bool HasSystemScope =
20905 SSID == SyncScope::System ||
20906 SSID == RMW->getContext().getOrInsertSyncScopeID(SSN: *getAtomicScopeIRString(
20907 T: getTargetMachine().getTargetTriple(), S: AtomicScope::System,
20908 /*OneAddressSpace=*/IsSingleAddressSpace: true));
20909
20910 auto Op = RMW->getOperation();
20911 switch (Op) {
20912 case AtomicRMWInst::Xchg:
20913 // PCIe supports add and xchg for system atomics.
20914 return isAtomicRMWLegalXChgTy(RMW)
20915 ? TargetLowering::AtomicExpansionKind::None
20916 : TargetLowering::AtomicExpansionKind::CmpXChg;
20917 case AtomicRMWInst::Add:
20918 // PCIe supports add and xchg for system atomics.
20919 return atomicSupportedIfLegalIntType(RMW);
20920 case AtomicRMWInst::Sub:
20921 case AtomicRMWInst::And:
20922 case AtomicRMWInst::Or:
20923 case AtomicRMWInst::Xor:
20924 case AtomicRMWInst::Max:
20925 case AtomicRMWInst::Min:
20926 case AtomicRMWInst::UMax:
20927 case AtomicRMWInst::UMin:
20928 case AtomicRMWInst::UIncWrap:
20929 case AtomicRMWInst::UDecWrap:
20930 case AtomicRMWInst::USubCond:
20931 case AtomicRMWInst::USubSat: {
20932 if (Op == AtomicRMWInst::USubCond && !Subtarget->hasCondSubInsts())
20933 return AtomicExpansionKind::CmpXChg;
20934 if (Op == AtomicRMWInst::USubSat && !Subtarget->hasSubClampInsts())
20935 return AtomicExpansionKind::CmpXChg;
20936 if (Op == AtomicRMWInst::USubCond || Op == AtomicRMWInst::USubSat) {
20937 auto *IT = dyn_cast<IntegerType>(Val: RMW->getType());
20938 if (!IT || IT->getBitWidth() != 32)
20939 return AtomicExpansionKind::CmpXChg;
20940 }
20941
20942 if (AMDGPU::isFlatGlobalAddrSpace(AS) ||
20943 AS == AMDGPUAS::BUFFER_FAT_POINTER) {
20944 if (Subtarget->hasEmulatedSystemScopeAtomics())
20945 return atomicSupportedIfLegalIntType(RMW);
20946
20947 // On most subtargets, for atomicrmw operations other than add/xchg,
20948 // whether or not the instructions will behave correctly depends on where
20949 // the address physically resides and what interconnect is used in the
20950 // system configuration. On some some targets the instruction will nop,
20951 // and in others synchronization will only occur at degraded device scope.
20952 //
20953 // If the allocation is known local to the device, the instructions should
20954 // work correctly.
20955 if (RMW->hasMetadata(Kind: "amdgpu.no.remote.memory"))
20956 return atomicSupportedIfLegalIntType(RMW);
20957
20958 // If fine-grained remote memory works at device scope, we don't need to
20959 // do anything.
20960 if (!HasSystemScope &&
20961 Subtarget->hasAgentScopeFineGrainedRemoteMemoryAtomics())
20962 return atomicSupportedIfLegalIntType(RMW);
20963
20964 // If we are targeting a remote allocated address, it depends what kind of
20965 // allocation the address belongs to.
20966 //
20967 // If the allocation is fine-grained (in host memory, or in PCIe peer
20968 // device memory), the operation will fail depending on the target.
20969 //
20970 // Note fine-grained host memory access does work on APUs or if XGMI is
20971 // used, but we do not know if we are targeting an APU or the system
20972 // configuration from the ISA version/target-cpu.
20973 if (RMW->hasMetadata(Kind: "amdgpu.no.fine.grained.memory"))
20974 return atomicSupportedIfLegalIntType(RMW);
20975
20976 if (Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Or ||
20977 Op == AtomicRMWInst::Xor) {
20978 // Atomic sub/or/xor do not work over PCI express, but atomic add
20979 // does. InstCombine transforms these with 0 to or, so undo that.
20980 // Sub-word types are not selectable and take the cmpxchg expansion.
20981 if (const Constant *ConstVal = dyn_cast<Constant>(Val: RMW->getValOperand());
20982 ConstVal && ConstVal->isNullValue() &&
20983 isAtomicRMWLegalIntTy(Ty: RMW->getType()))
20984 return AtomicExpansionKind::CustomExpand;
20985 }
20986
20987 // If the allocation could be in remote, fine-grained memory, the rmw
20988 // instructions may fail. cmpxchg should work, so emit that. On some
20989 // system configurations, PCIe atomics aren't supported so cmpxchg won't
20990 // even work, so you're out of luck anyway.
20991
20992 // In summary:
20993 //
20994 // Cases that may fail:
20995 // - fine-grained pinned host memory
20996 // - fine-grained migratable host memory
20997 // - fine-grained PCIe peer device
20998 //
20999 // Cases that should work, but may be treated overly conservatively.
21000 // - fine-grained host memory on an APU
21001 // - fine-grained XGMI peer device
21002 return AtomicExpansionKind::CmpXChg;
21003 }
21004
21005 return atomicSupportedIfLegalIntType(RMW);
21006 }
21007 case AtomicRMWInst::FAdd: {
21008 Type *Ty = RMW->getType();
21009
21010 // TODO: Handle REGION_ADDRESS
21011 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
21012 // DS F32 FP atomics do respect the denormal mode, but the rounding mode
21013 // is fixed to round-to-nearest-even.
21014 //
21015 // F64 / PK_F16 / PK_BF16 never flush and are also fixed to
21016 // round-to-nearest-even.
21017 //
21018 // We ignore the rounding mode problem, even in strictfp. The C++ standard
21019 // suggests it is OK if the floating-point mode may not match the calling
21020 // thread.
21021 if (Ty->isFloatTy()) {
21022 return Subtarget->hasLDSFPAtomicAddF32() ? AtomicExpansionKind::None
21023 : AtomicExpansionKind::CmpXChg;
21024 }
21025
21026 if (Ty->isDoubleTy()) {
21027 // Ignores denormal mode, but we don't consider flushing mandatory.
21028 return Subtarget->hasLDSFPAtomicAddF64() ? AtomicExpansionKind::None
21029 : AtomicExpansionKind::CmpXChg;
21030 }
21031
21032 if (Subtarget->hasAtomicDsPkAdd16Insts() && isV2F16OrV2BF16(Ty))
21033 return AtomicExpansionKind::None;
21034
21035 return AtomicExpansionKind::CmpXChg;
21036 }
21037
21038 // LDS atomics respect the denormal mode from the mode register.
21039 //
21040 // Traditionally f32 global/buffer memory atomics would unconditionally
21041 // flush denormals, but newer targets do not flush. f64/f16/bf16 cases never
21042 // flush.
21043 //
21044 // On targets with flat atomic fadd, denormals would flush depending on
21045 // whether the target address resides in LDS or global memory. We consider
21046 // this flat-maybe-flush as will-flush.
21047 if (Ty->isFloatTy() &&
21048 !Subtarget->hasMemoryAtomicFaddF32DenormalSupport() &&
21049 !atomicIgnoresDenormalModeOrFPModeIsFTZ(RMW))
21050 return AtomicExpansionKind::CmpXChg;
21051
21052 // FIXME: These ReportUnsafeHWInsts are imprecise. Some of these cases are
21053 // safe. The message phrasing also should be better.
21054 if (globalMemoryFPAtomicIsLegal(Subtarget: *Subtarget, RMW, HasSystemScope)) {
21055 if (AS == AMDGPUAS::FLAT_ADDRESS) {
21056 // gfx942, gfx12
21057 if (Subtarget->hasAtomicFlatPkAdd16Insts() && isV2F16OrV2BF16(Ty))
21058 return ReportUnsafeHWInst(AtomicExpansionKind::None);
21059 } else if (AMDGPU::isExtendedGlobalAddrSpace(AS)) {
21060 // gfx90a, gfx942, gfx12
21061 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
21062 return ReportUnsafeHWInst(AtomicExpansionKind::None);
21063
21064 // gfx942, gfx12
21065 if (Subtarget->hasAtomicGlobalPkAddBF16Inst() && isV2BF16(Ty))
21066 return ReportUnsafeHWInst(AtomicExpansionKind::None);
21067 } else if (AS == AMDGPUAS::BUFFER_FAT_POINTER) {
21068 // gfx90a, gfx942, gfx12
21069 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
21070 return ReportUnsafeHWInst(AtomicExpansionKind::None);
21071
21072 // While gfx90a/gfx942 supports v2bf16 for global/flat, it does not for
21073 // buffer. gfx12 does have the buffer version.
21074 if (Subtarget->hasAtomicBufferPkAddBF16Inst() && isV2BF16(Ty))
21075 return ReportUnsafeHWInst(AtomicExpansionKind::None);
21076 }
21077
21078 // global and flat atomic fadd f64: gfx90a, gfx942.
21079 if (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() && Ty->isDoubleTy())
21080 return ReportUnsafeHWInst(AtomicExpansionKind::None);
21081
21082 if (AS != AMDGPUAS::FLAT_ADDRESS) {
21083 if (Ty->isFloatTy()) {
21084 // global/buffer atomic fadd f32 no-rtn: gfx908, gfx90a, gfx942,
21085 // gfx11+.
21086 if (RMW->use_empty() && Subtarget->hasAtomicFaddNoRtnInsts())
21087 return ReportUnsafeHWInst(AtomicExpansionKind::None);
21088 // global/buffer atomic fadd f32 rtn: gfx90a, gfx942, gfx11+.
21089 if (!RMW->use_empty() && Subtarget->hasAtomicFaddRtnInsts())
21090 return ReportUnsafeHWInst(AtomicExpansionKind::None);
21091 } else {
21092 // gfx908
21093 if (RMW->use_empty() &&
21094 Subtarget->hasAtomicBufferGlobalPkAddF16NoRtnInsts() &&
21095 isV2F16(Ty))
21096 return ReportUnsafeHWInst(AtomicExpansionKind::None);
21097 }
21098 }
21099
21100 // flat atomic fadd f32: gfx942, gfx11+.
21101 if (AS == AMDGPUAS::FLAT_ADDRESS && Ty->isFloatTy()) {
21102 if (Subtarget->hasFlatAtomicFaddF32Inst())
21103 return ReportUnsafeHWInst(AtomicExpansionKind::None);
21104
21105 // If it is in flat address space, and the type is float, we will try to
21106 // expand it, if the target supports global and lds atomic fadd. The
21107 // reason we need that is, in the expansion, we emit the check of
21108 // address space. If it is in global address space, we emit the global
21109 // atomic fadd; if it is in shared address space, we emit the LDS atomic
21110 // fadd.
21111 if (Subtarget->hasLDSFPAtomicAddF32()) {
21112 if (RMW->use_empty() && Subtarget->hasAtomicFaddNoRtnInsts())
21113 return AtomicExpansionKind::CustomExpand;
21114 if (!RMW->use_empty() && Subtarget->hasAtomicFaddRtnInsts())
21115 return AtomicExpansionKind::CustomExpand;
21116 }
21117 }
21118 }
21119
21120 return AtomicExpansionKind::CmpXChg;
21121 }
21122 case AtomicRMWInst::FMin:
21123 case AtomicRMWInst::FMax: {
21124 Type *Ty = RMW->getType();
21125
21126 // LDS float and double fmin/fmax were always supported.
21127 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
21128 return Ty->isFloatTy() || Ty->isDoubleTy() ? AtomicExpansionKind::None
21129 : AtomicExpansionKind::CmpXChg;
21130 }
21131
21132 if (globalMemoryFPAtomicIsLegal(Subtarget: *Subtarget, RMW, HasSystemScope)) {
21133 // For flat and global cases:
21134 // float, double in gfx7. Manual claims denormal support.
21135 // Removed in gfx8.
21136 // float, double restored in gfx10.
21137 // double removed again in gfx11, so only f32 for gfx11/gfx12.
21138 //
21139 // For gfx9, gfx90a and gfx942 support f64 for global (same as fadd), but
21140 // no f32.
21141 if (AS == AMDGPUAS::FLAT_ADDRESS) {
21142 if (Subtarget->hasAtomicFMinFMaxF32FlatInsts() && Ty->isFloatTy())
21143 return ReportUnsafeHWInst(AtomicExpansionKind::None);
21144 if (Subtarget->hasAtomicFMinFMaxF64FlatInsts() && Ty->isDoubleTy())
21145 return ReportUnsafeHWInst(AtomicExpansionKind::None);
21146 } else if (AMDGPU::isExtendedGlobalAddrSpace(AS) ||
21147 AS == AMDGPUAS::BUFFER_FAT_POINTER) {
21148 if (Subtarget->hasAtomicFMinFMaxF32GlobalInsts() && Ty->isFloatTy())
21149 return ReportUnsafeHWInst(AtomicExpansionKind::None);
21150 if (Subtarget->hasAtomicFMinFMaxF64GlobalInsts() && Ty->isDoubleTy())
21151 return ReportUnsafeHWInst(AtomicExpansionKind::None);
21152 }
21153 }
21154
21155 return AtomicExpansionKind::CmpXChg;
21156 }
21157 case AtomicRMWInst::Nand:
21158 case AtomicRMWInst::FSub:
21159 default:
21160 return AtomicExpansionKind::CmpXChg;
21161 }
21162
21163 llvm_unreachable("covered atomicrmw op switch");
21164}
21165
21166TargetLowering::AtomicExpansionKind
21167SITargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
21168 return LI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS
21169 ? getPrivateAtomicExpansionKind(STI: *getSubtarget())
21170 : AtomicExpansionKind::None;
21171}
21172
21173TargetLowering::AtomicExpansionKind
21174SITargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
21175 return SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS
21176 ? getPrivateAtomicExpansionKind(STI: *getSubtarget())
21177 : AtomicExpansionKind::None;
21178}
21179
21180TargetLowering::AtomicExpansionKind
21181SITargetLowering::shouldExpandAtomicCmpXchgInIR(
21182 const AtomicCmpXchgInst *CmpX) const {
21183 unsigned AddrSpace = CmpX->getPointerAddressSpace();
21184 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)
21185 return getPrivateAtomicExpansionKind(STI: *getSubtarget());
21186
21187 if (AddrSpace != AMDGPUAS::FLAT_ADDRESS || !flatInstrMayAccessPrivate(I: CmpX))
21188 return AtomicExpansionKind::None;
21189
21190 const DataLayout &DL = CmpX->getDataLayout();
21191
21192 Type *ValTy = CmpX->getNewValOperand()->getType();
21193
21194 // If a 64-bit flat atomic may alias private, we need to avoid using the
21195 // atomic in the private case.
21196 return DL.getTypeSizeInBits(Ty: ValTy) == 64 ? AtomicExpansionKind::CustomExpand
21197 : AtomicExpansionKind::None;
21198}
21199
21200const TargetRegisterClass *
21201SITargetLowering::getRegClassFor(MVT VT, bool isDivergent) const {
21202 const TargetRegisterClass *RC = TargetLoweringBase::getRegClassFor(VT, isDivergent: false);
21203 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
21204 if (RC == &AMDGPU::VReg_1RegClass && !isDivergent)
21205 return Subtarget->isWave64() ? &AMDGPU::SReg_64RegClass
21206 : &AMDGPU::SReg_32RegClass;
21207 if (!TRI->isSGPRClass(RC) && !isDivergent)
21208 return TRI->getEquivalentSGPRClass(VRC: RC);
21209 if (TRI->isSGPRClass(RC) && isDivergent) {
21210 if (Subtarget->hasGFX90AInsts())
21211 return TRI->getEquivalentAVClass(SRC: RC);
21212 return TRI->getEquivalentVGPRClass(SRC: RC);
21213 }
21214
21215 return RC;
21216}
21217
21218// FIXME: This is a workaround for DivergenceAnalysis not understanding always
21219// uniform values (as produced by the mask results of control flow intrinsics)
21220// used outside of divergent blocks. The phi users need to also be treated as
21221// always uniform.
21222//
21223// FIXME: DA is no longer in-use. Does this still apply to UniformityAnalysis?
21224static bool hasCFUser(const Value *V, SmallPtrSet<const Value *, 16> &Visited,
21225 unsigned WaveSize) {
21226 // FIXME: We assume we never cast the mask results of a control flow
21227 // intrinsic.
21228 // Early exit if the type won't be consistent as a compile time hack.
21229 IntegerType *IT = dyn_cast<IntegerType>(Val: V->getType());
21230 if (!IT || IT->getBitWidth() != WaveSize)
21231 return false;
21232
21233 if (!isa<Instruction>(Val: V))
21234 return false;
21235 if (!Visited.insert(Ptr: V).second)
21236 return false;
21237 bool Result = false;
21238 for (const auto *U : V->users()) {
21239 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: U)) {
21240 if (V == U->getOperand(i: 1)) {
21241 switch (Intrinsic->getIntrinsicID()) {
21242 default:
21243 Result = false;
21244 break;
21245 case Intrinsic::amdgcn_if_break:
21246 case Intrinsic::amdgcn_if:
21247 case Intrinsic::amdgcn_else:
21248 Result = true;
21249 break;
21250 }
21251 }
21252 if (V == U->getOperand(i: 0)) {
21253 switch (Intrinsic->getIntrinsicID()) {
21254 default:
21255 Result = false;
21256 break;
21257 case Intrinsic::amdgcn_end_cf:
21258 case Intrinsic::amdgcn_loop:
21259 Result = true;
21260 break;
21261 }
21262 }
21263 } else {
21264 Result = hasCFUser(V: U, Visited, WaveSize);
21265 }
21266 if (Result)
21267 break;
21268 }
21269 return Result;
21270}
21271
21272bool SITargetLowering::requiresUniformRegister(MachineFunction &MF,
21273 const Value *V) const {
21274 if (const CallInst *CI = dyn_cast<CallInst>(Val: V)) {
21275 if (CI->isInlineAsm()) {
21276 // FIXME: This cannot give a correct answer. This should only trigger in
21277 // the case where inline asm returns mixed SGPR and VGPR results, used
21278 // outside the defining block. We don't have a specific result to
21279 // consider, so this assumes if any value is SGPR, the overall register
21280 // also needs to be SGPR.
21281 const SIRegisterInfo *SIRI = Subtarget->getRegisterInfo();
21282 TargetLowering::AsmOperandInfoVector TargetConstraints = ParseConstraints(
21283 DL: MF.getDataLayout(), TRI: Subtarget->getRegisterInfo(), Call: *CI);
21284 for (auto &TC : TargetConstraints) {
21285 if (TC.Type == InlineAsm::isOutput) {
21286 ComputeConstraintToUse(OpInfo&: TC, Op: SDValue());
21287 const TargetRegisterClass *RC =
21288 getRegForInlineAsmConstraint(TRI_: SIRI, Constraint: TC.ConstraintCode,
21289 VT: TC.ConstraintVT)
21290 .second;
21291 if (RC && SIRI->isSGPRClass(RC))
21292 return true;
21293 }
21294 }
21295 }
21296 }
21297 SmallPtrSet<const Value *, 16> Visited;
21298 return hasCFUser(V, Visited, WaveSize: Subtarget->getWavefrontSize());
21299}
21300
21301bool SITargetLowering::hasMemSDNodeUser(SDNode *N) const {
21302 for (SDUse &Use : N->uses()) {
21303 if (MemSDNode *M = dyn_cast<MemSDNode>(Val: Use.getUser())) {
21304 if (getBasePtrIndex(N: M) == Use.getOperandNo())
21305 return true;
21306 }
21307 }
21308 return false;
21309}
21310
21311bool SITargetLowering::isReassocProfitable(SelectionDAG &DAG, SDValue N0,
21312 SDValue N1) const {
21313 if (!N0.hasOneUse())
21314 return false;
21315 // Take care of the opportunity to keep N0 uniform
21316 if (N0->isDivergent() || !N1->isDivergent())
21317 return true;
21318 // Check if we have a good chance to form the memory access pattern with the
21319 // base and offset
21320 return (DAG.isBaseWithConstantOffset(Op: N0) &&
21321 hasMemSDNodeUser(N: *N0->user_begin()));
21322}
21323
21324bool SITargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
21325 Register N0, Register N1) const {
21326 return MRI.hasOneNonDBGUse(RegNo: N0); // FIXME: handle regbanks
21327}
21328
21329MachineMemOperand::Flags
21330SITargetLowering::getTargetMMOFlags(const Instruction &I) const {
21331 // Propagate metadata set by AMDGPUAnnotateUniformValues to the MMO of a load.
21332 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
21333 if (I.getMetadata(Kind: "amdgpu.noclobber"))
21334 Flags |= MONoClobber;
21335 if (I.getMetadata(Kind: "amdgpu.last.use"))
21336 Flags |= MOLastUse;
21337 return Flags;
21338}
21339
21340void SITargetLowering::emitExpandAtomicAddrSpacePredicate(
21341 Instruction *AI) const {
21342 // Given: atomicrmw fadd ptr %addr, float %val ordering
21343 //
21344 // With this expansion we produce the following code:
21345 // [...]
21346 // %is.shared = call i1 @llvm.amdgcn.is.shared(ptr %addr)
21347 // br i1 %is.shared, label %atomicrmw.shared, label %atomicrmw.check.private
21348 //
21349 // atomicrmw.shared:
21350 // %cast.shared = addrspacecast ptr %addr to ptr addrspace(3)
21351 // %loaded.shared = atomicrmw fadd ptr addrspace(3) %cast.shared,
21352 // float %val ordering
21353 // br label %atomicrmw.phi
21354 //
21355 // atomicrmw.check.private:
21356 // %is.private = call i1 @llvm.amdgcn.is.private(ptr %int8ptr)
21357 // br i1 %is.private, label %atomicrmw.private, label %atomicrmw.global
21358 //
21359 // atomicrmw.private:
21360 // %cast.private = addrspacecast ptr %addr to ptr addrspace(5)
21361 // %loaded.private = load float, ptr addrspace(5) %cast.private
21362 // %val.new = fadd float %loaded.private, %val
21363 // store float %val.new, ptr addrspace(5) %cast.private
21364 // br label %atomicrmw.phi
21365 //
21366 // atomicrmw.global:
21367 // %cast.global = addrspacecast ptr %addr to ptr addrspace(1)
21368 // %loaded.global = atomicrmw fadd ptr addrspace(1) %cast.global,
21369 // float %val ordering
21370 // br label %atomicrmw.phi
21371 //
21372 // atomicrmw.phi:
21373 // %loaded.phi = phi float [ %loaded.shared, %atomicrmw.shared ],
21374 // [ %loaded.private, %atomicrmw.private ],
21375 // [ %loaded.global, %atomicrmw.global ]
21376 // br label %atomicrmw.end
21377 //
21378 // atomicrmw.end:
21379 // [...]
21380 //
21381 //
21382 // For 64-bit atomics which may reside in private memory, we perform a simpler
21383 // version that only inserts the private check, and uses the flat operation.
21384
21385 IRBuilder<> Builder(AI);
21386 LLVMContext &Ctx = Builder.getContext();
21387
21388 auto *RMW = dyn_cast<AtomicRMWInst>(Val: AI);
21389 const unsigned PtrOpIdx = RMW ? AtomicRMWInst::getPointerOperandIndex()
21390 : AtomicCmpXchgInst::getPointerOperandIndex();
21391 Value *Addr = AI->getOperand(i: PtrOpIdx);
21392
21393 /// TODO: Only need to check private, then emit flat-known-not private (no
21394 /// need for shared block, or cast to global).
21395 AtomicCmpXchgInst *CX = dyn_cast<AtomicCmpXchgInst>(Val: AI);
21396
21397 Align Alignment;
21398 if (RMW)
21399 Alignment = RMW->getAlign();
21400 else if (CX)
21401 Alignment = CX->getAlign();
21402 else
21403 llvm_unreachable("unhandled atomic operation");
21404
21405 // FullFlatEmulation is true if we need to issue the private, shared, and
21406 // global cases.
21407 //
21408 // If this is false, we are only dealing with the flat-targeting-private case,
21409 // where we only insert a check for private and still use the flat instruction
21410 // for global and shared.
21411
21412 bool FullFlatEmulation =
21413 RMW && RMW->getOperation() == AtomicRMWInst::FAdd &&
21414 ((Subtarget->hasAtomicFaddInsts() && RMW->getType()->isFloatTy()) ||
21415 (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() &&
21416 RMW->getType()->isDoubleTy()));
21417
21418 // If the return value isn't used, do not introduce a false use in the phi.
21419 bool ReturnValueIsUsed = !AI->use_empty();
21420
21421 BasicBlock *BB = Builder.GetInsertBlock();
21422 Function *F = BB->getParent();
21423 BasicBlock *ExitBB =
21424 BB->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "atomicrmw.end");
21425 BasicBlock *SharedBB = nullptr;
21426
21427 BasicBlock *CheckPrivateBB = BB;
21428 if (FullFlatEmulation) {
21429 SharedBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.shared", Parent: F, InsertBefore: ExitBB);
21430 CheckPrivateBB =
21431 BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.check.private", Parent: F, InsertBefore: ExitBB);
21432 }
21433
21434 BasicBlock *PrivateBB =
21435 BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.private", Parent: F, InsertBefore: ExitBB);
21436 BasicBlock *GlobalBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.global", Parent: F, InsertBefore: ExitBB);
21437 BasicBlock *PhiBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.phi", Parent: F, InsertBefore: ExitBB);
21438
21439 std::prev(x: BB->end())->eraseFromParent();
21440 Builder.SetInsertPoint(BB);
21441
21442 Value *LoadedShared = nullptr;
21443 if (FullFlatEmulation) {
21444 Value *IsShared = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_is_shared,
21445 Args: {Addr}, FMFSource: nullptr, Name: "is.shared");
21446 Builder.CreateCondBr(Cond: IsShared, True: SharedBB, False: CheckPrivateBB);
21447 Builder.SetInsertPoint(SharedBB);
21448 Value *CastToLocal = Builder.CreateAddrSpaceCast(
21449 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::LOCAL_ADDRESS));
21450
21451 Instruction *Clone = AI->clone();
21452 Clone->insertInto(ParentBB: SharedBB, It: SharedBB->end());
21453 Clone->getOperandUse(i: PtrOpIdx).set(CastToLocal);
21454 LoadedShared = Clone;
21455
21456 Builder.CreateBr(Dest: PhiBB);
21457 Builder.SetInsertPoint(CheckPrivateBB);
21458 }
21459
21460 Value *IsPrivate = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_is_private,
21461 Args: {Addr}, FMFSource: nullptr, Name: "is.private");
21462 Builder.CreateCondBr(Cond: IsPrivate, True: PrivateBB, False: GlobalBB);
21463
21464 Builder.SetInsertPoint(PrivateBB);
21465
21466 Value *CastToPrivate = Builder.CreateAddrSpaceCast(
21467 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::PRIVATE_ADDRESS));
21468
21469 Value *LoadedPrivate;
21470 if (RMW) {
21471 LoadedPrivate = Builder.CreateAlignedLoad(
21472 Ty: RMW->getType(), Ptr: CastToPrivate, Align: RMW->getAlign(), isVolatile: RMW->isVolatile(),
21473 Name: "loaded.private");
21474
21475 Value *NewVal = buildAtomicRMWValue(Op: RMW->getOperation(), Builder,
21476 Loaded: LoadedPrivate, Val: RMW->getValOperand());
21477
21478 Builder.CreateAlignedStore(Val: NewVal, Ptr: CastToPrivate, Align: RMW->getAlign(),
21479 isVolatile: RMW->isVolatile());
21480 } else {
21481 auto [ResultLoad, Equal] = buildCmpXchgValue(
21482 Builder, Ptr: CastToPrivate, Cmp: CX->getCompareOperand(), Val: CX->getNewValOperand(),
21483 Alignment: CX->getAlign(), IsVolatile: CX->isVolatile());
21484
21485 Value *Insert = Builder.CreateInsertValue(Agg: PoisonValue::get(T: CX->getType()),
21486 Val: ResultLoad, Idxs: 0);
21487 LoadedPrivate = Builder.CreateInsertValue(Agg: Insert, Val: Equal, Idxs: 1);
21488 }
21489
21490 Builder.CreateBr(Dest: PhiBB);
21491
21492 Builder.SetInsertPoint(GlobalBB);
21493
21494 // Continue using a flat instruction if we only emitted the check for private.
21495 Instruction *LoadedGlobal = AI;
21496 if (FullFlatEmulation) {
21497 Value *CastToGlobal = Builder.CreateAddrSpaceCast(
21498 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::GLOBAL_ADDRESS));
21499 AI->getOperandUse(i: PtrOpIdx).set(CastToGlobal);
21500 }
21501
21502 AI->removeFromParent();
21503 AI->insertInto(ParentBB: GlobalBB, It: GlobalBB->end());
21504
21505 // The new atomicrmw may go through another round of legalization later.
21506 if (!FullFlatEmulation) {
21507 // We inserted the runtime check already, make sure we do not try to
21508 // re-expand this.
21509 // TODO: Should union with any existing metadata.
21510 MDBuilder MDB(F->getContext());
21511 MDNode *RangeNotPrivate =
21512 MDB.createRange(Lo: APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
21513 Hi: APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
21514 LoadedGlobal->setMetadata(KindID: LLVMContext::MD_noalias_addrspace,
21515 Node: RangeNotPrivate);
21516 }
21517
21518 Builder.CreateBr(Dest: PhiBB);
21519
21520 Builder.SetInsertPoint(PhiBB);
21521
21522 if (ReturnValueIsUsed) {
21523 PHINode *Loaded = Builder.CreatePHI(Ty: AI->getType(), NumReservedValues: 3);
21524 AI->replaceAllUsesWith(V: Loaded);
21525 if (FullFlatEmulation)
21526 Loaded->addIncoming(V: LoadedShared, BB: SharedBB);
21527 Loaded->addIncoming(V: LoadedPrivate, BB: PrivateBB);
21528 Loaded->addIncoming(V: LoadedGlobal, BB: GlobalBB);
21529 Loaded->takeName(V: AI);
21530 }
21531
21532 Builder.CreateBr(Dest: ExitBB);
21533}
21534
21535static void convertScratchAtomicToFlatAtomic(Instruction *I,
21536 unsigned PtrOpIdx) {
21537 Value *PtrOp = I->getOperand(i: PtrOpIdx);
21538 assert(PtrOp->getType()->getPointerAddressSpace() ==
21539 AMDGPUAS::PRIVATE_ADDRESS);
21540
21541 Type *FlatPtr = PointerType::get(C&: I->getContext(), AddressSpace: AMDGPUAS::FLAT_ADDRESS);
21542 Value *ASCast = CastInst::CreatePointerCast(S: PtrOp, Ty: FlatPtr, Name: "scratch.ascast",
21543 InsertBefore: I->getIterator());
21544 I->setOperand(i: PtrOpIdx, Val: ASCast);
21545}
21546
21547void SITargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
21548 AtomicRMWInst::BinOp Op = AI->getOperation();
21549
21550 if (AI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21551 return convertScratchAtomicToFlatAtomic(I: AI, PtrOpIdx: AI->getPointerOperandIndex());
21552
21553 if (Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Or ||
21554 Op == AtomicRMWInst::Xor) {
21555 if (const auto *ConstVal = dyn_cast<Constant>(Val: AI->getValOperand());
21556 ConstVal && ConstVal->isNullValue() &&
21557 isAtomicRMWLegalIntTy(Ty: AI->getType())) {
21558 // atomicrmw or %ptr, 0 -> atomicrmw add %ptr, 0
21559 AI->setOperation(AtomicRMWInst::Add);
21560
21561 // We may still need the private-alias-flat handling below.
21562
21563 // TODO: Skip this for cases where we cannot access remote memory.
21564 }
21565 }
21566
21567 // The non-flat expansions should only perform the de-canonicalization of
21568 // identity values.
21569 if (AI->getPointerAddressSpace() != AMDGPUAS::FLAT_ADDRESS)
21570 return;
21571
21572 emitExpandAtomicAddrSpacePredicate(AI);
21573}
21574
21575void SITargetLowering::emitExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) const {
21576 if (CI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21577 return convertScratchAtomicToFlatAtomic(I: CI, PtrOpIdx: CI->getPointerOperandIndex());
21578
21579 emitExpandAtomicAddrSpacePredicate(AI: CI);
21580}
21581
21582void SITargetLowering::emitExpandAtomicLoad(LoadInst *LI) const {
21583 if (LI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21584 return convertScratchAtomicToFlatAtomic(I: LI, PtrOpIdx: LI->getPointerOperandIndex());
21585
21586 llvm_unreachable(
21587 "Expand Atomic Load only handles SCRATCH -> FLAT conversion");
21588}
21589
21590void SITargetLowering::emitExpandAtomicStore(StoreInst *SI) const {
21591 if (SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21592 return convertScratchAtomicToFlatAtomic(I: SI, PtrOpIdx: SI->getPointerOperandIndex());
21593
21594 llvm_unreachable(
21595 "Expand Atomic Store only handles SCRATCH -> FLAT conversion");
21596}
21597